Support wiki-based repository index

New Plugins:
- bpkg-update: sync repo listings locally
- bpkg-list: list the repo names, with -d flag for details
- bpkg-utils: source-only shared utilities for bpkg plugins

Change-Id: I9ef04a86efa35c61cabd7c4ecfa7c7a134b41c6c

Makefile

@@ -4,7 +4,7 @@ BIN    ?= bpkg
 PREFIX ?= /usr/local
 
 # All 'bpkg' supported commands
-CMDS = json install package term suggest init
+CMDS = json install package term suggest init utils update list
 
 install: uninstall
 	@echo "  info: Installing $(PREFIX)/bin/$(BIN)..."

bpkg-list

@@ -0,0 +1 @@
+lib/list/list.sh

bpkg-update

@@ -0,0 +1 @@
+lib/update/update.sh

bpkg-utils

@@ -0,0 +1 @@
+lib/utils/utils.sh

lib/install/install.sh

@@ -361,5 +361,8 @@ if [[ ${BASH_SOURCE[0]} != $0 ]]; then
   export -f bpkg_install
 elif validate_parameters; then
   bpkg_install "${@}"
+  exit $?
+else
+  #param validation failed
+  exit $?
 fi
-exit $?

lib/list/list.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+VERSION="0.0.1"
+
+if ! type -f bpkg-utils &>/dev/null; then
+  echo "error: bpkg-utils not found, aborting"
+  exit 1
+else
+  source `which bpkg-utils`
+fi
+
+bpkg_initrc
+
+usage () {
+  echo "bpkg-list [-h|--help] [-V|--version] [-d|--details]"
+  echo
+  echo "List all known bash packages from the repo.  You first must run \`bpkg update' to sync the repo locally."
+
+  echo "Options:"
+  echo "  --help|-h     Print this help dialogue"
+  echo "  --version|-V  Print version and exit"
+  echo "  --details|-d  More verbose output"
+}
+
+bpkg_list () {
+  local verbose=0
+  for opt in "${@}"; do
+    case "$opt" in 
+      -V|--version)
+        echo "${VERSION}"
+        return 0
+        ;;
+      -h|--help)
+        usage
+        return 0
+        ;;
+      -d|--details)
+        verbose=1
+        ;;
+      *)
+        if [ "${opt:0:1}" == "-" ]; then
+          bpkg_error "unknown option: $opt"
+          return 1
+        fi
+    esac
+  done
+
+  local i=0
+  for remote in "${BPKG_REMOTES[@]}"; do
+    local git_remote="${BPKG_GIT_REMOTES[$i]}"
+    bpkg_select_remote "$remote" "$git_remote"
+    if [ ! -f "$BPKG_REMOTE_INDEX_FILE" ]; then
+      bpkg_warn "no index file found for remote: ${remote}"
+      bpkg_warn "You should run \`bpkg update' before running this command."
+      continue
+    fi
+    OLDIFS="$IFS"
+    IFS=$'\n'
+    for line in $(cat $BPKG_REMOTE_INDEX_FILE); do
+      local name=$(echo "$line" | cut -d\| -f1 | tr -d ' ')
+      local desc=$(echo "$line" | cut -d\| -f2)
+      local host=$BPKG_REMOTE_HOST
+      if [ "$verbose" == "1" ]; then
+        echo "$name [$host] - $desc"
+      else
+        echo $name
+      fi
+    done
+    IFS="$OLDIFS"
+    i=$((i+1))
+  done
+}
+
+
+if [[ ${BASH_SOURCE[0]} != $0 ]]; then
+  export -f bpkg_list
+elif bpkg_validate; then
+  bpkg_list "${@}"
+fi

lib/update/update.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+VERSION="0.0.1"
+
+if ! type -f bpkg-utils &>/dev/null; then
+  echo "error: bpkg-utils not found, aborting"
+  exit 1
+else
+  source `which bpkg-utils`
+fi
+
+bpkg_initrc
+
+usage () {
+  echo "bpkg-update [-h] [-V]"
+  echo
+  echo "Update local bpkg index for listing and searching packages"
+}
+
+bpkg_update_remote() {
+  local remote=$1
+  local git_remote=$2
+  local wiki_url=""
+  local wiki=""
+
+  bpkg_select_remote "$remote" "$git_remote"
+
+  local index=$BPKG_REMOTE_INDEX
+  mkdir -p "$index"
+
+  local index_file="$index/index.txt"
+  local auth=$BPKG_CURL_AUTH_PARAM
+
+  if [ "$auth" == "" ]; then
+    wiki_url="$BPKG_REMOTE/wiki/bpkg/bpkg/index.md"
+  else
+    # GHE wiki urls have a unique path structure
+    wiki_url="$BPKG_REMOTE/raw/wiki/bpkg/bpkg/index.md"
+  fi
+
+  #echo "curl -slo- $auth '$wiki_url' | grep -o '\[.*\](.*).*'"
+  repo_list=$(curl -sLo- $auth "$wiki_url" | grep -o '\[.*\](.*).*' | sed 's/\[\(.*\)\](.*)[ \-]*/\1|/' )
+
+  num_repos=$(echo "$repo_list" | wc -l | tr -d ' ')
+  bpkg_info "indexing ${num_repos} repos from $BPKG_REMOTE_HOST to $index_file"
+  echo "$repo_list" > $index_file
+}
+
+bpkg_update () {
+  for opt in "${@}"; do
+    case "$opt" in
+      -V|--version)
+        echo "${VERSION}"
+        return 0
+        ;;
+      -h|--help)
+        usage
+        return 0
+        ;;
+      *)
+        if [ "${opt:0:1}" == '-' ]; then
+          bpkg_error "unknown option: $opt"
+          return 1
+        fi
+        ;;
+    esac
+  done
+
+  local let i=0
+  for remote in "${BPKG_REMOTES[@]}"; do
+    local git_remote=${BPKG_GIT_REMOTES[$i]}
+    bpkg_update_remote "$remote" "$git_remote"
+    i=$((i+1))
+  done
+}
+
+if [[ ${BASH_SOURCE[0]} != $0 ]]; then
+  export -f bpkg_update
+elif bpkg_validate; then
+  bpkg_update "${@}"
+fi
+

lib/utils/utils.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+## Collection of shared bpkg functions
+
+## Init local config and set environmental defaults
+bpkg_initrc() {
+  local config=${BPKG_CONFIG:-"$HOME/.bpkgrc"}
+  [ -f "$config" ] && source "$config"
+  ## set defaults
+  if [ ${#BPKG_REMOTES[@]} -eq 0 ]; then
+    BPKG_REMOTES[0]=${BPKG_REMOTE-https://raw.githubusercontent.com}
+    BPKG_GIT_REMOTES[0]=${BPKG_GIT_REMOTE-https://github.com}
+  fi
+  BPKG_USER="${BPKG_USER:-"bpkg"}"
+  BPKG_INDEX=${BPKG_INDEX-"$HOME/.bpkg/index"}
+}
+
+## check parameter consistency
+bpkg_validate () {
+  bpkg_initrc
+  if [ ${#BPKG_GIT_REMOTES[@]} -ne ${#BPKG_REMOTES[@]} ]; then
+    mesg='BPKG_GIT_REMOTES[%d] differs in size from BPKG_REMOTES[%d] array'
+    fmesg=$(printf "$mesg" "${#BPKG_GIT_REMOTES[@]}" "${#BPKG_REMOTES[@]}")
+    error "$fmesg"
+    return 1
+  fi
+  return 0
+}
+
+## format and output message
+bpkg_message () {
+  if type -f bpkg-term > /dev/null 2>&1; then
+    bpkg-term color "${1}"
+  fi
+
+  shift
+  printf "    ${1}"
+  shift
+
+  if type -f bpkg-term > /dev/null 2>&1; then
+    bpkg-term reset
+  fi
+
+  printf ": "
+
+  if type -f bpkg-term > /dev/null 2>&1; then
+    bpkg-term reset
+    bpkg-term bright
+  fi
+
+  printf "%s\n" "${@}"
+
+  if type -f bpkg-term > /dev/null 2>&1; then
+    bpkg-term reset
+  fi
+}
+
+## output error
+bpkg_error () {
+  {
+    bpkg_message "red" "error" "${@}"
+  } >&2
+}
+
+## output warning
+bpkg_warn () {
+  {
+    bpkg_message "yellow" "warn" "${@}"
+  } >&2
+}
+
+## output info
+bpkg_info () {
+  local title="info"
+  if (( "${#}" > 1 )); then
+    title="${1}"
+    shift
+  fi
+  bpkg_message "cyan" "${title}" "${@}"
+}
+
+## takes a remote and git-remote and sets the globals:
+##  BPKG_REMOTE: remote URI
+##  BPKG_GIT_REMOTE: git remote with oauth info embedded,
+##  BPKG_OAUTH_TOKEN: token for x-oauth-basic
+bpkg_select_remote () {
+  local remote=$1
+  local git_remote=$2
+  BPKG_REMOTE_HOST=$(echo "$git_remote" | sed 's/.*:\/\///' | sed 's/\/$//' | tr '/' '_')
+  BPKG_REMOTE_INDEX="$BPKG_INDEX/$BPKG_REMOTE_HOST"
+  BPKG_REMOTE_INDEX_FILE="$BPKG_REMOTE_INDEX/index.txt"
+  BPKG_OAUTH_TOKEN=""
+  BPKG_CURL_AUTH_PARAM=""
+  if [ "${remote:0:10}" == "raw-oauth|" ]; then
+    OLDIFS="${IFS}"
+    IFS="|"
+    local remote_parts=($remote)
+    IFS="${OLDIFS}"
+    BPKG_OAUTH_TOKEN=${remote_parts[1]}
+    BPKG_CURL_AUTH_PARAM="-u $BPKG_OAUTH_TOKEN:x-oauth-basic"
+    BPKG_REMOTE=${remote_parts[2]}
+    if [[ "$git_remote" == https://* ]] && [[ "$git_remote" != *x-oauth-basic* ]] && [[ "$git_remote" != *${BPKG_OAUTH_TOKEN}* ]]; then
+      git_remote=${git_remote/https:\/\//https:\/\/$BPKG_OAUTH_TOKEN:x-oauth-basic@}
+    fi
+  else 
+    BPKG_REMOTE=$remote
+  fi
+  BPKG_GIT_REMOTE=$git_remote
+}
+
+## given a user and name, sets BPKG_RAW_PATH using the available
+## BPKG_REMOTE and BPKG_OAUTH_TOKEN details
+bpkg_select_raw_path() {
+  local user=$1
+  local name=$2
+  if [ "$BPKG_OAUTH_TOKEN" == "" ]; then
+    BPKG_RAW_PATH="$BPKG_REMOTE/$user/$name"
+  else
+    BPKG_RAW_PATH="$BPKG_REMOTE/$user/$name/raw"
+  fi
+  return 0
+}
+
+export -f bpkg_initrc
+export -f bpkg_validate
+
+export -f bpkg_message
+export -f bpkg_warn
+export -f bpkg_error
+export -f bpkg_info
+
+export -f bpkg_select_remote
+export -f bpkg_select_raw_path