Merge #118: Add verify commits

e1a5744 devtools: Auto-set branch to merge to in github-merge (Wladimir J. van der Laan)
7d0542c devtools: make github-merge.py use py3 (Wladimir J. van der Laan)
31f53d8 [copyright] add MIT license headers to .sh scripts where missing (isle2983)
2862e18 Add README for verify-commits (Peter Todd)
c2d1d62 Remove pointless warning (Peter Todd)
befe6a5 Make verify-commits path-independent (Matt Corallo)
acb55dd Make verify-commits POSIX-compliant (Matt Corallo)
fe83a6d Allow to whitelist commits signed with a revoked key (Matt Corallo)
e1873f6 Adjust verify-commits to gitian-builder (MarcoFalke)
3be7a42 Fix pre-push-hook regexes (Matt Corallo)
73f6969 Add script to verify all merge commits are signed (Matt Corallo)

contrib/README.md

@@ -0,0 +1,9 @@
+Repository Tools
+---------------------
+
+### [Developer tools](/contrib/devtools) ###
+Specific tools for developers working on this repository.
+Contains the script `github-merge.py` for merging github pull requests securely and signing them using GPG.
+
+### [Verify-Commits](/contrib/verify-commits) ###
+Tool to verify that every merge commit was signed by a developer using the above `github-merge.py` script.

contrib/devtools/github-merge.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # Copyright (c) 2016 Bitcoin Core Developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -19,6 +19,11 @@ import os,sys
 from sys import stdin,stdout,stderr
 import argparse
 import subprocess
+import json,codecs
+try:
+    from urllib.request import Request,urlopen
+except:
+    from urllib2 import Request,urlopen
 
 # External tools (can be overridden using environment)
 GIT = os.getenv('GIT','git')
@@ -38,38 +43,39 @@ def git_config_get(option, default=None):
     Get named configuration option from git repository.
     '''
     try:
-        return subprocess.check_output([GIT,'config','--get',option]).rstrip()
+        return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8')
     except subprocess.CalledProcessError as e:
         return default
 
-def retrieve_pr_title(repo,pull):
+def retrieve_pr_info(repo,pull):
     '''
-    Retrieve pull request title from github.
+    Retrieve pull request information from github.
     Return None if no title can be found, or an error happens.
     '''
-    import urllib2,json
     try:
-        req = urllib2.Request("https://api.github.com/repos/"+repo+"/pulls/"+pull)
+        req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull)
-        result = urllib2.urlopen(req)
+        result = urlopen(req)
-        result = json.load(result)
+        reader = codecs.getreader('utf-8')
-        return result['title']
+        obj = json.load(reader(result))
+        return obj
     except Exception as e:
-        print('Warning: unable to retrieve pull title from github: %s' % e)
+        print('Warning: unable to retrieve pull information from github: %s' % e)
         return None
 
 def ask_prompt(text):
     print(text,end=" ",file=stderr)
+    stderr.flush()
     reply = stdin.readline().rstrip()
     print("",file=stderr)
     return reply
 
-def parse_arguments(branch):
+def parse_arguments():
     epilog = '''
         In addition, you can set the following git configuration variables:
         githubmerge.repository (mandatory),
         user.signingkey (mandatory),
         githubmerge.host (default: git@github.com),
-        githubmerge.branch (default: master),
+        githubmerge.branch (no default),
         githubmerge.testcmd (default: none).
     '''
     parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests',
@@ -77,14 +83,14 @@ def parse_arguments(branch):
     parser.add_argument('pull', metavar='PULL', type=int, nargs=1,
         help='Pull request ID to merge')
     parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?',
-        default=branch, help='Branch to merge against (default: '+branch+')')
+        default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')')
     return parser.parse_args()
 
 def main():
     # Extract settings from git repo
     repo = git_config_get('githubmerge.repository')
     host = git_config_get('githubmerge.host','git@github.com')
-    branch = git_config_get('githubmerge.branch','master')
+    opt_branch = git_config_get('githubmerge.branch',None)
     testcmd = git_config_get('githubmerge.testcmd')
     signingkey = git_config_get('user.signingkey')
     if repo is None:
@@ -99,9 +105,20 @@ def main():
     host_repo = host+":"+repo # shortcut for push/pull target
 
     # Extract settings from command line
-    args = parse_arguments(branch)
+    args = parse_arguments()
     pull = str(args.pull[0])
-    branch = args.branch
+
+    # Receive pull information from github
+    info = retrieve_pr_info(repo,pull)
+    if info is None:
+        exit(1)
+    title = info['title']
+    # precedence order for destination branch argument:
+    #   - command line argument
+    #   - githubmerge.branch setting
+    #   - base branch for pull (as retrieved from github)
+    #   - 'master'
+    branch = args.branch or opt_branch or info['base']['ref'] or 'master'
 
     # Initialize source branches
     head_branch = 'pull/'+pull+'/head'
@@ -141,7 +158,6 @@ def main():
 
     try:
         # Create unsigned merge commit.
-        title = retrieve_pr_title(repo,pull)
         if title:
             firstline = 'Merge #%s: %s' % (pull,title)
         else:
@@ -159,7 +175,7 @@ def main():
             print("ERROR: Creating merge failed (already merged?).",file=stderr)
             exit(4)
 
-        print('%s#%s%s %s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title))
+        print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET))
         subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch])
         print()
         # Run test command if configured.

contrib/verify-commits/README.md

@@ -0,0 +1,26 @@
+Tooling for verification of PGP signed commits
+----------------------------------------------
+
+This is an incomplete work in progress, but currently includes a pre-push hook
+script (`pre-push-hook.sh`) for maintainers to ensure that their own commits
+are PGP signed (nearly always merge commits), as well as a script to verify
+commits against a trusted keys list.
+
+
+Using verify-commits.sh safely
+------------------------------
+
+Remember that you can't use an untrusted script to verify itself. This means
+that checking out code, then running `verify-commits.sh` against `HEAD` is
+_not_ safe, because the version of `verify-commits.sh` that you just ran could
+be backdoored. Instead, you need to use a trusted version of verify-commits
+prior to checkout to make sure you're checking out only code signed by trusted
+keys:
+
+    git fetch origin && \
+      ./contrib/verify-commits/verify-commits.sh origin/master && \
+      git checkout origin/master
+
+Note that the above isn't a good UI/UX yet, and needs significant improvements
+to make it more convenient and reduce the chance of errors; pull-reqs
+improving this process would be much appreciated.

contrib/verify-commits/gpg.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Copyright (c) 2014-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+INPUT=$(cat /dev/stdin)
+VALID=false
+REVSIG=false
+IFS='
+'
+for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do
+	case "$LINE" in
+	"[GNUPG:] VALIDSIG "*)
+		while read KEY; do
+			case "$LINE" in "[GNUPG:] VALIDSIG $KEY "*) VALID=true;; esac
+		done < ./contrib/verify-commits/trusted-keys
+		;;
+	"[GNUPG:] REVKEYSIG "*)
+		[ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1
+		while read KEY; do
+			case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY#????????????????????????} "*)
+				REVSIG=true
+				GOODREVSIG="[GNUPG:] GOODSIG ${KEY#????????????????????????} "
+			esac
+		done < ./contrib/verify-commits/trusted-keys
+		;;
+	esac
+done
+if ! $VALID; then
+	exit 1
+fi
+if $VALID && $REVSIG; then
+	echo "$INPUT" | gpg --trust-model always "$@" | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" 2>/dev/null
+	echo "$GOODREVSIG"
+else
+	echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null
+fi

contrib/verify-commits/pre-push-hook.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright (c) 2014-2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)devrandom/gitian-builder(.git)?$ ]]; then
+    exit 0
+fi
+
+while read LINE; do
+    set -- A $LINE
+    if [ "$4" != "refs/heads/master" ]; then
+        continue
+    fi
+    if ! ./contrib/verify-commits/verify-commits.sh $3 > /dev/null 2>&1; then
+        echo "ERROR: A commit is not signed, can't push"
+        ./contrib/verify-commits/verify-commits.sh
+        exit 1
+    fi
+done < /dev/stdin

contrib/verify-commits/trusted-git-root

@@ -0,0 +1 @@
+bb4f92f6cbde6ee78e39ae35b0934da3b55e154d

contrib/verify-commits/trusted-keys

@@ -0,0 +1 @@
+498FA3769A88C4AD1B187A7428EB4B0FB7AAF6B0

contrib/verify-commits/verify-commits.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+# Copyright (c) 2014-2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Not technically POSIX-compliant due to use of "local", but almost every
+# shell anyone uses today supports it, so its probably fine
+
+DIR=$(dirname "$0")
+[ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0")
+
+VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root")
+REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits")
+
+HAVE_FAILED=false
+IS_SIGNED () {
+	if [ $1 = $VERIFIED_ROOT ]; then
+		return 0;
+	fi
+	if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then
+		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1
+	else
+		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0
+	fi
+	if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then
+		return 1;
+	fi
+	local PARENTS
+	PARENTS=$(git show -s --format=format:%P $1)
+	for PARENT in $PARENTS; do
+		if IS_SIGNED $PARENT > /dev/null; then
+			return 0;
+		fi
+	done
+	if ! "$HAVE_FAILED"; then
+		echo "No parent of $1 was signed with a trusted key!" > /dev/stderr
+		echo "Parents are:" > /dev/stderr
+		for PARENT in $PARENTS; do
+			git show -s $PARENT > /dev/stderr
+		done
+		HAVE_FAILED=true
+	fi
+	return 1;
+}
+
+if [ x"$1" = "x" ]; then
+	TEST_COMMIT="HEAD"
+else
+	TEST_COMMIT="$1"
+fi
+
+IS_SIGNED "$TEST_COMMIT"
+RES=$?
+if [ "$RES" = 1 ]; then
+	if ! "$HAVE_FAILED"; then
+		echo "$TEST_COMMIT was not signed with a trusted key!"
+	fi
+else
+	echo "There is a valid path from $TEST_COMMIT to $VERIFIED_ROOT where all commits are signed!"
+fi
+
+exit $RES