From 992efcf3a5b4a2eb7112c904d495ecc70cb1b149 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 19 Jan 2016 09:53:26 +0100 Subject: [PATCH 1/5] devtools: replace github-merge with python version This is meant to be a direct translation of the bash script, with the difference that it retrieves the PR title from github, thus creating pull messages like: Merge #12345: Expose transaction temperature over RPC --- contrib/devtools/README.md | 37 +++++ contrib/devtools/github-merge.py | 223 +++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 contrib/devtools/README.md create mode 100755 contrib/devtools/github-merge.py diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md new file mode 100644 index 0000000..c1a13e6 --- /dev/null +++ b/contrib/devtools/README.md @@ -0,0 +1,37 @@ +Contents +======== +This directory contains tools for developers working on this repository. + +github-merge.py +=============== + +A small script to automate merging pull-requests securely and sign them with GPG. + +For example: + + ./github-merge.py 3077 + +(in any git repository) will help you merge pull request #3077 for the +bitcoin/bitcoin repository. + +What it does: +* Fetch master and the pull request. +* Locally construct a merge commit. +* Show the diff that merge results in. +* Ask you to verify the resulting source tree (so you can do a make +check or whatever). +* Ask you whether to GPG sign the merge commit. +* Ask you whether to push the result upstream. + +This means that there are no potential race conditions (where a +pullreq gets updated while you're reviewing it, but before you click +merge), and when using GPG signatures, that even a compromised github +couldn't mess with the sources. + +Setup +--------- +Configuring the github-merge tool for the bitcoin repository is done in the following way: + + git config githubmerge.repository bitcoin/bitcoin + git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) + git config --global user.signingkey mykeyid (if you want to GPG sign) diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py new file mode 100755 index 0000000..33d33b7 --- /dev/null +++ b/contrib/devtools/github-merge.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python2 +# Copyright (c) 2016 Bitcoin Core Developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# This script will locally construct a merge commit for a pull request on a +# github repository, inspect it, sign it and optionally push it. + +# The following temporary branches are created/overwritten and deleted: +# * pull/$PULL/base (the current master we're merging onto) +# * pull/$PULL/head (the current state of the remote pull request) +# * pull/$PULL/merge (github's merge) +# * pull/$PULL/local-merge (our merge) + +# In case of a clean merge that is accepted by the user, the local branch with +# name $BRANCH is overwritten with the merged result, and optionally pushed. +from __future__ import division,print_function,unicode_literals +import os,sys +from sys import stdin,stdout,stderr +import argparse +import subprocess + +# External tools (can be overridden using environment) +GIT = os.getenv('GIT','git') +BASH = os.getenv('BASH','bash') + +def git_config_get(option, default=None): + ''' + Get named configuration option from git repository. + ''' + try: + return subprocess.check_output([GIT,'config','--get',option]).rstrip() + except subprocess.CalledProcessError as e: + return default + +def retrieve_pr_title(repo,pull): + ''' + Retrieve pull request title from github. + Return None if no title can be found, or an error happens. + ''' + import urllib2,json + try: + req = urllib2.Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) + result = urllib2.urlopen(req) + result = json.load(result) + return result['title'] + except Exception as e: + print('Warning: unable to retrieve pull title from github: %s' % e) + return None + +def ask_prompt(text): + print(text,end=" ",file=stderr) + reply = stdin.readline().rstrip() + print("",file=stderr) + return reply + +def parse_arguments(branch): + epilog = ''' + In addition, you can set the following git configuration variables: + githubmerge.repository (mandatory), + user.signingkey (mandatory), + githubmerge.host (default: git@github.com), + githubmerge.branch (default: master), + githubmerge.testcmd (default: none). + ''' + parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', + epilog=epilog) + parser.add_argument('pull', metavar='PULL', type=int, nargs=1, + help='Pull request ID to merge') + parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', + default=branch, help='Branch to merge against (default: '+branch+')') + return parser.parse_args() + +def main(): + # Extract settings from git repo + repo = git_config_get('githubmerge.repository') + host = git_config_get('githubmerge.host','git@github.com') + branch = git_config_get('githubmerge.branch','master') + testcmd = git_config_get('githubmerge.testcmd') + signingkey = git_config_get('user.signingkey') + if repo is None: + print("ERROR: No repository configured. Use this command to set:", file=stderr) + print("git config githubmerge.repository /", file=stderr) + exit(1) + if signingkey is None: + print("ERROR: No GPG signing key set. Set one using:",file=stderr) + print("git config --global user.signingkey ",file=stderr) + exit(1) + + host_repo = host+":"+repo # shortcut for push/pull target + + # Extract settings from command line + args = parse_arguments(branch) + pull = str(args.pull[0]) + branch = args.branch + + # Initialize source branches + head_branch = 'pull/'+pull+'/head' + base_branch = 'pull/'+pull+'/base' + merge_branch = 'pull/'+pull+'/merge' + local_merge_branch = 'pull/'+pull+'/local-merge' + + devnull = open(os.devnull,'w') + try: + subprocess.check_call([GIT,'checkout','-q',branch]) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot check out branch %s." % (branch), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*']) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find pull request #%s on %s." % (pull,host_repo), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr) + exit(3) + try: + subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/heads/'+branch+':refs/heads/'+base_branch]) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot find branch %s on %s." % (branch,host_repo), file=stderr) + exit(3) + subprocess.check_call([GIT,'checkout','-q',base_branch]) + subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull) + subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch]) + + try: + # Create unsigned merge commit. + title = retrieve_pr_title(repo,pull) + if title: + firstline = 'Merge #%s: %s' % (pull,title) + else: + firstline = 'Merge #%s' % (pull,) + message = firstline + '\n\n' + message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]) + try: + subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message,head_branch]) + except subprocess.CalledProcessError as e: + print("ERROR: Cannot be merged cleanly.",file=stderr) + subprocess.check_call([GIT,'merge','--abort']) + exit(4) + logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']) + if logmsg.rstrip() != firstline.rstrip(): + print("ERROR: Creating merge failed (already merged?).",file=stderr) + exit(4) + + # Run test command if configured. + if testcmd: + # Go up to the repository's root. + toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']) + os.chdir(toplevel) + if subprocess.call(testcmd,shell=True): + print("ERROR: Running %s failed." % testcmd,file=stderr) + exit(5) + + # Show the created merge. + diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch]) + subprocess.check_call([GIT,'diff',base_branch+'..'+local_merge_branch]) + if diff: + print("WARNING: merge differs from github!",file=stderr) + reply = ask_prompt("Type 'ignore' to continue.") + if reply.lower() == 'ignore': + print("Difference with github ignored.",file=stderr) + else: + exit(6) + reply = ask_prompt("Press 'd' to accept the diff.") + if reply.lower() == 'd': + print("Diff accepted.",file=stderr) + else: + print("ERROR: Diff rejected.",file=stderr) + exit(6) + else: + # Verify the result manually. + print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) + print("Run 'git diff HEAD~' to show the changes being merged.",file=stderr) + print("Type 'exit' when done.",file=stderr) + if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt + os.putenv('debian_chroot',pull) + subprocess.call([BASH,'-i']) + reply = ask_prompt("Type 'm' to accept the merge.") + if reply.lower() == 'm': + print("Merge accepted.",file=stderr) + else: + print("ERROR: Merge rejected.",file=stderr) + exit(7) + + # Sign the merge commit. + reply = ask_prompt("Type 's' to sign off on the merge.") + if reply == 's': + try: + subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) + except subprocess.CalledProcessError as e: + print("Error signing, exiting.",file=stderr) + exit(1) + else: + print("Not signing off on merge, exiting.",file=stderr) + exit(1) + + # Put the result in branch. + subprocess.check_call([GIT,'checkout','-q',branch]) + subprocess.check_call([GIT,'reset','-q','--hard',local_merge_branch]) + finally: + # Clean up temporary branches. + subprocess.call([GIT,'checkout','-q',branch]) + subprocess.call([GIT,'branch','-q','-D',head_branch],stderr=devnull) + subprocess.call([GIT,'branch','-q','-D',base_branch],stderr=devnull) + subprocess.call([GIT,'branch','-q','-D',merge_branch],stderr=devnull) + subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) + + # Push the result. + reply = ask_prompt("Type 'push' to push the result to %s, branch %s." % (host_repo,branch)) + if reply.lower() == 'push': + subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + +if __name__ == '__main__': + main() + From 0ae5a180695735c324c05a875b68edb04163cc9d Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 22 Jan 2016 16:37:21 +0100 Subject: [PATCH 2/5] devtools: show pull and commit information in github-merge Print the number and title of the pull, as well as the commits to be merged. --- contrib/devtools/github-merge.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 33d33b7..6854ecb 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -24,6 +24,15 @@ import subprocess GIT = os.getenv('GIT','git') BASH = os.getenv('BASH','bash') +# OS specific configuration for terminal attributes +ATTR_RESET = '' +ATTR_PR = '' +COMMIT_FORMAT = '%h %s (%an)%d' +if os.name == 'posix': # if posix, assume we can use basic terminal escapes + ATTR_RESET = '\033[0m' + ATTR_PR = '\033[1;36m' + COMMIT_FORMAT = '%C(bold blue)%h%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset' + def git_config_get(option, default=None): ''' Get named configuration option from git repository. @@ -150,6 +159,9 @@ def main(): print("ERROR: Creating merge failed (already merged?).",file=stderr) exit(4) + print('%s#%s%s %s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title)) + subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) + print() # Run test command if configured. if testcmd: # Go up to the repository's root. From 7d99dee286d6f28a58d3fd5a67b3b67ce92dd643 Mon Sep 17 00:00:00 2001 From: Andrew C Date: Sat, 23 Jan 2016 10:35:27 -0500 Subject: [PATCH 3/5] [devtools] github-merge get toplevel dir without extra whitespace Fixes a bug in github merge when it runs the tests where the toplevel directory has an extra '\n' appended to the path string. Now it doesn't. --- contrib/devtools/github-merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 6854ecb..f7781cc 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -165,7 +165,7 @@ def main(): # Run test command if configured. if testcmd: # Go up to the repository's root. - toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']) + toplevel = subprocess.check_output([GIT,'rev-parse','--show-toplevel']).strip() os.chdir(toplevel) if subprocess.call(testcmd,shell=True): print("ERROR: Running %s failed." % testcmd,file=stderr) From ee6dee4a2a497ee7cd8762413ae2b8bbcc2e5640 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 27 Jan 2016 11:39:58 +0100 Subject: [PATCH 4/5] devtools: Fix utf-8 support in messages for github-merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use 'utf-8' instead of the Python 2 default of 'ascii' to encode/decode commit messages. This can be removed when switching to Python 3, as 'utf-8' is the default there. Necessary for merging #7422 due to the ฿ in ฿tcDrak. --- contrib/devtools/github-merge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index f7781cc..c8dcaae 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -147,14 +147,14 @@ def main(): else: firstline = 'Merge #%s' % (pull,) message = firstline + '\n\n' - message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]) + message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') try: - subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message,head_branch]) + subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch]) except subprocess.CalledProcessError as e: print("ERROR: Cannot be merged cleanly.",file=stderr) subprocess.check_call([GIT,'merge','--abort']) exit(4) - logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']) + logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8') if logmsg.rstrip() != firstline.rstrip(): print("ERROR: Creating merge failed (already merged?).",file=stderr) exit(4) From 784d87896bf4ade9528bfb5656af5a43955a903b Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 14 Mar 2016 22:40:02 +0100 Subject: [PATCH 5/5] Adjust readme for gitian-builder --- contrib/devtools/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index c1a13e6..1af476c 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -12,7 +12,7 @@ For example: ./github-merge.py 3077 (in any git repository) will help you merge pull request #3077 for the -bitcoin/bitcoin repository. +devrandom/gitian-builder repository. What it does: * Fetch master and the pull request. @@ -30,8 +30,8 @@ couldn't mess with the sources. Setup --------- -Configuring the github-merge tool for the bitcoin repository is done in the following way: +Configuring the github-merge tool for this repository is done in the following way: - git config githubmerge.repository bitcoin/bitcoin + git config githubmerge.repository devrandom/gitian-builder git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing) git config --global user.signingkey mykeyid (if you want to GPG sign)