initial
devrandom
14 years ago
commit
4db2a9dfa0
@ -0,0 +1,8 @@
|
||||
*-stamp
|
||||
*.substvars
|
||||
*.log
|
||||
*.pyc
|
||||
build
|
||||
var
|
||||
result
|
||||
inputs
|
||||
@ -0,0 +1,25 @@
|
||||
# Gitian
|
||||
|
||||
Read about the project goals at the "project home page":https://gitian.org/ .
|
||||
|
||||
This package can do a deterministic build of a package inside a VM.
|
||||
|
||||
## Deterministic build inside a VM
|
||||
|
||||
This performs a build inside a VM, with deterministic inputs and outputs. If the build script takes care of all sources of non-determinism (mostly caused by timestamps), the result will always be the same. This allows multiple independent verifiers to sign a binary with the assurance that it really came from the source they reviewed.
|
||||
|
||||
Synopsis:
|
||||
|
||||
* Install prereqs:
|
||||
|
||||
sudo apt-get install python-vm-builder qemu-kvm
|
||||
|
||||
* This will create the base VM for use in further builds (requires sudo):
|
||||
|
||||
bin/make-base-vm
|
||||
|
||||
* This will build using a YAML description file (can be run as non-root):
|
||||
|
||||
bin/gbuild _package_-desc.yml
|
||||
|
||||
The resulting report will appear in result/_package_-res.yml
|
||||
@ -0,0 +1,160 @@
|
||||
#!/usr/bin/ruby
|
||||
|
||||
require 'optparse'
|
||||
require 'yaml'
|
||||
require 'fileutils'
|
||||
require 'pathname'
|
||||
|
||||
@options = {}
|
||||
|
||||
def system!(cmd)
|
||||
system(cmd) or raise "failed to run #{cmd}"
|
||||
end
|
||||
|
||||
def sanitize(str, where)
|
||||
raise "unsanitary string in #{where}" if (str =~ /[^\w.-]/)
|
||||
str
|
||||
end
|
||||
|
||||
def info(str)
|
||||
puts str unless @options[:quiet]
|
||||
end
|
||||
|
||||
OptionParser.new do |opts|
|
||||
opts.banner = "Usage: build [options] <build-description>.yml"
|
||||
|
||||
opts.on("-i", "--skip-image", "reuse current target image") do |v|
|
||||
@options[:skip_image] = v
|
||||
end
|
||||
opts.on("-q", "--quiet", "be quiet") do |v|
|
||||
@options[:skip_image] = v
|
||||
end
|
||||
end.parse!
|
||||
|
||||
base_dir = Pathname.new(__FILE__).expand_path.dirname.parent
|
||||
libexec_dir = base_dir + 'libexec'
|
||||
|
||||
ENV['PATH'] = libexec_dir.to_s + ":" + ENV['PATH']
|
||||
ENV['GITIAN_BASE'] = base_dir.to_s
|
||||
|
||||
build_desc_file = ARGV.shift or raise "must supply YAML build description file"
|
||||
|
||||
in_sums = []
|
||||
|
||||
build_desc = YAML.load_file(build_desc_file)
|
||||
|
||||
package_name = build_desc["name"] or raise "must supply name"
|
||||
package_name = sanitize(package_name, "package name")
|
||||
|
||||
desc_sum = `sha256sum #{build_desc_file}`
|
||||
desc_sum = desc_sum.sub(build_desc_file, "#{package_name}-desc.yml")
|
||||
in_sums << desc_sum
|
||||
|
||||
reference_datetime = build_desc["reference_datetime"] or raise "must supply reference_datetime"
|
||||
|
||||
build_dir = 'build'
|
||||
result_dir = 'result'
|
||||
|
||||
FileUtils.rm_rf(build_dir)
|
||||
FileUtils.mkdir(build_dir)
|
||||
FileUtils.mkdir_p(result_dir)
|
||||
|
||||
info "Stopping target if it is up"
|
||||
system "stop-target"
|
||||
|
||||
sleep 1
|
||||
|
||||
unless @options[:skip_image]
|
||||
info "Making a new image copy"
|
||||
system! "cp base.qcow2 target.qcow2"
|
||||
end
|
||||
|
||||
info "Starting target"
|
||||
system! "start-target &"
|
||||
|
||||
$stdout.write "Checking if target is up"
|
||||
|
||||
(1..10).each do
|
||||
system "on-target true 2> /dev/null" and break
|
||||
sleep 2
|
||||
$stdout.write '.'
|
||||
end
|
||||
|
||||
info ''
|
||||
|
||||
system! "on-target true"
|
||||
|
||||
info "Installing additional packages (log in var/install.log)"
|
||||
system! "on-target -u root apt-get -y install #{build_desc["packages"].join(" ")} > var/install.log 2>&1"
|
||||
|
||||
info "Grabbing package manifest"
|
||||
system! "on-target -u root bash < target-bin/grab-packages.sh > var/base.manifest"
|
||||
|
||||
info "Preparing build environment"
|
||||
system! "on-target bash < target-bin/init-build.sh"
|
||||
|
||||
build_desc["files"].each do |filename|
|
||||
filename = sanitize(filename, "files section")
|
||||
system! "cd inputs ; copy-to-target #{filename} build/"
|
||||
in_sums << `cd inputs ; sha256sum #{filename}`
|
||||
end
|
||||
|
||||
info "Creating build script (var/build-script)"
|
||||
|
||||
File.open("var/build-script", "w") do |script|
|
||||
script.puts "#!/bin/bash"
|
||||
script.puts "set -e"
|
||||
script.puts "export OUTDIR=$HOME/out"
|
||||
script.puts "MAKEOPTS=(-j2)"
|
||||
(ref_date, ref_time) = reference_datetime.split
|
||||
script.puts "REFERENCE_DATETIME='#{reference_datetime}'"
|
||||
script.puts "REFERENCE_DATE='#{ref_date}'"
|
||||
script.puts "REFERENCE_TIME='#{ref_time}'"
|
||||
script.puts
|
||||
build_desc["remotes"].each do |remote|
|
||||
script.puts "git clone -q #{remote["url"]} build/#{remote["dir"]}"
|
||||
script.puts "(cd build/#{remote["dir"]} ; git checkout -q #{remote["commit"]})"
|
||||
end
|
||||
script.puts "cd build"
|
||||
script.puts build_desc["script"]
|
||||
end
|
||||
|
||||
info "Running build script (log in var/build.log)"
|
||||
system! "on-target bash < var/build-script > var/build.log 2>&1"
|
||||
|
||||
info "Grabbing results"
|
||||
system! "copy-from-target out #{build_dir}"
|
||||
|
||||
out_dir = File.join(build_dir, "out")
|
||||
out_sums = {}
|
||||
|
||||
info "Generating report"
|
||||
Dir.new(out_dir).each do |file|
|
||||
next if file.start_with?(".")
|
||||
file = sanitize(file, out_dir)
|
||||
out_sums[file] = `cd #{out_dir} ; sha256sum #{file}`
|
||||
raise "failed to sum #{file}" unless $? == 0
|
||||
puts out_sums[file] unless @options[:quiet]
|
||||
end
|
||||
|
||||
out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
|
||||
|
||||
base_manifest = File.read('var/base.manifest')
|
||||
|
||||
in_manifest = in_sums.join('')
|
||||
|
||||
# Use Omap to keep result deterministic
|
||||
report = YAML::Omap[
|
||||
'out_manifest', out_manifest,
|
||||
'in_manifest', in_manifest,
|
||||
'base_manifest', base_manifest,
|
||||
]
|
||||
|
||||
result_file = "#{package_name}-res.yml"
|
||||
File.open(File.join(result_dir, result_file), "w") do |io|
|
||||
io.write report.to_yaml
|
||||
end
|
||||
|
||||
system!("cd #{result_dir} ; sha256sum #{result_file}") unless @options[:quiet]
|
||||
|
||||
info "Done."
|
||||
@ -0,0 +1,50 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "../lib"))
|
||||
sys.path.append('/usr/lib/gitian')
|
||||
from gitian_util import *
|
||||
|
||||
def check_command(command):
|
||||
if commands.get(command) is None:
|
||||
print>>sys.stderr, "usage: %s CMD\ntry: %s help"%(prog, prog)
|
||||
exit(1)
|
||||
return find_command(command)
|
||||
|
||||
args = sys.argv[:]
|
||||
prog = args.pop(0)
|
||||
|
||||
if len(args) < 1:
|
||||
print>>sys.stderr, "usage: %s CMD\n\ntry:\n %s help\nor:\n %s help CMD"%(prog, prog, prog)
|
||||
exit(1)
|
||||
|
||||
commands = {
|
||||
"release-build": "Build all packages into the 'dist' directory",
|
||||
"package-build": "Build a single package into the 'dist' directory",
|
||||
"package-new": "Insert a new package into the distribution",
|
||||
"release-upload": "Upload a release to a web server",
|
||||
}
|
||||
if args[0] == "help":
|
||||
if len(args) == 1:
|
||||
for command in commands.keys():
|
||||
print command, " - ", commands[command]
|
||||
else:
|
||||
command = args[1]
|
||||
command_path = find_command(command)
|
||||
ret = subprocess.call([command_path, "-h"])
|
||||
elif args[0] == 'shell-complete':
|
||||
if len(args) == 1 or args[1] == "help":
|
||||
for command in commands.keys():
|
||||
print "%s:%s"%(command, commands[command])
|
||||
else:
|
||||
command = args[1]
|
||||
command_path = find_command(command)
|
||||
ret = subprocess.call([command_path, "--shell-complete"])
|
||||
else:
|
||||
command = args.pop(0)
|
||||
command_path = find_command(command)
|
||||
args.insert(0, command_path)
|
||||
os.execv(command_path, args)
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
SUITE=lucid
|
||||
ARCH=amd64
|
||||
|
||||
if [ ! -e var/id_dsa ]; then
|
||||
ssh-keygen -t dsa -f var/id_dsa -N ""
|
||||
fi
|
||||
sudo vmbuilder kvm ubuntu --arch=$ARCH --suite=$SUITE --addpkg=openssh-server,pciutils,build-essential,git-core,mercurial,subversion --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=http://localhost:3142/ubuntu --dest=base --flavour=virtual --overwrite
|
||||
mv base/*.qcow2 base.qcow2
|
||||
@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
|
||||
. gconfig
|
||||
|
||||
TUSER=ubuntu
|
||||
|
||||
usage() {
|
||||
echo "Usage: ${0##*/} [OPTION]... <command>"
|
||||
echo "Run command on build target."
|
||||
echo
|
||||
cat << EOF
|
||||
--help display this help and exit
|
||||
--user=U run as U instead of ubuntu
|
||||
EOF
|
||||
}
|
||||
|
||||
if [ $# != 0 ] ; then
|
||||
while true ; do
|
||||
case "$1" in
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--user|-u)
|
||||
TUSER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--*)
|
||||
echo "unrecognized option $1"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if [ $# = 0 ] ; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT -r $TUSER@localhost:$1 $2
|
||||
@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
|
||||
. gconfig
|
||||
|
||||
TUSER=ubuntu
|
||||
|
||||
usage() {
|
||||
echo "Usage: ${0##*/} [OPTION]... <command>"
|
||||
echo "Run command on build target."
|
||||
echo
|
||||
cat << EOF
|
||||
--help display this help and exit
|
||||
--user=U run as U instead of ubuntu
|
||||
EOF
|
||||
}
|
||||
|
||||
if [ $# != 0 ] ; then
|
||||
while true ; do
|
||||
case "$1" in
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--user|-u)
|
||||
TUSER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--*)
|
||||
echo "unrecognized option $1"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if [ $# = 0 ] ; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
scp -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -P $VM_SSH_PORT $1 $TUSER@localhost:$2
|
||||
@ -0,0 +1 @@
|
||||
VM_SSH_PORT=2223
|
||||
@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
|
||||
. gconfig
|
||||
|
||||
TUSER=ubuntu
|
||||
|
||||
usage() {
|
||||
echo "Usage: ${0##*/} [OPTION]... <command>"
|
||||
echo "Run command on build target."
|
||||
echo
|
||||
cat << EOF
|
||||
--help display this help and exit
|
||||
--user=U run as U instead of ubuntu
|
||||
EOF
|
||||
}
|
||||
|
||||
if [ $# != 0 ] ; then
|
||||
while true ; do
|
||||
case "$1" in
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--user|-u)
|
||||
TUSER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--*)
|
||||
echo "unrecognized option $1"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
#if [ $# = 0 ] ; then
|
||||
# usage
|
||||
# exit 1
|
||||
#fi
|
||||
|
||||
ssh -oConnectTimeout=2 -i ${GITIAN_BASE:-.}/var/id_dsa -p $VM_SSH_PORT $TUSER@localhost $*
|
||||
@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
|
||||
. gconfig
|
||||
|
||||
kvm -m 2000 -smp 2 -drive file=target.qcow2 -net nic,model=virtio -net user,hostfwd=tcp::$VM_SSH_PORT-:22 -vnc :16 > var/target.log 2>&1 &
|
||||
|
||||
echo $! > var/target.pid
|
||||
wait
|
||||
rm var/target.pid
|
||||
@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
|
||||
if [ ! -e var/target.pid ]; then exit; fi
|
||||
|
||||
on-target -u root halt
|
||||
sleep 5
|
||||
|
||||
if [ ! -e var/target.pid ]; then exit; fi
|
||||
sleep 5
|
||||
|
||||
if [ ! -e var/target.pid ]; then exit; fi
|
||||
|
||||
echo Killing target since it did not shutdown within 10 seconds
|
||||
kill `cat var/target.pid`
|
||||
@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Get an installed package manifest
|
||||
|
||||
set -e
|
||||
|
||||
cd /var/cache/apt/archives
|
||||
|
||||
#apt-get clean
|
||||
|
||||
dpkg-query -W -f '${Package}\n' | xargs -n 50 apt-get install --reinstall -y -d > /dev/null
|
||||
sha256sum *.deb | sort --key 2
|
||||
@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
rm -rf inst out build
|
||||
mkdir build
|
||||
mkdir out
|
||||
mkdir inst
|
||||
Loading…
Reference in New Issue