build: fix doc generation

also moved MD5 to koreader-base/ffi
pull/1843/head
Qingping Hou 8 years ago
parent 042d94c3ef
commit 69f9ebd528

@ -1,16 +1,41 @@
#!/usr/bin/env bash
CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${CI_DIR}/common.sh"
set +e
make coverage
cd koreader-*/koreader && luajit $(which luacov-coveralls) -v
# get deploy key for doc repo
openssl aes-256-cbc -K $encrypted_dc71a4fb8382_key -iv $encrypted_dc71a4fb8382_iv \
-in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d
ssh-add ~/.ssh/koreader_doc
git clone git@github.com:koreader/doc.git
# push doc update
make doc
cp -r doc/html/* doc/
pushd doc
git add .
git push origin gh-pages
pushd koreader-*/koreader
luajit $(which luacov-coveralls) -v
popd
if [ $TRAVIS_BRANCH == 'master' ]; then
travis_retry luarocks --local install ldoc
# get deploy key for doc repo
openssl aes-256-cbc -k $doc_build_secret -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d
chmod 600 ~/.ssh/koreader_doc # make agent happy
eval "$(ssh-agent)" > /dev/null
ssh-add ~/.ssh/koreader_doc > /dev/null
echo -e "\n${ANSI_GREEN}Check out koreader/doc for update."
git clone git@github.com:koreader/doc.git koreader_doc
# push doc update
pushd doc
luajit $(which ldoc) . 2> /dev/null
if [ ! -d html ]; then
echo "Failed to generate documents..."
exit 1
fi
popd
cp -r doc/html/* koreader_doc/
pushd koreader_doc
echo -e "\n${ANSI_GREEN}Pusing document update..."
git -c user.name="KOReader build bot" -c user.email="non-reply@koreader.rocks" \
commit -a --amend -m 'Automated documentation build from travis-ci.'
git push -f --quiet origin gh-pages > /dev/null
echo -e "\n${ANSI_GREEN}Document update pushed."
fi

@ -1,3 +1,6 @@
set -e
set -o pipefail
ANSI_RED="\033[31;1m"
ANSI_GREEN="\033[32;1m"
ANSI_RESET="\033[0m"
@ -6,6 +9,8 @@ ANSI_CLEAR="\033[0K"
travis_retry() {
local result=0
local count=1
set +e
while [ $count -le 3 ]; do
[ $result -ne 0 ] && {
echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2
@ -21,6 +26,7 @@ travis_retry() {
echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2
}
set -e
return $result
}

@ -15,6 +15,7 @@ mkdir $HOME/.luarocks
cp ${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua $HOME/.luarocks/config.lua
echo "wrap_bin_scripts = false" >> $HOME/.luarocks/config.lua
travis_retry luarocks --local install luafilesystem
# for verbose_print module
travis_retry luarocks --local install ansicolors
travis_retry luarocks --local install busted 2.0.rc11-0
#- travis_retry luarocks --local install busted 1.11.1-1
@ -25,4 +26,3 @@ travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linu
travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
travis_retry luarocks --local install luacheck
travis_retry luarocks --local install lanes # for parallel luacheck
travis_retry luarocks --local install ldoc

Binary file not shown.

@ -6,5 +6,6 @@ source "${CI_DIR}/common.sh"
travis_retry make fetchthirdparty
make all
travis_retry make testfront
set +o pipefail
luajit $(which luacheck) --no-color -q frontend | tee ./luacheck.out
test $(grep Total ./luacheck.out | awk '{print $2}') -le 63

2
.gitignore vendored

@ -33,7 +33,7 @@ i18n
koreader-android-arm-linux-androideabi
koreader-kindle-legacy-arm-kindle-linux-gnueabi
koreader-kindle-arm-linux-gnueabi
koreader-kobo-arm-linux-gnueabihf
koreader-kobo-arm-linux-gnueabihf*
koreader-emulator-i686-w64-mingw32
koreader-emulator-x86_64-linux-gnu
koreader-emulator-x86_64-pc-linux-gnu

@ -1,6 +1,8 @@
language: c
sudo: false
# sudo: false
sudo: true
dist: trusty
compiler:
- gcc

@ -1 +1 @@
Subproject commit 74e57274ab8d21b99e614bcf4c050e32f407feee
Subproject commit 4c41adf0d224b9170fd506c2fcdf639c7b6bc062

@ -1,246 +0,0 @@
--[[--
MD5 hash library.
]]
local ffi = require "ffi"
local bit = require "bit"
local bxor = bit.bxor
local bnot = bit.bnot
local band = bit.band
local bor = bit.bor
local rshift = bit.rshift
local lshift = bit.lshift
local copy = ffi.copy
local fill = ffi.fill
ffi.cdef[[
typedef struct MD5Context {
uint32_t buf[4];
uint32_t bits[2];
unsigned char input[64];
} MD5_CTX;
]]
local function byteReverse(buf, len)
-- TODO: implement for big-endian architectures?
end
local function F1(x, y, z) return bxor(z, band(x, bxor(y, z))) end
local function F2(x, y, z) return F1(z, x, y) end
local function F3(x, y, z) return bxor(x, y, z) end
local function F4(x, y, z) return bxor(y, bor(x, bnot(z))) end
local function MD5STEP(f, w, x, y, z, data, s)
w = w + f(x, y, z) + data
w = bor(lshift(w,s), rshift(w,(32-s)))
w = w + x
return w
end
-- Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
-- initialization constants.
local function MD5Init(ctx)
ctx.buf[0] = 0x67452301
ctx.buf[1] = 0xefcdab89
ctx.buf[2] = 0x98badcfe
ctx.buf[3] = 0x10325476
ctx.bits[0] = 0
ctx.bits[1] = 0
end
local function MD5Transform(buf, input)
local a = buf[0]
local b = buf[1]
local c = buf[2]
local d = buf[3]
a = MD5STEP(F1, a, b, c, d, input[0] + 0xd76aa478, 7)
d = MD5STEP(F1, d, a, b, c, input[1] + 0xe8c7b756, 12)
c = MD5STEP(F1, c, d, a, b, input[2] + 0x242070db, 17)
b = MD5STEP(F1, b, c, d, a, input[3] + 0xc1bdceee, 22)
a = MD5STEP(F1, a, b, c, d, input[4] + 0xf57c0faf, 7)
d = MD5STEP(F1, d, a, b, c, input[5] + 0x4787c62a, 12)
c = MD5STEP(F1, c, d, a, b, input[6] + 0xa8304613, 17)
b = MD5STEP(F1, b, c, d, a, input[7] + 0xfd469501, 22)
a = MD5STEP(F1, a, b, c, d, input[8] + 0x698098d8, 7)
d = MD5STEP(F1, d, a, b, c, input[9] + 0x8b44f7af, 12)
c = MD5STEP(F1, c, d, a, b, input[10] + 0xffff5bb1, 17)
b = MD5STEP(F1, b, c, d, a, input[11] + 0x895cd7be, 22)
a = MD5STEP(F1, a, b, c, d, input[12] + 0x6b901122, 7)
d = MD5STEP(F1, d, a, b, c, input[13] + 0xfd987193, 12)
c = MD5STEP(F1, c, d, a, b, input[14] + 0xa679438e, 17)
b = MD5STEP(F1, b, c, d, a, input[15] + 0x49b40821, 22)
a = MD5STEP(F2, a, b, c, d, input[1] + 0xf61e2562, 5)
d = MD5STEP(F2, d, a, b, c, input[6] + 0xc040b340, 9)
c = MD5STEP(F2, c, d, a, b, input[11] + 0x265e5a51, 14)
b = MD5STEP(F2, b, c, d, a, input[0] + 0xe9b6c7aa, 20)
a = MD5STEP(F2, a, b, c, d, input[5] + 0xd62f105d, 5)
d = MD5STEP(F2, d, a, b, c, input[10] + 0x02441453, 9)
c = MD5STEP(F2, c, d, a, b, input[15] + 0xd8a1e681, 14)
b = MD5STEP(F2, b, c, d, a, input[4] + 0xe7d3fbc8, 20)
a = MD5STEP(F2, a, b, c, d, input[9] + 0x21e1cde6, 5)
d = MD5STEP(F2, d, a, b, c, input[14] + 0xc33707d6, 9)
c = MD5STEP(F2, c, d, a, b, input[3] + 0xf4d50d87, 14)
b = MD5STEP(F2, b, c, d, a, input[8] + 0x455a14ed, 20)
a = MD5STEP(F2, a, b, c, d, input[13] + 0xa9e3e905, 5)
d = MD5STEP(F2, d, a, b, c, input[2] + 0xfcefa3f8, 9)
c = MD5STEP(F2, c, d, a, b, input[7] + 0x676f02d9, 14)
b = MD5STEP(F2, b, c, d, a, input[12] + 0x8d2a4c8a, 20)
a = MD5STEP(F3, a, b, c, d, input[5] + 0xfffa3942, 4)
d = MD5STEP(F3, d, a, b, c, input[8] + 0x8771f681, 11)
c = MD5STEP(F3, c, d, a, b, input[11] + 0x6d9d6122, 16)
b = MD5STEP(F3, b, c, d, a, input[14] + 0xfde5380c, 23)
a = MD5STEP(F3, a, b, c, d, input[1] + 0xa4beea44, 4)
d = MD5STEP(F3, d, a, b, c, input[4] + 0x4bdecfa9, 11)
c = MD5STEP(F3, c, d, a, b, input[7] + 0xf6bb4b60, 16)
b = MD5STEP(F3, b, c, d, a, input[10] + 0xbebfbc70, 23)
a = MD5STEP(F3, a, b, c, d, input[13] + 0x289b7ec6, 4)
d = MD5STEP(F3, d, a, b, c, input[0] + 0xeaa127fa, 11)
c = MD5STEP(F3, c, d, a, b, input[3] + 0xd4ef3085, 16)
b = MD5STEP(F3, b, c, d, a, input[6] + 0x04881d05, 23)
a = MD5STEP(F3, a, b, c, d, input[9] + 0xd9d4d039, 4)
d = MD5STEP(F3, d, a, b, c, input[12] + 0xe6db99e5, 11)
c = MD5STEP(F3, c, d, a, b, input[15] + 0x1fa27cf8, 16)
b = MD5STEP(F3, b, c, d, a, input[2] + 0xc4ac5665, 23)
a = MD5STEP(F4, a, b, c, d, input[0] + 0xf4292244, 6)
d = MD5STEP(F4, d, a, b, c, input[7] + 0x432aff97, 10)
c = MD5STEP(F4, c, d, a, b, input[14] + 0xab9423a7, 15)
b = MD5STEP(F4, b, c, d, a, input[5] + 0xfc93a039, 21)
a = MD5STEP(F4, a, b, c, d, input[12] + 0x655b59c3, 6)
d = MD5STEP(F4, d, a, b, c, input[3] + 0x8f0ccc92, 10)
c = MD5STEP(F4, c, d, a, b, input[10] + 0xffeff47d, 15)
b = MD5STEP(F4, b, c, d, a, input[1] + 0x85845dd1, 21)
a = MD5STEP(F4, a, b, c, d, input[8] + 0x6fa87e4f, 6)
d = MD5STEP(F4, d, a, b, c, input[15] + 0xfe2ce6e0, 10)
c = MD5STEP(F4, c, d, a, b, input[6] + 0xa3014314, 15)
b = MD5STEP(F4, b, c, d, a, input[13] + 0x4e0811a1, 21)
a = MD5STEP(F4, a, b, c, d, input[4] + 0xf7537e82, 6)
d = MD5STEP(F4, d, a, b, c, input[11] + 0xbd3af235, 10)
c = MD5STEP(F4, c, d, a, b, input[2] + 0x2ad7d2bb, 15)
b = MD5STEP(F4, b, c, d, a, input[9] + 0xeb86d391, 21)
buf[0] = band(buf[0] + a, 0xFFFFFFFF)
buf[1] = band(buf[1] + b, 0xFFFFFFFF)
buf[2] = band(buf[2] + c, 0xFFFFFFFF)
buf[3] = band(buf[3] + d, 0xFFFFFFFF)
end
local function MD5Update(ctx, buf, len)
local t
t = ctx.bits[0]
ctx.bits[0] = t + lshift( len, 3)
if (ctx.bits[0] < t) then
ctx.bits[1] = ctx.bits[1] + 1
end
ctx.bits[1] = ctx.bits[1] + rshift(len, 29)
t = band(rshift(t, 3), 0x3f)
if (t > 0) then
local p = ffi.cast("unsigned char *", ctx.input + t)
t = 64 - t
if (len < t) then
copy(p, buf, len)
return
end
copy(p, buf, t)
byteReverse(ctx.input, 16)
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
buf = buf + t
len = len - t
end
while (len >= 64) do
copy(ctx.input, buf, 64)
byteReverse(ctx.input, 16)
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
buf = buf + 64
len = len - 64
end
copy(ctx.input, buf, len)
end
local function MD5Final(digest, ctx)
local count
local p
count = band(rshift(ctx.bits[0], 3), 0x3F)
p = ctx.input + count
p[0] = 0x80
p = p + 1
count = 64 - 1 - count
if (count < 8) then
fill(p, count, 0)
byteReverse(ctx.input, 16)
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
fill(ctx.input, 56, 0)
else
fill(p, count - 8, 0)
end
byteReverse(ctx.input, 14)
ffi.cast("uint32_t *", ctx.input)[14] = ctx.bits[0]
ffi.cast("uint32_t *", ctx.input)[15] = ctx.bits[1]
MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input))
byteReverse(ffi.cast("unsigned char *",ctx.buf), 4)
copy(digest, ctx.buf, 16)
fill(ffi.cast("char *", ctx), ffi.sizeof(ctx), 0)
end
local hex = ffi.new("const char[16]", "0123456789abcdef")
local function bin2str(output, input, len)
if len > 0 then
output[0] = hex[rshift(input[0], 4)]
output[1] = hex[band(input[0], 0xF)]
return bin2str(output+2, input+1, len-1)
end
end
local md5 = {}
--- Create a new md5 hashing instance.
---- @return md5 instance
function md5:new()
self.ctx = ffi.new("MD5_CTX")
MD5Init(self.ctx)
end
--- Feed content to md5 hashing instance.
---- @param luastr Lua string
function md5:update(luastr)
MD5Update(self.ctx, ffi.cast("const char*", luastr), #luastr)
end
--- Calcualte md5 sum.
---- @param luastr Lua string
---- @return md5 sum in Lua string
function md5:sum(luastr)
local buf = ffi.new("char[33]")
local hash = ffi.new("uint8_t[16]")
if luastr then
md5:new()
md5:update(luastr)
end
MD5Final(hash, self.ctx)
bin2str(buf, hash, ffi.sizeof(hash))
return ffi.string(buf)
end
return md5

@ -1,7 +1,7 @@
--[[
A global LRU cache
]]--
local md5 = require("MD5")
local md5 = require("ffi/MD5")
local lfs = require("libs/libkoreader-lfs")
local DataStorage = require("datastorage")
local DEBUG = require("dbg")
@ -123,7 +123,7 @@ function Cache:check(key, ItemClass)
end
return self.cache[key]
elseif ItemClass then
local cached = self.cached[md5:sum(key)]
local cached = self.cached[md5.sum(key)]
if cached then
local item = ItemClass:new{}
local ok, msg = pcall(item.load, item, cached)
@ -153,14 +153,14 @@ function Cache:serialize()
cached_size = cached_size + (lfs.attributes(file, "size") or 0)
end
table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end)
-- serialize the most recently used cache
-- only serialize the most recently used cache
local cache_size = 0
for _, key in ipairs(self.cache_order) do
local cache_item = self.cache[key]
-- only dump cache item that requests serialization explicitly
if cache_item.persistent and cache_item.dump then
DEBUG("dump cache item", key)
cache_size = cache_item:dump(cache_path..md5:sum(key)) or 0
cache_size = cache_item:dump(cache_path..md5.sum(key)) or 0
if cache_size > 0 then break end
end
end

@ -109,23 +109,23 @@ end
-- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data
-- by highlighting in koreader may change the digest value.
function Document:fastDigest()
local md5 = require("MD5")
local md5 = require("ffi/MD5")
local lshift = bit.lshift
local file = io.open(self.file, 'rb')
if file then
local step, size = 1024, 1024
md5:new()
local m = md5.new()
for i = -1, 10 do
file:seek("set", lshift(step, 2*i))
local sample = file:read(size)
if sample then
md5:update(sample)
m:update(sample)
else
break
end
end
file:close()
return md5:sum()
return m:sum()
end
end

10
kodev

@ -42,7 +42,7 @@ OPTIONS:
TARGET:
${SUPPORTED_TARGETS}"
while [[ $1 == '--'* ]]; do
while [[ $1 == '-'* ]]; do
PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | awk -F= '{print $2}'`
case $PARAM in
@ -59,7 +59,7 @@ ${SUPPORTED_TARGETS}"
exit 1
;;
esac
shift
shift 1
done
case $1 in
@ -215,7 +215,7 @@ OPTIONS:
--no-build run reader without rebuilding
--disable-touch use this if you want to simulate keyboard only devices
"
while [[ $1 == '--'* ]]; do
while [[ $1 == '-'* ]]; do
PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | awk -F= '{print $2}'`
case $PARAM in
@ -274,7 +274,7 @@ OPTIONS:
--tags=TAGS only run tests with given tags
"
while [[ $1 == '--'* ]]; do
while [[ $1 == '-'* ]]; do
PARAM=`echo $1 | awk -F= '{print $1}'`
VALUE=`echo $1 | awk -F= '{print $2}'`
case $PARAM in
@ -308,7 +308,7 @@ OPTIONS:
if [ ! -z $2 ]; then
test_path="${test_path}/$2"
fi
busted ${opts} -o verbose_print --exclude-tags=notest ${test_path}
busted --lua=./luajit ${opts} -o ./spec/$1/unit/verbose_print --exclude-tags=notest ${test_path}
popd
}

@ -1,7 +1,7 @@
require("commonrequire")
local UIManager = require("ui/uimanager")
local DEBUG = require("dbg")
local md5 = require("MD5")
local md5 = require("ffi/MD5")
--DEBUG:turnOn()
local service = [[
@ -70,7 +70,7 @@ describe("KOSync modules #notest #nocov", function()
req.headers['x-auth-key'] = args.userkey
end
-- password should be hashed before submitting to server
local username, password = "koreader", md5:sum("koreader")
local username, password = "koreader", md5.sum("koreader")
-- fake progress data
local doc, percentage, progress, device =
"41cce710f34e5ec21315e19c99821415", -- fast digest of the document

@ -1,19 +0,0 @@
require("commonrequire")
local md5 = require("MD5")
describe("MD5 module", function()
it("should calculate correct MD5 hashes", function()
assert.is_equal(md5:sum(""), "d41d8cd98f00b204e9800998ecf8427e")
assert.is_equal(md5:sum("\0"), "93b885adfe0da089cdf634904fd59f71")
assert.is_equal(md5:sum("0123456789abcdefX"), "1b05aba914a8b12315c7ee52b42f3d35")
end)
it("should calculate MD5 sum by updating", function()
md5:new()
md5:update("0123456789")
md5:update("abcdefghij")
local md5sum = md5:sum()
assert.is_equal(md5sum, md5:sum("0123456789abcdefghij"))
end)
end)
Loading…
Cancel
Save