diff --git a/.ci/after_success.sh b/.ci/after_success.sh index 64709c2a5..58f02660a 100755 --- a/.ci/after_success.sh +++ b/.ci/after_success.sh @@ -1,16 +1,41 @@ #!/usr/bin/env bash +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${CI_DIR}/common.sh" + +set +e + make coverage -cd koreader-*/koreader && luajit $(which luacov-coveralls) -v - -# get deploy key for doc repo -openssl aes-256-cbc -K $encrypted_dc71a4fb8382_key -iv $encrypted_dc71a4fb8382_iv \ - -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d -ssh-add ~/.ssh/koreader_doc -git clone git@github.com:koreader/doc.git -# push doc update -make doc -cp -r doc/html/* doc/ -pushd doc -git add . -git push origin gh-pages +pushd koreader-*/koreader + luajit $(which luacov-coveralls) -v +popd + +if [ $TRAVIS_BRANCH == 'master' ]; then + + travis_retry luarocks --local install ldoc + # get deploy key for doc repo + openssl aes-256-cbc -k $doc_build_secret -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d + chmod 600 ~/.ssh/koreader_doc # make agent happy + eval "$(ssh-agent)" > /dev/null + ssh-add ~/.ssh/koreader_doc > /dev/null + echo -e "\n${ANSI_GREEN}Check out koreader/doc for update." + git clone git@github.com:koreader/doc.git koreader_doc + + # push doc update + pushd doc + luajit $(which ldoc) . 2> /dev/null + if [ ! -d html ]; then + echo "Failed to generate documents..." + exit 1 + fi + popd + cp -r doc/html/* koreader_doc/ + pushd koreader_doc + + echo -e "\n${ANSI_GREEN}Pusing document update..." + git -c user.name="KOReader build bot" -c user.email="non-reply@koreader.rocks" \ + commit -a --amend -m 'Automated documentation build from travis-ci.' + git push -f --quiet origin gh-pages > /dev/null + echo -e "\n${ANSI_GREEN}Document update pushed." + +fi diff --git a/.ci/common.sh b/.ci/common.sh index 7fbabd4e3..0ee4bd104 100644 --- a/.ci/common.sh +++ b/.ci/common.sh @@ -1,3 +1,6 @@ +set -e +set -o pipefail + ANSI_RED="\033[31;1m" ANSI_GREEN="\033[32;1m" ANSI_RESET="\033[0m" @@ -6,6 +9,8 @@ ANSI_CLEAR="\033[0K" travis_retry() { local result=0 local count=1 + set +e + while [ $count -le 3 ]; do [ $result -ne 0 ] && { echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 @@ -21,6 +26,7 @@ travis_retry() { echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2 } + set -e return $result } diff --git a/.ci/install.sh b/.ci/install.sh index 95e7ca261..f43812d03 100755 --- a/.ci/install.sh +++ b/.ci/install.sh @@ -15,6 +15,7 @@ mkdir $HOME/.luarocks cp ${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua $HOME/.luarocks/config.lua echo "wrap_bin_scripts = false" >> $HOME/.luarocks/config.lua travis_retry luarocks --local install luafilesystem +# for verbose_print module travis_retry luarocks --local install ansicolors travis_retry luarocks --local install busted 2.0.rc11-0 #- travis_retry luarocks --local install busted 1.11.1-1 @@ -25,4 +26,3 @@ travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linu travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev travis_retry luarocks --local install luacheck travis_retry luarocks --local install lanes # for parallel luacheck -travis_retry luarocks --local install ldoc diff --git a/.ci/koreader_doc.enc b/.ci/koreader_doc.enc index c841e5e0c..c961a46a9 100644 Binary files a/.ci/koreader_doc.enc and b/.ci/koreader_doc.enc differ diff --git a/.ci/script.sh b/.ci/script.sh index 16042e7f2..64be8cb4a 100755 --- a/.ci/script.sh +++ b/.ci/script.sh @@ -6,5 +6,6 @@ source "${CI_DIR}/common.sh" travis_retry make fetchthirdparty make all travis_retry make testfront +set +o pipefail luajit $(which luacheck) --no-color -q frontend | tee ./luacheck.out test $(grep Total ./luacheck.out | awk '{print $2}') -le 63 diff --git a/.gitignore b/.gitignore index a7b14504f..7e9ad4999 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,7 @@ i18n koreader-android-arm-linux-androideabi koreader-kindle-legacy-arm-kindle-linux-gnueabi koreader-kindle-arm-linux-gnueabi -koreader-kobo-arm-linux-gnueabihf +koreader-kobo-arm-linux-gnueabihf* koreader-emulator-i686-w64-mingw32 koreader-emulator-x86_64-linux-gnu koreader-emulator-x86_64-pc-linux-gnu diff --git a/.travis.yml b/.travis.yml index 2dc6ca0a0..63ef40283 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: c -sudo: false +# sudo: false +sudo: true +dist: trusty compiler: - gcc diff --git a/base b/base index 74e57274a..4c41adf0d 160000 --- a/base +++ b/base @@ -1 +1 @@ -Subproject commit 74e57274ab8d21b99e614bcf4c050e32f407feee +Subproject commit 4c41adf0d224b9170fd506c2fcdf639c7b6bc062 diff --git a/frontend/MD5.lua b/frontend/MD5.lua deleted file mode 100644 index 9754f5f9d..000000000 --- a/frontend/MD5.lua +++ /dev/null @@ -1,246 +0,0 @@ ---[[-- -MD5 hash library. - ]] - -local ffi = require "ffi" -local bit = require "bit" -local bxor = bit.bxor -local bnot = bit.bnot -local band = bit.band -local bor = bit.bor -local rshift = bit.rshift -local lshift = bit.lshift -local copy = ffi.copy -local fill = ffi.fill - -ffi.cdef[[ -typedef struct MD5Context { - uint32_t buf[4]; - uint32_t bits[2]; - unsigned char input[64]; -} MD5_CTX; -]] - -local function byteReverse(buf, len) - -- TODO: implement for big-endian architectures? -end - -local function F1(x, y, z) return bxor(z, band(x, bxor(y, z))) end -local function F2(x, y, z) return F1(z, x, y) end -local function F3(x, y, z) return bxor(x, y, z) end -local function F4(x, y, z) return bxor(y, bor(x, bnot(z))) end - -local function MD5STEP(f, w, x, y, z, data, s) - w = w + f(x, y, z) + data - w = bor(lshift(w,s), rshift(w,(32-s))) - w = w + x - - return w -end - --- Start MD5 accumulation. Set bit count to 0 and buffer to mysterious --- initialization constants. -local function MD5Init(ctx) - ctx.buf[0] = 0x67452301 - ctx.buf[1] = 0xefcdab89 - ctx.buf[2] = 0x98badcfe - ctx.buf[3] = 0x10325476 - - ctx.bits[0] = 0 - ctx.bits[1] = 0 -end - -local function MD5Transform(buf, input) - local a = buf[0] - local b = buf[1] - local c = buf[2] - local d = buf[3] - - a = MD5STEP(F1, a, b, c, d, input[0] + 0xd76aa478, 7) - d = MD5STEP(F1, d, a, b, c, input[1] + 0xe8c7b756, 12) - c = MD5STEP(F1, c, d, a, b, input[2] + 0x242070db, 17) - b = MD5STEP(F1, b, c, d, a, input[3] + 0xc1bdceee, 22) - a = MD5STEP(F1, a, b, c, d, input[4] + 0xf57c0faf, 7) - d = MD5STEP(F1, d, a, b, c, input[5] + 0x4787c62a, 12) - c = MD5STEP(F1, c, d, a, b, input[6] + 0xa8304613, 17) - b = MD5STEP(F1, b, c, d, a, input[7] + 0xfd469501, 22) - a = MD5STEP(F1, a, b, c, d, input[8] + 0x698098d8, 7) - d = MD5STEP(F1, d, a, b, c, input[9] + 0x8b44f7af, 12) - c = MD5STEP(F1, c, d, a, b, input[10] + 0xffff5bb1, 17) - b = MD5STEP(F1, b, c, d, a, input[11] + 0x895cd7be, 22) - a = MD5STEP(F1, a, b, c, d, input[12] + 0x6b901122, 7) - d = MD5STEP(F1, d, a, b, c, input[13] + 0xfd987193, 12) - c = MD5STEP(F1, c, d, a, b, input[14] + 0xa679438e, 17) - b = MD5STEP(F1, b, c, d, a, input[15] + 0x49b40821, 22) - - a = MD5STEP(F2, a, b, c, d, input[1] + 0xf61e2562, 5) - d = MD5STEP(F2, d, a, b, c, input[6] + 0xc040b340, 9) - c = MD5STEP(F2, c, d, a, b, input[11] + 0x265e5a51, 14) - b = MD5STEP(F2, b, c, d, a, input[0] + 0xe9b6c7aa, 20) - a = MD5STEP(F2, a, b, c, d, input[5] + 0xd62f105d, 5) - d = MD5STEP(F2, d, a, b, c, input[10] + 0x02441453, 9) - c = MD5STEP(F2, c, d, a, b, input[15] + 0xd8a1e681, 14) - b = MD5STEP(F2, b, c, d, a, input[4] + 0xe7d3fbc8, 20) - a = MD5STEP(F2, a, b, c, d, input[9] + 0x21e1cde6, 5) - d = MD5STEP(F2, d, a, b, c, input[14] + 0xc33707d6, 9) - c = MD5STEP(F2, c, d, a, b, input[3] + 0xf4d50d87, 14) - b = MD5STEP(F2, b, c, d, a, input[8] + 0x455a14ed, 20) - a = MD5STEP(F2, a, b, c, d, input[13] + 0xa9e3e905, 5) - d = MD5STEP(F2, d, a, b, c, input[2] + 0xfcefa3f8, 9) - c = MD5STEP(F2, c, d, a, b, input[7] + 0x676f02d9, 14) - b = MD5STEP(F2, b, c, d, a, input[12] + 0x8d2a4c8a, 20) - - a = MD5STEP(F3, a, b, c, d, input[5] + 0xfffa3942, 4) - d = MD5STEP(F3, d, a, b, c, input[8] + 0x8771f681, 11) - c = MD5STEP(F3, c, d, a, b, input[11] + 0x6d9d6122, 16) - b = MD5STEP(F3, b, c, d, a, input[14] + 0xfde5380c, 23) - a = MD5STEP(F3, a, b, c, d, input[1] + 0xa4beea44, 4) - d = MD5STEP(F3, d, a, b, c, input[4] + 0x4bdecfa9, 11) - c = MD5STEP(F3, c, d, a, b, input[7] + 0xf6bb4b60, 16) - b = MD5STEP(F3, b, c, d, a, input[10] + 0xbebfbc70, 23) - a = MD5STEP(F3, a, b, c, d, input[13] + 0x289b7ec6, 4) - d = MD5STEP(F3, d, a, b, c, input[0] + 0xeaa127fa, 11) - c = MD5STEP(F3, c, d, a, b, input[3] + 0xd4ef3085, 16) - b = MD5STEP(F3, b, c, d, a, input[6] + 0x04881d05, 23) - a = MD5STEP(F3, a, b, c, d, input[9] + 0xd9d4d039, 4) - d = MD5STEP(F3, d, a, b, c, input[12] + 0xe6db99e5, 11) - c = MD5STEP(F3, c, d, a, b, input[15] + 0x1fa27cf8, 16) - b = MD5STEP(F3, b, c, d, a, input[2] + 0xc4ac5665, 23) - - a = MD5STEP(F4, a, b, c, d, input[0] + 0xf4292244, 6) - d = MD5STEP(F4, d, a, b, c, input[7] + 0x432aff97, 10) - c = MD5STEP(F4, c, d, a, b, input[14] + 0xab9423a7, 15) - b = MD5STEP(F4, b, c, d, a, input[5] + 0xfc93a039, 21) - a = MD5STEP(F4, a, b, c, d, input[12] + 0x655b59c3, 6) - d = MD5STEP(F4, d, a, b, c, input[3] + 0x8f0ccc92, 10) - c = MD5STEP(F4, c, d, a, b, input[10] + 0xffeff47d, 15) - b = MD5STEP(F4, b, c, d, a, input[1] + 0x85845dd1, 21) - a = MD5STEP(F4, a, b, c, d, input[8] + 0x6fa87e4f, 6) - d = MD5STEP(F4, d, a, b, c, input[15] + 0xfe2ce6e0, 10) - c = MD5STEP(F4, c, d, a, b, input[6] + 0xa3014314, 15) - b = MD5STEP(F4, b, c, d, a, input[13] + 0x4e0811a1, 21) - a = MD5STEP(F4, a, b, c, d, input[4] + 0xf7537e82, 6) - d = MD5STEP(F4, d, a, b, c, input[11] + 0xbd3af235, 10) - c = MD5STEP(F4, c, d, a, b, input[2] + 0x2ad7d2bb, 15) - b = MD5STEP(F4, b, c, d, a, input[9] + 0xeb86d391, 21) - - buf[0] = band(buf[0] + a, 0xFFFFFFFF) - buf[1] = band(buf[1] + b, 0xFFFFFFFF) - buf[2] = band(buf[2] + c, 0xFFFFFFFF) - buf[3] = band(buf[3] + d, 0xFFFFFFFF) -end - -local function MD5Update(ctx, buf, len) - local t - - t = ctx.bits[0] - ctx.bits[0] = t + lshift( len, 3) - if (ctx.bits[0] < t) then - ctx.bits[1] = ctx.bits[1] + 1 - end - - ctx.bits[1] = ctx.bits[1] + rshift(len, 29) - - t = band(rshift(t, 3), 0x3f) - - if (t > 0) then - local p = ffi.cast("unsigned char *", ctx.input + t) - - t = 64 - t - if (len < t) then - copy(p, buf, len) - return - end - - copy(p, buf, t) - byteReverse(ctx.input, 16) - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - buf = buf + t - len = len - t - end - - while (len >= 64) do - copy(ctx.input, buf, 64) - byteReverse(ctx.input, 16) - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - buf = buf + 64 - len = len - 64 - end - - copy(ctx.input, buf, len) -end - -local function MD5Final(digest, ctx) - - local count - local p - - count = band(rshift(ctx.bits[0], 3), 0x3F) - - p = ctx.input + count - p[0] = 0x80 - p = p + 1 - count = 64 - 1 - count - - if (count < 8) then - fill(p, count, 0) - byteReverse(ctx.input, 16) - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - fill(ctx.input, 56, 0) - else - fill(p, count - 8, 0) - end - - byteReverse(ctx.input, 14) - - ffi.cast("uint32_t *", ctx.input)[14] = ctx.bits[0] - ffi.cast("uint32_t *", ctx.input)[15] = ctx.bits[1] - - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - byteReverse(ffi.cast("unsigned char *",ctx.buf), 4) - copy(digest, ctx.buf, 16) - fill(ffi.cast("char *", ctx), ffi.sizeof(ctx), 0) -end - -local hex = ffi.new("const char[16]", "0123456789abcdef") -local function bin2str(output, input, len) - if len > 0 then - output[0] = hex[rshift(input[0], 4)] - output[1] = hex[band(input[0], 0xF)] - return bin2str(output+2, input+1, len-1) - end -end - -local md5 = {} - ---- Create a new md5 hashing instance. ----- @return md5 instance -function md5:new() - self.ctx = ffi.new("MD5_CTX") - MD5Init(self.ctx) -end - ---- Feed content to md5 hashing instance. ----- @param luastr Lua string -function md5:update(luastr) - MD5Update(self.ctx, ffi.cast("const char*", luastr), #luastr) -end - ---- Calcualte md5 sum. ----- @param luastr Lua string ----- @return md5 sum in Lua string -function md5:sum(luastr) - local buf = ffi.new("char[33]") - local hash = ffi.new("uint8_t[16]") - if luastr then - md5:new() - md5:update(luastr) - end - MD5Final(hash, self.ctx) - - bin2str(buf, hash, ffi.sizeof(hash)) - - return ffi.string(buf) -end - -return md5 diff --git a/frontend/cache.lua b/frontend/cache.lua index 4fb859e79..099696687 100644 --- a/frontend/cache.lua +++ b/frontend/cache.lua @@ -1,7 +1,7 @@ --[[ A global LRU cache ]]-- -local md5 = require("MD5") +local md5 = require("ffi/MD5") local lfs = require("libs/libkoreader-lfs") local DataStorage = require("datastorage") local DEBUG = require("dbg") @@ -123,7 +123,7 @@ function Cache:check(key, ItemClass) end return self.cache[key] elseif ItemClass then - local cached = self.cached[md5:sum(key)] + local cached = self.cached[md5.sum(key)] if cached then local item = ItemClass:new{} local ok, msg = pcall(item.load, item, cached) @@ -153,14 +153,14 @@ function Cache:serialize() cached_size = cached_size + (lfs.attributes(file, "size") or 0) end table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end) - -- serialize the most recently used cache + -- only serialize the most recently used cache local cache_size = 0 for _, key in ipairs(self.cache_order) do local cache_item = self.cache[key] -- only dump cache item that requests serialization explicitly if cache_item.persistent and cache_item.dump then DEBUG("dump cache item", key) - cache_size = cache_item:dump(cache_path..md5:sum(key)) or 0 + cache_size = cache_item:dump(cache_path..md5.sum(key)) or 0 if cache_size > 0 then break end end end diff --git a/frontend/document/document.lua b/frontend/document/document.lua index 5536fa8d7..09992cb21 100644 --- a/frontend/document/document.lua +++ b/frontend/document/document.lua @@ -109,23 +109,23 @@ end -- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data -- by highlighting in koreader may change the digest value. function Document:fastDigest() - local md5 = require("MD5") + local md5 = require("ffi/MD5") local lshift = bit.lshift local file = io.open(self.file, 'rb') if file then local step, size = 1024, 1024 - md5:new() + local m = md5.new() for i = -1, 10 do file:seek("set", lshift(step, 2*i)) local sample = file:read(size) if sample then - md5:update(sample) + m:update(sample) else break end end file:close() - return md5:sum() + return m:sum() end end diff --git a/kodev b/kodev index 9dfecdd74..28fd38f2e 100755 --- a/kodev +++ b/kodev @@ -42,7 +42,7 @@ OPTIONS: TARGET: ${SUPPORTED_TARGETS}" - while [[ $1 == '--'* ]]; do + while [[ $1 == '-'* ]]; do PARAM=`echo $1 | awk -F= '{print $1}'` VALUE=`echo $1 | awk -F= '{print $2}'` case $PARAM in @@ -59,7 +59,7 @@ ${SUPPORTED_TARGETS}" exit 1 ;; esac - shift + shift 1 done case $1 in @@ -215,7 +215,7 @@ OPTIONS: --no-build run reader without rebuilding --disable-touch use this if you want to simulate keyboard only devices " - while [[ $1 == '--'* ]]; do + while [[ $1 == '-'* ]]; do PARAM=`echo $1 | awk -F= '{print $1}'` VALUE=`echo $1 | awk -F= '{print $2}'` case $PARAM in @@ -274,7 +274,7 @@ OPTIONS: --tags=TAGS only run tests with given tags " - while [[ $1 == '--'* ]]; do + while [[ $1 == '-'* ]]; do PARAM=`echo $1 | awk -F= '{print $1}'` VALUE=`echo $1 | awk -F= '{print $2}'` case $PARAM in @@ -308,7 +308,7 @@ OPTIONS: if [ ! -z $2 ]; then test_path="${test_path}/$2" fi - busted ${opts} -o verbose_print --exclude-tags=notest ${test_path} + busted --lua=./luajit ${opts} -o ./spec/$1/unit/verbose_print --exclude-tags=notest ${test_path} popd } diff --git a/spec/unit/kosync_spec.lua b/spec/unit/kosync_spec.lua index 99e9adb7d..2ad3034cc 100644 --- a/spec/unit/kosync_spec.lua +++ b/spec/unit/kosync_spec.lua @@ -1,7 +1,7 @@ require("commonrequire") local UIManager = require("ui/uimanager") local DEBUG = require("dbg") -local md5 = require("MD5") +local md5 = require("ffi/MD5") --DEBUG:turnOn() local service = [[ @@ -70,7 +70,7 @@ describe("KOSync modules #notest #nocov", function() req.headers['x-auth-key'] = args.userkey end -- password should be hashed before submitting to server - local username, password = "koreader", md5:sum("koreader") + local username, password = "koreader", md5.sum("koreader") -- fake progress data local doc, percentage, progress, device = "41cce710f34e5ec21315e19c99821415", -- fast digest of the document diff --git a/spec/unit/md5_spec.lua b/spec/unit/md5_spec.lua deleted file mode 100644 index 4efbe212a..000000000 --- a/spec/unit/md5_spec.lua +++ /dev/null @@ -1,19 +0,0 @@ -require("commonrequire") - -local md5 = require("MD5") - -describe("MD5 module", function() - it("should calculate correct MD5 hashes", function() - assert.is_equal(md5:sum(""), "d41d8cd98f00b204e9800998ecf8427e") - assert.is_equal(md5:sum("\0"), "93b885adfe0da089cdf634904fd59f71") - assert.is_equal(md5:sum("0123456789abcdefX"), "1b05aba914a8b12315c7ee52b42f3d35") - end) - it("should calculate MD5 sum by updating", function() - md5:new() - md5:update("0123456789") - md5:update("abcdefghij") - local md5sum = md5:sum() - assert.is_equal(md5sum, md5:sum("0123456789abcdefghij")) - end) -end) -