From 69f9ebd528f39649bb96c2aca4336e5c8e913dae Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Fri, 19 Feb 2016 09:39:05 -0800 Subject: [PATCH] build: fix doc generation also moved MD5 to koreader-base/ffi --- .ci/after_success.sh | 51 +++++-- .ci/common.sh | 6 + .ci/install.sh | 2 +- .ci/koreader_doc.enc | Bin 3248 -> 3264 bytes .ci/script.sh | 1 + .gitignore | 2 +- .travis.yml | 4 +- base | 2 +- frontend/MD5.lua | 246 --------------------------------- frontend/cache.lua | 8 +- frontend/document/document.lua | 8 +- kodev | 10 +- spec/unit/kosync_spec.lua | 4 +- spec/unit/md5_spec.lua | 19 --- 14 files changed, 66 insertions(+), 297 deletions(-) delete mode 100644 frontend/MD5.lua delete mode 100644 spec/unit/md5_spec.lua diff --git a/.ci/after_success.sh b/.ci/after_success.sh index 64709c2a5..58f02660a 100755 --- a/.ci/after_success.sh +++ b/.ci/after_success.sh @@ -1,16 +1,41 @@ #!/usr/bin/env bash +CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${CI_DIR}/common.sh" + +set +e + make coverage -cd koreader-*/koreader && luajit $(which luacov-coveralls) -v - -# get deploy key for doc repo -openssl aes-256-cbc -K $encrypted_dc71a4fb8382_key -iv $encrypted_dc71a4fb8382_iv \ - -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d -ssh-add ~/.ssh/koreader_doc -git clone git@github.com:koreader/doc.git -# push doc update -make doc -cp -r doc/html/* doc/ -pushd doc -git add . -git push origin gh-pages +pushd koreader-*/koreader + luajit $(which luacov-coveralls) -v +popd + +if [ $TRAVIS_BRANCH == 'master' ]; then + + travis_retry luarocks --local install ldoc + # get deploy key for doc repo + openssl aes-256-cbc -k $doc_build_secret -in .ci/koreader_doc.enc -out ~/.ssh/koreader_doc -d + chmod 600 ~/.ssh/koreader_doc # make agent happy + eval "$(ssh-agent)" > /dev/null + ssh-add ~/.ssh/koreader_doc > /dev/null + echo -e "\n${ANSI_GREEN}Check out koreader/doc for update." + git clone git@github.com:koreader/doc.git koreader_doc + + # push doc update + pushd doc + luajit $(which ldoc) . 2> /dev/null + if [ ! -d html ]; then + echo "Failed to generate documents..." + exit 1 + fi + popd + cp -r doc/html/* koreader_doc/ + pushd koreader_doc + + echo -e "\n${ANSI_GREEN}Pusing document update..." + git -c user.name="KOReader build bot" -c user.email="non-reply@koreader.rocks" \ + commit -a --amend -m 'Automated documentation build from travis-ci.' + git push -f --quiet origin gh-pages > /dev/null + echo -e "\n${ANSI_GREEN}Document update pushed." + +fi diff --git a/.ci/common.sh b/.ci/common.sh index 7fbabd4e3..0ee4bd104 100644 --- a/.ci/common.sh +++ b/.ci/common.sh @@ -1,3 +1,6 @@ +set -e +set -o pipefail + ANSI_RED="\033[31;1m" ANSI_GREEN="\033[32;1m" ANSI_RESET="\033[0m" @@ -6,6 +9,8 @@ ANSI_CLEAR="\033[0K" travis_retry() { local result=0 local count=1 + set +e + while [ $count -le 3 ]; do [ $result -ne 0 ] && { echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 @@ -21,6 +26,7 @@ travis_retry() { echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2 } + set -e return $result } diff --git a/.ci/install.sh b/.ci/install.sh index 95e7ca261..f43812d03 100755 --- a/.ci/install.sh +++ b/.ci/install.sh @@ -15,6 +15,7 @@ mkdir $HOME/.luarocks cp ${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua $HOME/.luarocks/config.lua echo "wrap_bin_scripts = false" >> $HOME/.luarocks/config.lua travis_retry luarocks --local install luafilesystem +# for verbose_print module travis_retry luarocks --local install ansicolors travis_retry luarocks --local install busted 2.0.rc11-0 #- travis_retry luarocks --local install busted 1.11.1-1 @@ -25,4 +26,3 @@ travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linu travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev travis_retry luarocks --local install luacheck travis_retry luarocks --local install lanes # for parallel luacheck -travis_retry luarocks --local install ldoc diff --git a/.ci/koreader_doc.enc b/.ci/koreader_doc.enc index c841e5e0c56edeb67047b20b50b7ef1ab975dc62..c961a46a9987b80db2ae8a73d6ce7af218b9222f 100644 GIT binary patch literal 3264 zcmV;x3_tTzVQh3|WM5zYeHmuSYu0ef7msGXsi{{y%eUpNw4IP`zX-|s zGF$Dpglq!Bd0Ln=B@wMxy;0FQ1Gu%WWmnoy%pM?zC9^^4pBgyQ&%M$MTUixA;1X%aw!`ry4)T=NPW_-HgbAn&IS- z#&+v~QiBkokrBg0W?;WxF%aVx-`TFlQs=SGxW)+Po2Ah^plLwVXt}DZx;OzwZLazy z3%51CMA#DLQGOi~Ow+m7L3|$EzxEt6?EGHN-kk5HW^XT;6!^OBx#ux)KeRDD` z*m=D2eZ)xY#@#`zT>nCy?!SLtgFlKWe>P!0tR8VCCz4tfP0vU3xlLD99=E{)0k9bFLHwG>=67i1`Ncf3#XO0 zJxZVYu^ZF0dYgc5u? z)(>V-nQTet$s-Hy|4NaCo|oc*W5=D~&I~i;>4D{=&EfQJN9eelns$jszITvXFH4`} zoy^)F>KO&NR`!xI0c0HhMMm-BuATv(J_q6zfs?;0sv#$yG13qn|GFPXipHHcV^`@E zMO-A{I$M9?R`EWROpgeN)B}Uy!YUU%6D73cpHab<>dm_O!$pCj!ZQ@uyFvA=ZJhe5 zU_A&70KDg38sS@qjJ#faWXIvEYusHTQ(~n^tn$C85R{(88I$0%XA6^ETDb9U@c@$U z%@+v=FD*6@^3@-^?rLwk4kjGC=2)> z|Dzt2Bj+v$+&TL@5fv7TcE}jnG>m$YBV0-YK=`9?_t9r(*cF^UMCq9@ zG%}gRiE{}?qbQ@#x$Q}+Dk}jUJV5~J_h+4f5sM8!FIqP?KF=);H&HiPo>a^^){=;` z&Yfp>q=a+f*LrtTcfgIG1)uNveIQcsX+VtE=o~9(Zrb-1PGJ~RbSer+I}yvJNdk*M z2dZx=VDm7sP+{QPz1oi6 zYv~i6w-+Dk;X$Yna0<~y_jz@7>ZvO69|0{6a1*HVjU3rKtF}K`64LG}CJnYoULvT2cHnM@1Iw zFLE>fhGXz)VewXU)H;iw6M5R9?hKW8ml#+LmD{0EQNmeKp{UWJ*vpv%>Itq zuL>QCkMX)56pMeDVb{0-h=bAE=y}3171p9`Me|;l>N11RV-ei2>%}t&+Q%9TZ$F$<|-ypQ!; zN<9o=sWQm8GKcrMv|AmY`^U~1PQOkM1D#=j5IsCFkYcL6_08l#_=ljuh-;BjW)e*cu;Fh&fbC>ggK0B#2f2C9l7t`+uru6OaI!hD^*13O&|>Ltihw zi1|3iePtAQBzmeLZRK^2`ycP_CU}daT+BSW+83^{^9dMH4oLC^Skx>$?QfkvhZExf9v2M zYn!qYA1*=-aEJa)mFav#Z>6~O6_AA@p?H{>9yN8Jc)5R~F5K>ZcPA85paNUCNkW)X zeU)_}CGD?8jCe7@6&NZ7|4JyCHU;ltn>=!*g9q-98A+izAjA;(p)@v8Ij4?}dO>7@ zprH-O{BXD#W+q1Y5m@8LsH))K+G)SmOGD#q2TC1gQAdZN&}u>(R^cVyHtR&u2ly73r&0D{oZ7s7Vlxet2ecMg(n4E{IFe$V`FlsnO9D3Pjke*GO?3Ajx zwSp0MgI&uP?*)qH-uYkG$8{WQ!6VKt z3?AhFQqACI^M#PeZ6O@SM_DTcRTS6g!&4%^@JQg2IRPE&r15Q)KFys9uekl20Mf3% zhhsALaxlSVBH0hf7IdIv4Va-^hI?#9IiqgTvc5l$7(o4nyqbHNVA2(u%n!lwPodOh zb+@N*q~pS_?1Z)<(tF*|1O|UGKQ?4HAFTkH6;=`Xks1%#UF$hbAgR^-m9z|k`rs9A zYDPanvm~fr`l^UjMVbLpaFq)POmwX7Q-hf}`9%qSML^ap_AmDDU2Gh4OG&Z5z0RBPa&haXX@}jmw@P7PHED0Y33FG5n!IV!zN^@&n$|*5Y8kgS7w>lfB5h`ODPGW0>`u*aT(^u}nsTWW0595=wvCE^7jd!h6j+ zSHdI5-ag5kJamNNg%4Zv29z10Y2r|L-2F9IxvLU|4x-ITRJKcf4r-dnaZiNG7^KNw zYd8mvGS@bi7keE8)^ml1)&0hcdRYL9Ihf^iB@T?7zU2U%Yf~6ev@q8a5LfIpu5IJ| z^eiRpnVV79G(NCh$o72X4MmnmF*@EDV%GmOR>B^q=9l&?+QNUsgl*?P|9P*C8Wrnr-xF;y{!!^bDPmO zizYTm0^hH4<7lpSRhB`b^~~2y$w5(^Hy>U+P>kgE4^v!dK7!8chMDWluMng2FWdTmMV|X{Ncy01P9IT$lh2A+_pNa z578`)c)<+5#E+(!V4@Pz-|Gpa=5y^8g42Te=BYymT-QZ5-J+(oli++P367ilnq)Kd ztdqDodXg+A<=p{Tn$P^iT%KZ#jR?IEwhw}YE=MN_H4?^J`>1{hRyV#q2K41o0H8Ot z$2f21+5PSA8CyBMoXwoq0!V5^2*0;fWjnau{TUID&L`&bG#r$$CatWFX&+q}n_&-Dqlv%pq z$}DsGic7U_`bkcGSG-3j6(z~)DGC6MkDoRG_u@A^z!uj7-OfT$XcA~6W|lYODHS6hj4R{&dLd;gkZEBY@HkmPq zkU94u@q2MhT?T~*+Ohuf^LSB$9-78{HB%%~sJJ_W*tK4A*Kdh+`V*S@FAA1HilWSh{oqV0aP1T z7o_SVdYrj6B=IzSmhw`nzS}1#P1H%je~6iay-;_AEjschuS+l@lftJ1bGvz2(78wS8Jda!(43+iYR7>2QPhT`3#4 z110^!D8{+S9QAJm7$k7mYTjw*SLWJzW{SE;T*RZU&u&_1lc(QtO`ep zNvIIi-ea*2%i=+3`smTjK!U(CxAqbCG)y%lJT9efbNdvRl*f|UeIp&MBfX$B+ox)% z+d5iCU6a_C{j6tNs>Ey3^$1?Im{4TpWM@h75@pyzD$_>5Th1rvJni`)-lrF2mQ0c1 zyj!)=B@Iwti&hh55{gN}wNr}R7Av5p(w+#ZAb#D<3CZqab=Lclu00q}4V{JTnB-N$ z#LzpF2wnE6MYL{QXt&NP1eyOEL4w%vX|li>8hk%$2m;coy)*stFRc{BPU)(%-`TWM z`sMSo61tZK(Kozh{b)Ws)oI-yK3WAy!0N)KjK~$2BPSZ)dqk5e;hnN`9J&BhGtMlp z@Zh8isQd*;YYRgvM0_TqaPHrqv*tdwmpu2z}-< zS{rLg*WZfQ{!XK&jl$s85iV7}?5v^wGa9)ZQF*M`4hUv8q=hTb8jXIstTq)3AtLyomT z<8l3OVHTKgd`HNBxa%h`1!=;hCs$dI+4H(!2Z0XoJ_1_PbYgt2o!$Kwsg#XlDJ`Bw^Gak4{9 zc`+9E>-uC=c%QI&;cD+oZzmav+yo4%tf^kC0&N0p;hzoir$%wX5^oz0l*=+j1D@w# zo!4ltCjx(x)#QoBl@qUT1F2C;>Vs?Syco`-I}(Thnh~rp&O>h!Q17U{I4z>W;|(yf z>UYJwx9zBDa5#n2vkwQD>{dv$rdbEV(h8kB) zlr=6Fm0UIob~%OGi0vbl4 zek-94dB4#h$)B%1V{R7=v}gkEdh+YHT5Co&bhNTVqh!K{RCJ0C>=+N`7Ghicj_%Fw zNudqjxh!J+F(9OMv^>F$4ZWz>;mD&%fo^SqoOoJ;jzT?LOGXpmXF@h#hQmuhf~anm zJ|eBkPK-2nA3w!b@psriSH_am|9gOTls-S=>As6N7ZG&68F4?R)a7R9`GgNuY~$v; z7U{0gy%?h)%j|4Ctx2!WB5Nf(HW!`9Rx*Y_(;7KC&=|Wx0z9U4A zV|-e;6M*&6*luIC7{|IBi%vRg-QxoaXl5Vi)92L=%>OD_pwtbZ=(HpE58hNZ6a|2JNtoXsse@bv?k>#a#lNuha<=V?7b z^6^*zd@b3-&}vQW80cGFEv6WXsWkwsxlE(z7Y@15cB9lWuJAM2>~Lwri5~kfI*Ki3 zRgeT}Bn&yVv`r&{NgfgPa7g~^a6M;5e zs!n?rS%r&#zZ4DHJu=XU5{qFwHGxho$&5HmH3yBn zxGD5eg5Hnu95V=;UhfaqgOWky+-D iV-mH-tBAI!kjZKS*dl@-8%!3iaI0vyr 0) then - local p = ffi.cast("unsigned char *", ctx.input + t) - - t = 64 - t - if (len < t) then - copy(p, buf, len) - return - end - - copy(p, buf, t) - byteReverse(ctx.input, 16) - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - buf = buf + t - len = len - t - end - - while (len >= 64) do - copy(ctx.input, buf, 64) - byteReverse(ctx.input, 16) - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - buf = buf + 64 - len = len - 64 - end - - copy(ctx.input, buf, len) -end - -local function MD5Final(digest, ctx) - - local count - local p - - count = band(rshift(ctx.bits[0], 3), 0x3F) - - p = ctx.input + count - p[0] = 0x80 - p = p + 1 - count = 64 - 1 - count - - if (count < 8) then - fill(p, count, 0) - byteReverse(ctx.input, 16) - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - fill(ctx.input, 56, 0) - else - fill(p, count - 8, 0) - end - - byteReverse(ctx.input, 14) - - ffi.cast("uint32_t *", ctx.input)[14] = ctx.bits[0] - ffi.cast("uint32_t *", ctx.input)[15] = ctx.bits[1] - - MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)) - byteReverse(ffi.cast("unsigned char *",ctx.buf), 4) - copy(digest, ctx.buf, 16) - fill(ffi.cast("char *", ctx), ffi.sizeof(ctx), 0) -end - -local hex = ffi.new("const char[16]", "0123456789abcdef") -local function bin2str(output, input, len) - if len > 0 then - output[0] = hex[rshift(input[0], 4)] - output[1] = hex[band(input[0], 0xF)] - return bin2str(output+2, input+1, len-1) - end -end - -local md5 = {} - ---- Create a new md5 hashing instance. ----- @return md5 instance -function md5:new() - self.ctx = ffi.new("MD5_CTX") - MD5Init(self.ctx) -end - ---- Feed content to md5 hashing instance. ----- @param luastr Lua string -function md5:update(luastr) - MD5Update(self.ctx, ffi.cast("const char*", luastr), #luastr) -end - ---- Calcualte md5 sum. ----- @param luastr Lua string ----- @return md5 sum in Lua string -function md5:sum(luastr) - local buf = ffi.new("char[33]") - local hash = ffi.new("uint8_t[16]") - if luastr then - md5:new() - md5:update(luastr) - end - MD5Final(hash, self.ctx) - - bin2str(buf, hash, ffi.sizeof(hash)) - - return ffi.string(buf) -end - -return md5 diff --git a/frontend/cache.lua b/frontend/cache.lua index 4fb859e79..099696687 100644 --- a/frontend/cache.lua +++ b/frontend/cache.lua @@ -1,7 +1,7 @@ --[[ A global LRU cache ]]-- -local md5 = require("MD5") +local md5 = require("ffi/MD5") local lfs = require("libs/libkoreader-lfs") local DataStorage = require("datastorage") local DEBUG = require("dbg") @@ -123,7 +123,7 @@ function Cache:check(key, ItemClass) end return self.cache[key] elseif ItemClass then - local cached = self.cached[md5:sum(key)] + local cached = self.cached[md5.sum(key)] if cached then local item = ItemClass:new{} local ok, msg = pcall(item.load, item, cached) @@ -153,14 +153,14 @@ function Cache:serialize() cached_size = cached_size + (lfs.attributes(file, "size") or 0) end table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end) - -- serialize the most recently used cache + -- only serialize the most recently used cache local cache_size = 0 for _, key in ipairs(self.cache_order) do local cache_item = self.cache[key] -- only dump cache item that requests serialization explicitly if cache_item.persistent and cache_item.dump then DEBUG("dump cache item", key) - cache_size = cache_item:dump(cache_path..md5:sum(key)) or 0 + cache_size = cache_item:dump(cache_path..md5.sum(key)) or 0 if cache_size > 0 then break end end end diff --git a/frontend/document/document.lua b/frontend/document/document.lua index 5536fa8d7..09992cb21 100644 --- a/frontend/document/document.lua +++ b/frontend/document/document.lua @@ -109,23 +109,23 @@ end -- 1048576, 4194304, 16777216, 67108864, 268435456 or 1073741824, appending data -- by highlighting in koreader may change the digest value. function Document:fastDigest() - local md5 = require("MD5") + local md5 = require("ffi/MD5") local lshift = bit.lshift local file = io.open(self.file, 'rb') if file then local step, size = 1024, 1024 - md5:new() + local m = md5.new() for i = -1, 10 do file:seek("set", lshift(step, 2*i)) local sample = file:read(size) if sample then - md5:update(sample) + m:update(sample) else break end end file:close() - return md5:sum() + return m:sum() end end diff --git a/kodev b/kodev index 9dfecdd74..28fd38f2e 100755 --- a/kodev +++ b/kodev @@ -42,7 +42,7 @@ OPTIONS: TARGET: ${SUPPORTED_TARGETS}" - while [[ $1 == '--'* ]]; do + while [[ $1 == '-'* ]]; do PARAM=`echo $1 | awk -F= '{print $1}'` VALUE=`echo $1 | awk -F= '{print $2}'` case $PARAM in @@ -59,7 +59,7 @@ ${SUPPORTED_TARGETS}" exit 1 ;; esac - shift + shift 1 done case $1 in @@ -215,7 +215,7 @@ OPTIONS: --no-build run reader without rebuilding --disable-touch use this if you want to simulate keyboard only devices " - while [[ $1 == '--'* ]]; do + while [[ $1 == '-'* ]]; do PARAM=`echo $1 | awk -F= '{print $1}'` VALUE=`echo $1 | awk -F= '{print $2}'` case $PARAM in @@ -274,7 +274,7 @@ OPTIONS: --tags=TAGS only run tests with given tags " - while [[ $1 == '--'* ]]; do + while [[ $1 == '-'* ]]; do PARAM=`echo $1 | awk -F= '{print $1}'` VALUE=`echo $1 | awk -F= '{print $2}'` case $PARAM in @@ -308,7 +308,7 @@ OPTIONS: if [ ! -z $2 ]; then test_path="${test_path}/$2" fi - busted ${opts} -o verbose_print --exclude-tags=notest ${test_path} + busted --lua=./luajit ${opts} -o ./spec/$1/unit/verbose_print --exclude-tags=notest ${test_path} popd } diff --git a/spec/unit/kosync_spec.lua b/spec/unit/kosync_spec.lua index 99e9adb7d..2ad3034cc 100644 --- a/spec/unit/kosync_spec.lua +++ b/spec/unit/kosync_spec.lua @@ -1,7 +1,7 @@ require("commonrequire") local UIManager = require("ui/uimanager") local DEBUG = require("dbg") -local md5 = require("MD5") +local md5 = require("ffi/MD5") --DEBUG:turnOn() local service = [[ @@ -70,7 +70,7 @@ describe("KOSync modules #notest #nocov", function() req.headers['x-auth-key'] = args.userkey end -- password should be hashed before submitting to server - local username, password = "koreader", md5:sum("koreader") + local username, password = "koreader", md5.sum("koreader") -- fake progress data local doc, percentage, progress, device = "41cce710f34e5ec21315e19c99821415", -- fast digest of the document diff --git a/spec/unit/md5_spec.lua b/spec/unit/md5_spec.lua deleted file mode 100644 index 4efbe212a..000000000 --- a/spec/unit/md5_spec.lua +++ /dev/null @@ -1,19 +0,0 @@ -require("commonrequire") - -local md5 = require("MD5") - -describe("MD5 module", function() - it("should calculate correct MD5 hashes", function() - assert.is_equal(md5:sum(""), "d41d8cd98f00b204e9800998ecf8427e") - assert.is_equal(md5:sum("\0"), "93b885adfe0da089cdf634904fd59f71") - assert.is_equal(md5:sum("0123456789abcdefX"), "1b05aba914a8b12315c7ee52b42f3d35") - end) - it("should calculate MD5 sum by updating", function() - md5:new() - md5:update("0123456789") - md5:update("abcdefghij") - local md5sum = md5:sum() - assert.is_equal(md5sum, md5:sum("0123456789abcdefghij")) - end) -end) -