From 775e5ea3b48bb447894dc335aae5ba3450e236de Mon Sep 17 00:00:00 2001 From: chrox Date: Wed, 30 Apr 2014 23:24:44 +0800 Subject: [PATCH] serialize the most recently used blitbuffer/koptcontext to speedup koreader startup for PDF/DJVU documents especially when reflowing --- frontend/MD5.lua | 233 ++++++++++++++++++++ frontend/cache.lua | 69 +++++- frontend/document/credocument.lua | 13 +- frontend/document/document.lua | 5 +- frontend/document/koptinterface.lua | 29 ++- frontend/document/tilecacheitem.lua | 17 ++ frontend/memutils.lua | 131 +++++++++++ frontend/stringzutils.lua | 322 ++++++++++++++++++++++++++++ koreader-base | 2 +- 9 files changed, 810 insertions(+), 11 deletions(-) create mode 100644 frontend/MD5.lua create mode 100644 frontend/memutils.lua create mode 100644 frontend/stringzutils.lua diff --git a/frontend/MD5.lua b/frontend/MD5.lua new file mode 100644 index 000000000..bf72a0d47 --- /dev/null +++ b/frontend/MD5.lua @@ -0,0 +1,233 @@ + +local ffi = require "ffi" +local bit = require "bit" +local bxor = bit.bxor +local bnot = bit.bnot +local band = bit.band +local bor = bit.bor +local rshift = bit.rshift +local lshift = bit.lshift + +require "memutils" +require "stringzutils" + +ffi.cdef[[ +typedef struct MD5Context { + uint32_t buf[4]; + uint32_t bits[2]; + unsigned char input[64]; +} MD5_CTX; +]] + +MD5_CTX = ffi.typeof("MD5_CTX"); + +function byteReverse(buf, len) +end + +function F1(x, y, z) return bxor(z, band(x, bxor(y, z))) end +function F2(x, y, z) return F1(z, x, y) end +function F3(x, y, z) return bxor(x, y, z) end +function F4(x, y, z) return bxor(y, bor(x, bnot(z))) end + +function MD5STEP(f, w, x, y, z, data, s) + w = w + f(x, y, z) + data; + w = bor(lshift(w,s), rshift(w,(32-s))) + w = w + x; + + return w; +end + +function printmd5ctx(ctx) + for i=0,3 do + print(string.format("ctx.buf[%d]: 0x%x", i, ctx.buf[i])); + end + + print(string.format("ctx.bits[0]: %d", ctx.bits[0])); + print(string.format("ctx.bits[1]: %d", ctx.bits[1])); +end + +-- Start MD5 accumulation. Set bit count to 0 and buffer to mysterious +-- initialization constants. +function MD5Init(ctx) + ctx.buf[0] = 0x67452301; + ctx.buf[1] = 0xefcdab89; + ctx.buf[2] = 0x98badcfe; + ctx.buf[3] = 0x10325476; + + ctx.bits[0] = 0; + ctx.bits[1] = 0; +end + +function MD5Transform(buf, input) + local a = buf[0]; + local b = buf[1]; + local c = buf[2]; + local d = buf[3]; + + a = MD5STEP(F1, a, b, c, d, input[0] + 0xd76aa478, 7); + d = MD5STEP(F1, d, a, b, c, input[1] + 0xe8c7b756, 12); + c = MD5STEP(F1, c, d, a, b, input[2] + 0x242070db, 17); + b = MD5STEP(F1, b, c, d, a, input[3] + 0xc1bdceee, 22); + a = MD5STEP(F1, a, b, c, d, input[4] + 0xf57c0faf, 7); + d = MD5STEP(F1, d, a, b, c, input[5] + 0x4787c62a, 12); + c = MD5STEP(F1, c, d, a, b, input[6] + 0xa8304613, 17); + b = MD5STEP(F1, b, c, d, a, input[7] + 0xfd469501, 22); + a = MD5STEP(F1, a, b, c, d, input[8] + 0x698098d8, 7); + d = MD5STEP(F1, d, a, b, c, input[9] + 0x8b44f7af, 12); + c = MD5STEP(F1, c, d, a, b, input[10] + 0xffff5bb1, 17); + b = MD5STEP(F1, b, c, d, a, input[11] + 0x895cd7be, 22); + a = MD5STEP(F1, a, b, c, d, input[12] + 0x6b901122, 7); + d = MD5STEP(F1, d, a, b, c, input[13] + 0xfd987193, 12); + c = MD5STEP(F1, c, d, a, b, input[14] + 0xa679438e, 17); + b = MD5STEP(F1, b, c, d, a, input[15] + 0x49b40821, 22); + + a = MD5STEP(F2, a, b, c, d, input[1] + 0xf61e2562, 5); + d = MD5STEP(F2, d, a, b, c, input[6] + 0xc040b340, 9); + c = MD5STEP(F2, c, d, a, b, input[11] + 0x265e5a51, 14); + b = MD5STEP(F2, b, c, d, a, input[0] + 0xe9b6c7aa, 20); + a = MD5STEP(F2, a, b, c, d, input[5] + 0xd62f105d, 5); + d = MD5STEP(F2, d, a, b, c, input[10] + 0x02441453, 9); + c = MD5STEP(F2, c, d, a, b, input[15] + 0xd8a1e681, 14); + b = MD5STEP(F2, b, c, d, a, input[4] + 0xe7d3fbc8, 20); + a = MD5STEP(F2, a, b, c, d, input[9] + 0x21e1cde6, 5); + d = MD5STEP(F2, d, a, b, c, input[14] + 0xc33707d6, 9); + c = MD5STEP(F2, c, d, a, b, input[3] + 0xf4d50d87, 14); + b = MD5STEP(F2, b, c, d, a, input[8] + 0x455a14ed, 20); + a = MD5STEP(F2, a, b, c, d, input[13] + 0xa9e3e905, 5); + d = MD5STEP(F2, d, a, b, c, input[2] + 0xfcefa3f8, 9); + c = MD5STEP(F2, c, d, a, b, input[7] + 0x676f02d9, 14); + b = MD5STEP(F2, b, c, d, a, input[12] + 0x8d2a4c8a, 20); + + a = MD5STEP(F3, a, b, c, d, input[5] + 0xfffa3942, 4); + d = MD5STEP(F3, d, a, b, c, input[8] + 0x8771f681, 11); + c = MD5STEP(F3, c, d, a, b, input[11] + 0x6d9d6122, 16); + b = MD5STEP(F3, b, c, d, a, input[14] + 0xfde5380c, 23); + a = MD5STEP(F3, a, b, c, d, input[1] + 0xa4beea44, 4); + d = MD5STEP(F3, d, a, b, c, input[4] + 0x4bdecfa9, 11); + c = MD5STEP(F3, c, d, a, b, input[7] + 0xf6bb4b60, 16); + b = MD5STEP(F3, b, c, d, a, input[10] + 0xbebfbc70, 23); + a = MD5STEP(F3, a, b, c, d, input[13] + 0x289b7ec6, 4); + d = MD5STEP(F3, d, a, b, c, input[0] + 0xeaa127fa, 11); + c = MD5STEP(F3, c, d, a, b, input[3] + 0xd4ef3085, 16); + b = MD5STEP(F3, b, c, d, a, input[6] + 0x04881d05, 23); + a = MD5STEP(F3, a, b, c, d, input[9] + 0xd9d4d039, 4); + d = MD5STEP(F3, d, a, b, c, input[12] + 0xe6db99e5, 11); + c = MD5STEP(F3, c, d, a, b, input[15] + 0x1fa27cf8, 16); + b = MD5STEP(F3, b, c, d, a, input[2] + 0xc4ac5665, 23); + + a = MD5STEP(F4, a, b, c, d, input[0] + 0xf4292244, 6); + d = MD5STEP(F4, d, a, b, c, input[7] + 0x432aff97, 10); + c = MD5STEP(F4, c, d, a, b, input[14] + 0xab9423a7, 15); + b = MD5STEP(F4, b, c, d, a, input[5] + 0xfc93a039, 21); + a = MD5STEP(F4, a, b, c, d, input[12] + 0x655b59c3, 6); + d = MD5STEP(F4, d, a, b, c, input[3] + 0x8f0ccc92, 10); + c = MD5STEP(F4, c, d, a, b, input[10] + 0xffeff47d, 15); + b = MD5STEP(F4, b, c, d, a, input[1] + 0x85845dd1, 21); + a = MD5STEP(F4, a, b, c, d, input[8] + 0x6fa87e4f, 6); + d = MD5STEP(F4, d, a, b, c, input[15] + 0xfe2ce6e0, 10); + c = MD5STEP(F4, c, d, a, b, input[6] + 0xa3014314, 15); + b = MD5STEP(F4, b, c, d, a, input[13] + 0x4e0811a1, 21); + a = MD5STEP(F4, a, b, c, d, input[4] + 0xf7537e82, 6); + d = MD5STEP(F4, d, a, b, c, input[11] + 0xbd3af235, 10); + c = MD5STEP(F4, c, d, a, b, input[2] + 0x2ad7d2bb, 15); + b = MD5STEP(F4, b, c, d, a, input[9] + 0xeb86d391, 21); + + buf[0] = (buf[0] + a)%0xffffffff; + buf[1] = (buf[1] + b)%0xffffffff; + buf[2] = (buf[2] + c)%0xffffffff; + buf[3] = (buf[3] + d)%0xffffffff; +end + +function MD5Update(ctx, buf, len) + local t; + + t = ctx.bits[0]; + ctx.bits[0] = t + lshift( len, 3) + if (ctx.bits[0] < t) then + ctx.bits[1] = ctx.bits[1] + 1; + end + + ctx.bits[1] = ctx.bits[1] + rshift(len, 29); + + t = band(rshift(t, 3), 0x3f); + + if (t > 0) then + p = ffi.cast("unsigned char *", ctx.input + t); + + t = 64 - t; + if (len < t) then + memcpy(p, buf, len); + return; + end + + memcpy(p, buf, t); + byteReverse(ctx.input, 16); + MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)); + buf = buf + t; + len = len - t; + end + + while (len >= 64) do + memcpy(ctx.input, buf, 64); + byteReverse(ctx.input, 16); + MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)); + buf = buf + 64; + len = len - 64; + end + + memcpy(ctx.input, buf, len); +end + +function MD5Final(digest, ctx) + + local count; + local p; + + count = band(rshift(ctx.bits[0], 3), 0x3F); + + p = ctx.input + count; + p[0] = 0x80; + p = p + 1; + count = 64 - 1 - count; + + if (count < 8) then + memset(p, 0, count); + byteReverse(ctx.input, 16); + MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)); + memset(ctx.input, 0, 56); + else + memset(p, 0, count - 8); + end + + byteReverse(ctx.input, 14); + + ffi.cast("uint32_t *", ctx.input)[14] = ctx.bits[0]; + ffi.cast("uint32_t *", ctx.input)[15] = ctx.bits[1]; + + MD5Transform(ctx.buf, ffi.cast("uint32_t *", ctx.input)); + byteReverse(ffi.cast("unsigned char *",ctx.buf), 4); + memcpy(digest, ctx.buf, 16); + memset(ffi.cast("char *", ctx), 0, ffi.sizeof(ctx)); +end + + +function md5(luastr) + local buf = ffi.new("char[33]"); + local hash = ffi.new("uint8_t[16]"); + local len = #luastr + local p = ffi.cast("const char *", luastr); + + local ctx = MD5_CTX(); + + MD5Init(ctx); + + MD5Update(ctx, p, len); + MD5Final(hash, ctx); + bin2str(buf, hash, ffi.sizeof(hash)); + + return ffi.string(buf); +end + + + + diff --git a/frontend/cache.lua b/frontend/cache.lua index 4d4312706..3d4627f97 100644 --- a/frontend/cache.lua +++ b/frontend/cache.lua @@ -1,6 +1,9 @@ --[[ A global LRU cache ]]-- +require("MD5") +local DEBUG = require("dbg") + local function calcFreeMem() local meminfo = io.open("/proc/meminfo", "r") local freemem = 0 @@ -26,6 +29,22 @@ local function calcCacheMemSize() return math.min(max, math.max(min, calc)) end +local cache_path = lfs.currentdir().."/cache/" + +--[[ +-- return a snapshot of disk cached items for subsequent check +--]] +function getDiskCache() + local cached = {} + for key_md5 in lfs.dir(cache_path) do + local file = cache_path..key_md5 + if lfs.attributes(file, "mode") == "file" then + cached[key_md5] = file + end + end + return cached +end + local Cache = { -- cache configuration: max_memsize = calcCacheMemSize(), @@ -34,7 +53,9 @@ local Cache = { -- associative cache cache = {}, -- this will hold the LRU order of the cache - cache_order = {} + cache_order = {}, + -- disk Cache snapshot + cached = getDiskCache(), } function Cache:new(o) @@ -64,7 +85,11 @@ function Cache:insert(key, object) self.current_memsize = self.current_memsize + object.size end -function Cache:check(key) +--[[ +-- check for cache item for key +-- if ItemClass is given, disk cache is also checked. +--]] +function Cache:check(key, ItemClass) if self.cache[key] then if self.cache_order[1] ~= key then -- put key in front of the LRU list @@ -76,6 +101,18 @@ function Cache:check(key) table.insert(self.cache_order, 1, key) end return self.cache[key] + elseif ItemClass then + local cached = self.cached[md5(key)] + if cached then + local item = ItemClass:new{} + local ok, msg = pcall(item.load, item, cached) + if ok then + self:insert(key, item) + return item + else + DEBUG("discard cache", msg) + end + end end end @@ -86,6 +123,34 @@ function Cache:willAccept(size) end end +function Cache:serialize() + -- calculate disk cache size + local cached_size = 0 + local sorted_caches = {} + for _,file in pairs(self.cached) do + table.insert(sorted_caches, {file=file, time=lfs.attributes(file, "access")}) + cached_size = cached_size + (lfs.attributes(file, "size") or 0) + end + table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end) + -- serialize the most recently used cache + local cache_size = 0 + for _, key in ipairs(self.cache_order) do + if self.cache[key].dump then + cache_size = self.cache[key]:dump(cache_path..md5(key)) or 0 + if cache_size > 0 then break end + end + end + -- set disk cache the same limit as memory cache + while cached_size + cache_size - self.max_memsize > 0 do + -- discard the least recently used cache + local discarded = table.remove(sorted_caches) + cached_size = cached_size - lfs.attributes(discarded.file, "size") + os.remove(discarded.file) + end + -- disk cache may have changes so need to refresh disk cache snapshot + self.cached = getDiskCache() +end + -- blank the cache function Cache:clear() for k, _ in pairs(self.cache) do diff --git a/frontend/document/credocument.lua b/frontend/document/credocument.lua index 16914099e..8ec52d0d0 100644 --- a/frontend/document/credocument.lua +++ b/frontend/document/credocument.lua @@ -40,10 +40,21 @@ function CreDocument.zipContentExt(self, fname) return string.lower(string.match(s, ".+%.([^.]+)")) end +function CreDocument:cacheInit() + -- remove legacy cr3cache directory + if lfs.attributes("./cr3cache", "mode") == "directory" then + os.execute("rm -r ./cr3cache") + end + cre.initCache("./cache/cr3cache", 1024*1024*32) +end + function CreDocument:engineInit() if not engine_initilized then -- initialize cache - cre.initCache(1024*1024*64) + self:cacheInit() + + -- initialize hyph dictionaries + cre.initHyphDict("./data/hyph") -- we need to initialize the CRE font list local fonts = Font:getFontList() diff --git a/frontend/document/document.lua b/frontend/document/document.lua index 9d180b3b4..c04184607 100644 --- a/frontend/document/document.lua +++ b/frontend/document/document.lua @@ -78,6 +78,7 @@ function Document:close() self.is_open = false self._document:close() end + Cache:serialize() end -- this might be overridden by a document implementation @@ -253,7 +254,7 @@ end -- TODO: this should trigger a background operation function Document:hintPage(pageno, zoom, rotation, gamma, render_mode) local hash_full_page = "renderpg|"..self.file.."|"..pageno.."|"..zoom.."|"..rotation.."|"..gamma.."|"..render_mode - if not Cache:check(hash_full_page) then + if not Cache:check(hash_full_page, TileCacheItem) then DEBUG("hinting page", pageno) self:renderPage(pageno, nil, zoom, rotation, gamma, render_mode) end @@ -270,7 +271,7 @@ Draw page content to blitbuffer. function Document:drawPage(target, x, y, rect, pageno, zoom, rotation, gamma, render_mode) local hash_full_page = "renderpg|"..self.file.."|"..pageno.."|"..zoom.."|"..rotation.."|"..gamma.."|"..render_mode local hash_excerpt = hash_full_page.."|"..tostring(rect) - local tile = Cache:check(hash_full_page) + local tile = Cache:check(hash_full_page, TileCacheItem) if not tile then tile = Cache:check(hash_excerpt) if not tile then diff --git a/frontend/document/koptinterface.lua b/frontend/document/koptinterface.lua index e36c58ffb..3d92e0882 100644 --- a/frontend/document/koptinterface.lua +++ b/frontend/document/koptinterface.lua @@ -1,11 +1,12 @@ +local TileCacheItem = require("document/tilecacheitem") +local KOPTContext = require("ffi/koptcontext") local Document = require("document/document") -local Cache = require("cache") local CacheItem = require("cacheitem") local Screen = require("ui/screen") local Geom = require("ui/geometry") -local TileCacheItem = require("document/tilecacheitem") +local serial = require("serialize") +local Cache = require("cache") local DEBUG = require("dbg") -local KOPTContext = require("ffi/koptcontext") local KoptInterface = { ocrengine = "ocrengine", @@ -26,6 +27,20 @@ function ContextCacheItem:onFree() end end +function ContextCacheItem:dump(filename) + if self.kctx:isPreCache() == 0 then + DEBUG("dumping koptcontext to", filename) + return serial.dump(self.size, KOPTContext.totable(self.kctx), filename) + end +end + +function ContextCacheItem:load(filename) + DEBUG("loading koptcontext from", filename) + local size, kc_table = serial.load(filename) + self.size = size + self.kctx = KOPTContext.fromtable(kc_table) +end + local OCREngine = CacheItem:new{} function OCREngine:onFree() @@ -182,7 +197,7 @@ function KoptInterface:getCachedContext(doc, pageno) local bbox = doc:getPageBBox(pageno) local context_hash = self:getContextHash(doc, pageno, bbox) local kctx_hash = "kctx|"..context_hash - local cached = Cache:check(kctx_hash) + local cached = Cache:check(kctx_hash, ContextCacheItem) if not cached then -- If kctx is not cached, create one and get reflowed bmp in foreground. local kc = self:createContext(doc, pageno, bbox) @@ -304,7 +319,11 @@ function KoptInterface:renderOptimizedPage(doc, pageno, rect, zoom, rotation, re -- prepare cache item with contained blitbuffer local tile = TileCacheItem:new{ size = fullwidth * fullheight / 2 + 64, -- estimation - excerpt = Geom:new{ w = fullwidth, h = fullheight }, + excerpt = Geom:new{ + x = 0, y = 0, + w = fullwidth, + h = fullheight + }, pageno = pageno, } tile.bb = kc:dstToBlitBuffer() diff --git a/frontend/document/tilecacheitem.lua b/frontend/document/tilecacheitem.lua index a532d52d2..a3c2b38f7 100644 --- a/frontend/document/tilecacheitem.lua +++ b/frontend/document/tilecacheitem.lua @@ -1,4 +1,6 @@ +local Blitbuffer = require("ffi/blitbuffer") local CacheItem = require("cacheitem") +local serial = require("serialize") local DEBUG = require("dbg") local TileCacheItem = CacheItem:new{} @@ -10,4 +12,19 @@ function TileCacheItem:onFree() end end +function TileCacheItem:dump(filename) + DEBUG("dumping tile cache to", filename, self.excerpt) + return serial.dump(self.size, self.excerpt, self.pageno, + self.bb.w, self.bb.h, self.bb.pitch, self.bb:getType(), + Blitbuffer.tostring(self.bb), filename) +end + +function TileCacheItem:load(filename) + local w, h, pitch, bb_type, bb_data + self.size, self.excerpt, self.pageno, + w, h, pitch, bb_type, bb_data = serial.load(filename) + self.bb = Blitbuffer.fromstring(w, h, bb_type, bb_data, pitch) + DEBUG("loading tile cache from", filename, self) +end + return TileCacheItem diff --git a/frontend/memutils.lua b/frontend/memutils.lua new file mode 100644 index 000000000..c1bb4fa78 --- /dev/null +++ b/frontend/memutils.lua @@ -0,0 +1,131 @@ +local ffi = require "ffi" +local bit = require "bit" +local band = bit.band +local bor = bit.bor +local rshift = bit.rshift +local lshift = bit.lshift + + + + +ffi.cdef[[ +void * malloc ( size_t size ); +void free ( void * ptr ); +void * realloc ( void * ptr, size_t size ); +]] + +function bzero(dest, nbytes) + ffi.fill(dest, nbytes) + return dest +end + +function bcopy(src, dest, nbytes) + ffi.copy(dest, src, nbytes) +end + +function bcmp(ptr1, ptr2, nbytes) + for i=0,nbytes do + if ptr1[i] ~= ptr2[i] then return -1 end + end + + return 0 +end + + + +function memset(dest, c, len) + ffi.fill(dest, len, c) + return dest +end + +function memcpy(dest, src, nbytes) + ffi.copy(dest, src, nbytes) +end + +function memcmp(ptr1, ptr2, nbytes) + local p1 = ffi.cast("const uint8_t *", ptr1) + local p2 = ffi.cast("const uint8_t *", ptr2) + + for i=0,nbytes do + if p1[i] ~= p2[i] then return -1 end + end + + return 0 +end + +function memchr(ptr, value, num) + local p = ffi.cast("const uint8_t *", ptr) + for i=0,num-1 do + if p[i] == value then return p+i end + end + + return nil +end + +function memmove(dst, src, num) + local srcptr = ffi.cast("const uint8_t*", src) + + -- If equal, just return + if dst == srcptr then return dst end + + + if srcptr < dst then + -- copy from end + for i=num-1,0, -1 do + dst[i] = srcptr[i]; + end + else + -- copy from beginning + for i=0,num-1 do + dst[i] = srcptr[i]; + end + end + return dst +end + +local function memreverse(buff, bufflen) + local i = 0; + local tmp + + while (i < (bufflen)/2) do + tmp = buff[i]; + buff[i] = buff[bufflen-i-1]; + buff[bufflen-i-1] = tmp; + + i = i + 1; + end + return buff +end + +local function getreverse(src, len) + if not len then + if type(src) == "string" then + len = #src + else + return nil, "unknown length" + end + end + + local srcptr = ffi.cast("const uint8_t *", src); + local dst = ffi.new("uint8_t[?]", len) + + for i = 0, len-1 do + dst[i] = srcptr[len-1-i]; + end + + return dst, len +end + +return { + bcmp = bcmp, + bcopy = bcopy, + bzero = bzero, + + memchr = memchr, + memcpy = memcpy, + memcmp = memcmp, + memmove = memmove, + memset = memset, + + memreverse = memreverse, +} diff --git a/frontend/stringzutils.lua b/frontend/stringzutils.lua new file mode 100644 index 000000000..17577bcf9 --- /dev/null +++ b/frontend/stringzutils.lua @@ -0,0 +1,322 @@ +local ffi = require "ffi" +local bit = require "bit" +local band = bit.band +local bor = bit.bor +local rshift = bit.rshift +local lshift = bit.lshift + +--[[ + String Functions + + strlen + strndup + strdup + strcpy + strlcpy + strlcat + + strchr + strcmp + strncmp + strcasecmp + strncasecmp + + strrchr + strstr + + strpbrk + + bin2str +--]] + + + +function strcmp(s1, s2) + local s1ptr = ffi.cast("const uint8_t *", s1); + local s2ptr = ffi.cast("const uint8_t *", s2); + + -- uint8_t + local uc1; + local uc2; + + -- Move s1 and s2 to the first differing characters + -- in each string, or the ends of the strings if they + -- are identical. + while (s1ptr[0] ~= 0 and s1ptr[0] == s2ptr[0]) do + s1ptr = s1ptr + 1 + s2ptr = s2ptr + 1 + end + + -- Compare the characters as unsigned char and + -- return the difference. + uc1 = s1ptr[0]; + uc2 = s2ptr[0]; + + if (uc1 < uc2) then + return -1 + elseif (uc1 > uc2) then + return 1 + end + + return 0 +end + + +function strncmp(str1, str2, num) + local ptr1 = ffi.cast("const uint8_t*", str1) + local ptr2 = ffi.cast("const uint8_t*", str2) + + for i=0,num-1 do + if str1[i] == 0 or str2[i] == 0 then return 0 end + + if ptr1[i] > ptr2[i] then return 1 end + if ptr1[i] < ptr2[i] then return -1 end + end + + return 0 +end + +function strncasecmp(str1, str2, num) + local ptr1 = ffi.cast("const uint8_t*", str1) + local ptr2 = ffi.cast("const uint8_t*", str2) + + for i=0,num-1 do + if str1[i] == 0 or str2[i] == 0 then return 0 end + + if ptr1[i] > ptr2[i] then return 1 end + if ptr1[i] < ptr2[i] then return -1 end + end + + return 0 +end + + +function strcasecmp(str1, str2) + local ptr1 = ffi.cast("const uint8_t*", str1) + local ptr2 = ffi.cast("const uint8_t*", str2) + + local num = math.min(strlen(ptr1), strlen(ptr2)) + for i=0,num-1 do + if str1[i] == 0 or str2[i] == 0 then return 0 end + + if tolower(ptr1[i]) > tolower(ptr2[i]) then return 1 end + if tolower(ptr1[i]) < tolower(ptr2[i]) then return -1 end + end + + return 0 +end + +function strlen(str) + local ptr = ffi.cast("uint8_t *", str); + local idx = 0 + while ptr[idx] ~= 0 do + idx = idx + 1 + end + + return idx +end + +function strndup(str,n) + local len = strlen(str) + local len = math.min(n,len) + + local newstr = ffi.new("char["..(len+1).."]"); + ffi.copy(newstr, str, len) + newstr[len] = 0 + + return newstr +end + +function strdup(str) + -- In the case of a Lua string + -- create a VLA and initialize + if type(str) == "string" then + return ffi.new("uint8_t [?]", #str+1, str) + end + + -- Most dangerous, assuming it's a null terminated + -- string. + local len = strlen(str) + local newstr = ffi.new("char[?]", (len+1)); + local strptr = ffi.cast("const char *", str) + + ffi.copy(newstr, ffi.cast("const char *", str), len) + newstr[len] = 0 + + return newstr +end + +function strcpy(dst, src) + local dstptr = ffi.cast("char *", dst) + local srcptr = ffi.cast("const char *", src) + + -- Do the copying in a loop. + while (srcptr[0] ~= 0) do + dstptr[0] = srcptr[0]; + dstptr = dstptr + 1; + srcptr = srcptr + 1; + end + + -- Return the destination string. + return dst; +end + +function strlcpy(dst, src, size) + local dstptr = ffi.cast("char *", dst) + local srcptr = ffi.cast("const char *", src) + + local len = strlen(src) + local len = math.min(size-1,len) + + ffi.copy(dstptr, srcptr, len) + dstptr[len] = 0 + + return len +end + +function strlcat(dst, src, size) + local dstptr = ffi.cast("char *", dst) + local srcptr = ffi.cast("const char *", src) + + local dstlen = strlen(dstptr); + local dstremaining = size-dstlen-1 + local srclen = strlen(srcptr); + local len = math.min(dstremaining, srclen) + + + for idx=dstlen,dstlen+len do + dstptr[idx] = srcptr[idx-dstlen]; + end + + return dstlen+len +end + + + +function strchr(s, c) + local p = ffi.cast("const char *", s); + + while p[0] ~= c do + if p[0] == 0 then + return nil + end + p = p + 1; + end + + return p +end + +function strrchr(s, c) + local p = ffi.cast("const char *", s); + local offset = strlen(p); + + while offset >= 0 do + if p[offset] == c then + return p+offset + end + offset = offset - 1; + end + + return nil +end + +function strstr(str, target) + + if (target == nil or target[0] == 0) then + return str; + end + + local p1 = ffi.cast("const char *", str); + + while (p1[0] ~= 0) do + + local p1Begin = p1; + local p2 = target; + + while (p1[0]~=0 and p2[0]~=0 and p1[0] == p2[0]) do + p1 = p1 + 1; + p2 = p2 + 1; + end + + if (p2[0] == 0) then + return p1Begin; + end + + p1 = p1Begin + 1; + end + + return nil; +end + + +--[[ + String Helpers +--]] + +-- Given two null terminated strings +-- return how many bytes they have in common +-- this is for prefix matching +function string_same(a, b) + local p1 = ffi.cast("const char *", a); + local p2 = ffi.cast("const char *", b); + + local bytes = 0; + + while (p1[bytes] ~= 0 and p2[bytes] ~= 0 and p1[bytes] == p2[bytes]) do + bytes = bytes+1 + end + + return bytes; +end + +-- Stringify binary data. Output buffer must be twice as big as input, +-- because each byte takes 2 bytes in string representation + +local hex = strdup("0123456789abcdef") + +function bin2str(to, p, len) +--print("bin2str, len: ", len); + local off1, off2; + while (len > 0) do + off1 = rshift(p[0], 4) + + to[0] = hex[off1]; + to = to + 1; + off2 = band(p[0], 0x0f); + to[0] = hex[off2]; + to = to + 1; + p = p + 1; + len = len - 1; + +-- print(off1, off2); + end + to[0] = 0; +end + + +local function bintohex(s) + return (s:gsub('(.)', function(c) + return string.format('%02x', string.byte(c)) + end)) +end + +local function hextobin(s) + return (s:gsub('(%x%x)', function(hex) + return string.char(tonumber(hex, 16)) + end)) +end + +return { + strchr = strchr, + strcmp = strcmp, + strncmp = strncmp, + strncasecmp = strncasecmp, + strcpy = strcpy, + strndup = strndup, + strdup = strdup, + + strlen = strlen, + + bintohex = bintohex, + hextobin = hextobin, +} diff --git a/koreader-base b/koreader-base index acbcbe71d..3b5663851 160000 --- a/koreader-base +++ b/koreader-base @@ -1 +1 @@ -Subproject commit acbcbe71d2c2086afe1b044edebe98833980d0bf +Subproject commit 3b566385128b722594cde00fa599f035621168fa