From 9299b9a5e8bc0c5f478b5ff74e7d714472d18626 Mon Sep 17 00:00:00 2001 From: chrox Date: Mon, 2 Mar 2015 17:15:26 +0800 Subject: [PATCH 1/6] add async http client it uses non-blocking turbo I/O looper to process http request so that multiple http request can be handled simultaneously and http request won't block user input, and most importantly, in Lua's way. --- base | 2 +- frontend/httpclient.lua | 52 +++++++++++ frontend/ui/uimanager.lua | 163 +++++++++++++++++++++------------- spec/unit/httpclient_spec.lua | 36 ++++++++ 4 files changed, 192 insertions(+), 61 deletions(-) create mode 100644 frontend/httpclient.lua create mode 100644 spec/unit/httpclient_spec.lua diff --git a/base b/base index 9aa76dc81..8d31203c7 160000 --- a/base +++ b/base @@ -1 +1 @@ -Subproject commit 9aa76dc818c7d874e9dd09a903722915be63af09 +Subproject commit 8d31203c7f3667ba75744ec50c8201f01ebc5286 diff --git a/frontend/httpclient.lua b/frontend/httpclient.lua new file mode 100644 index 000000000..66043442a --- /dev/null +++ b/frontend/httpclient.lua @@ -0,0 +1,52 @@ +local UIManager = require("ui/uimanager") +local DEBUG = require("dbg") + +local HTTPClient = { + headers = {}, + input_timeouts = 0, + INPUT_TIMEOUT = 100*1000, +} + +function HTTPClient:new() + local o = {} + setmetatable(o, self) + self.__index = self + return o +end + +function HTTPClient:addHeader(header, value) + self.headers[header] = value +end + +function HTTPClient:removeHeader(header) + self.headers[header] = nil +end + +function HTTPClient:request(request, response_callback, error_callback) + request.on_headers = function(headers) + for header, value in pairs(self.headers) do + headers[header] = value + end + end + request.connect_timeout = 10 + request.request_timeout = 20 + UIManager:initLooper() + UIManager:handleTask(function() + -- avoid endless waiting for input + UIManager.INPUT_TIMEOUT = self.INPUT_TIMEOUT + self.input_timeouts = self.input_timeouts + 1 + local turbo = require("turbo") + local res = coroutine.yield( + turbo.async.HTTPClient():fetch(request.url, request)) + -- reset INPUT_TIMEOUT to nil when all HTTP requests are fullfilled. + self.input_timeouts = self.input_timeouts - 1 + UIManager.INPUT_TIMEOUT = self.input_timeouts > 0 and self.INPUT_TIMEOUT or nil + if res.error and error_callback then + error_callback(res) + elseif response_callback then + response_callback(res) + end + end) +end + +return HTTPClient diff --git a/frontend/ui/uimanager.lua b/frontend/ui/uimanager.lua index 0ccac2850..44c42845f 100644 --- a/frontend/ui/uimanager.lua +++ b/frontend/ui/uimanager.lua @@ -6,6 +6,7 @@ local Geom = require("ui/geometry") local util = require("ffi/util") local DEBUG = require("dbg") local _ = require("gettext") +local ffi = require("ffi") -- there is only one instance of this local UIManager = { @@ -246,6 +247,7 @@ end function UIManager:quit() DEBUG("quit uimanager") self._running = false + self._run_forever = nil for i = #self._window_stack, 1, -1 do table.remove(self._window_stack, i) end @@ -256,6 +258,10 @@ function UIManager:quit() self._zeromqs[i]:stop() table.remove(self._zeromqs, i) end + if self.looper then + self.looper:close() + self.looper = nil + end end -- transmit an event to registered widgets @@ -430,73 +436,110 @@ function UIManager:_repaint() self.refresh_counted = false end --- this is the main loop of the UI controller --- it is intended to manage input events and delegate --- them to dialogs -function UIManager:run() - self._running = true - while self._running do - local wait_until, now - -- run this in a loop, so that paints can trigger events - -- that will be honored when calculating the time to wait - -- for input events: - repeat - wait_until, now = self:_checkTasks() - - --DEBUG("---------------------------------------------------") - --DEBUG("exec stack", self._execution_stack) - --DEBUG("window stack", self._window_stack) - --DEBUG("dirty stack", self._dirty) - --DEBUG("---------------------------------------------------") - - -- stop when we have no window to show - if #self._window_stack == 0 then - DEBUG("no dialog left to show") - self:quit() - return nil - end +function UIManager:handleInput() + local wait_until, now + -- run this in a loop, so that paints can trigger events + -- that will be honored when calculating the time to wait + -- for input events: + repeat + wait_until, now = self:_checkTasks() + + --DEBUG("---------------------------------------------------") + --DEBUG("exec stack", self._execution_stack) + --DEBUG("window stack", self._window_stack) + --DEBUG("dirty stack", self._dirty) + --DEBUG("---------------------------------------------------") + + -- stop when we have no window to show + if #self._window_stack == 0 and not self._run_forever then + DEBUG("no dialog left to show") + self:quit() + return nil + end - self:_repaint() - until not self._execution_stack_dirty - - -- wait for next event - -- note that we will skip that if we have tasks that are ready to run - local input_event = nil - if not wait_until then - if #self._zeromqs > 0 then - -- pending message queue, wait 100ms for input - input_event = Input:waitEvent(1000*100) - if not input_event or input_event.handler == "onInputError" then - for _, zeromq in ipairs(self._zeromqs) do - input_event = zeromq:waitEvent() - if input_event then break end - end + self:_repaint() + until not self._execution_stack_dirty + + -- wait for next event + -- note that we will skip that if we have tasks that are ready to run + local input_event = nil + if not wait_until then + if #self._zeromqs > 0 then + -- pending message queue, wait 100ms for input + input_event = Input:waitEvent(1000*100) + if not input_event or input_event.handler == "onInputError" then + for _, zeromq in ipairs(self._zeromqs) do + input_event = zeromq:waitEvent() + if input_event then break end end - else - -- no pending task, wait without timeout - input_event = Input:waitEvent() - end - elseif wait_until[1] > now[1] - or wait_until[1] == now[1] and wait_until[2] > now[2] then - local wait_for = { s = wait_until[1] - now[1], us = wait_until[2] - now[2] } - if wait_for.us < 0 then - wait_for.s = wait_for.s - 1 - wait_for.us = 1000000 + wait_for.us end - -- wait until next task is pending - input_event = Input:waitEvent(wait_for.us, wait_for.s) + else + -- no pending task, wait without timeout + input_event = Input:waitEvent(self.INPUT_TIMEOUT) end + elseif wait_until[1] > now[1] + or wait_until[1] == now[1] and wait_until[2] > now[2] then + local wait_for = { s = wait_until[1] - now[1], us = wait_until[2] - now[2] } + if wait_for.us < 0 then + wait_for.s = wait_for.s - 1 + wait_for.us = 1000000 + wait_for.us + end + -- wait until next task is pending + input_event = Input:waitEvent(wait_for.us, wait_for.s) + end - -- delegate input_event to handler - if input_event then - local handler = self.event_handlers[input_event] - if handler then - handler(input_event) - else - self.event_handlers["__default__"](input_event) - end + -- delegate input_event to handler + if input_event then + local handler = self.event_handlers[input_event] + if handler then + handler(input_event) + else + self.event_handlers["__default__"](input_event) end end + + -- handle next input + self:handleTask(function() self:handleInput() end) +end + +-- handle task(callback function) in Turbo I/O looper +-- or run task immediately if looper is not available +function UIManager:handleTask(task) + if self.looper then + DEBUG("handle task in turbo I/O looper") + self.looper:add_callback(task) + else + DEBUG("run task") + task() + end +end + +function UIManager:initLooper() + if not self.looper then + TURBO_SSL = true + local turbo = require("turbo") + self.looper = turbo.ioloop.instance() + end +end + +-- this is the main loop of the UI controller +-- it is intended to manage input events and delegate +-- them to dialogs +function UIManager:run() + self._running = true + if ffi.os == "Windows" then + self:handleInput() + else + self:initLooper() + self:handleTask(function() self:handleInput() end) + self.looper:start() + end +end + +-- run uimanager forever for testing purpose +function UIManager:runForever() + self._run_forever = true + self:run() end UIManager:init() diff --git a/spec/unit/httpclient_spec.lua b/spec/unit/httpclient_spec.lua new file mode 100644 index 000000000..ff02c7a89 --- /dev/null +++ b/spec/unit/httpclient_spec.lua @@ -0,0 +1,36 @@ +require("commonrequire") +local UIManager = require("ui/uimanager") +local HTTPClient = require("httpclient") +local DEBUG = require("dbg") +DEBUG:turnOn() + +describe("HTTP client module", function() + local requests = 0 + local function response_callback(res) + requests = requests - 1 + if requests == 0 then UIManager:quit() end + assert(res.body) + end + local function error_callback(res) + requests = requests - 1 + if requests == 0 then UIManager:quit() end + assert(false, "error occurs") + end + local async_client = HTTPClient:new() + it("should get response from async GET request", function() + UIManager:quit() + local urls = { + "http://www.example.com", + "http://www.example.org", + "https://www.example.com", + "https://www.example.org", + } + requests = #urls + for _, url in ipairs(urls) do + async_client:request({ + url = url, + }, response_callback, error_callback) + end + UIManager:runForever() + end) +end) From e74d1fa55708b2c2841238c8692f0d3128f89f1b Mon Sep 17 00:00:00 2001 From: chrox Date: Mon, 2 Mar 2015 17:24:35 +0800 Subject: [PATCH 2/6] turn off debug in unit test --- spec/unit/httpclient_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/httpclient_spec.lua b/spec/unit/httpclient_spec.lua index ff02c7a89..bffb1091c 100644 --- a/spec/unit/httpclient_spec.lua +++ b/spec/unit/httpclient_spec.lua @@ -2,7 +2,7 @@ require("commonrequire") local UIManager = require("ui/uimanager") local HTTPClient = require("httpclient") local DEBUG = require("dbg") -DEBUG:turnOn() +--DEBUG:turnOn() describe("HTTP client module", function() local requests = 0 From 4053b5adac83bf35d820653d8889a40e1faa5fcd Mon Sep 17 00:00:00 2001 From: chrox Date: Tue, 3 Mar 2015 18:00:38 +0800 Subject: [PATCH 3/6] fix unit test with latest busted --- .travis.yml | 5 ++--- Makefile | 4 ++-- base | 2 +- spec/unit/document_spec.lua | 2 ++ 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 496f62688..bdacbd4b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,14 +14,13 @@ before_install: install: - sudo apt-get install libsdl1.2-dev luarocks nasm - - git clone https://github.com/Olivine-Labs/busted/ - - cd busted && git checkout v1.10.0 && sudo luarocks make && cd .. - - sudo luarocks install luacov + - sudo luarocks install busted luacov - sudo luarocks install luacov-coveralls --server=http://rocks.moonscript.org/dev script: - make fetchthirdparty all - sudo cp base/build/*/luajit /usr/bin/ + - sudo ln -sf /usr/bin/luajit /usr/bin/lua - make testfront after_success: diff --git a/Makefile b/Makefile index 55f7f669b..8ad990ab8 100644 --- a/Makefile +++ b/Makefile @@ -93,14 +93,14 @@ $(INSTALL_DIR)/koreader/.luacov: ln -sf ../../.luacov $(INSTALL_DIR)/koreader testfront: $(INSTALL_DIR)/koreader/.busted - cd $(INSTALL_DIR)/koreader && busted -l ./luajit + cd $(INSTALL_DIR)/koreader && busted test: $(MAKE) -C $(KOR_BASE) test $(MAKE) testfront coverage: $(INSTALL_DIR)/koreader/.luacov - cd $(INSTALL_DIR)/koreader && busted -c -l ./luajit --exclude-tags=nocov + cd $(INSTALL_DIR)/koreader && busted -c --exclude-tags=nocov # coverage report summary cd $(INSTALL_DIR)/koreader && tail -n \ +$$(($$(grep -nm1 Summary luacov.report.out|cut -d: -f1)-1)) \ diff --git a/base b/base index 8d31203c7..702583005 160000 --- a/base +++ b/base @@ -1 +1 @@ -Subproject commit 8d31203c7f3667ba75744ec50c8201f01ebc5286 +Subproject commit 7025830053c47496db413802375236300eaf4e6e diff --git a/spec/unit/document_spec.lua b/spec/unit/document_spec.lua index 4338b7a91..69bdb9356 100644 --- a/spec/unit/document_spec.lua +++ b/spec/unit/document_spec.lua @@ -3,6 +3,7 @@ local DocumentRegistry = require("document/documentregistry") describe("PDF document module", function() local sample_pdf = "spec/front/unit/data/tall.pdf" + local doc it("should open document", function() doc = DocumentRegistry:openDocument(sample_pdf) assert.truthy(doc) @@ -38,6 +39,7 @@ end) describe("EPUB document module", function() local sample_epub = "spec/front/unit/data/leaves.epub" + local doc it("should open document", function() doc = DocumentRegistry:openDocument(sample_epub) assert.truthy(doc) From f62028c3ca1e4726efe3607514adcdc86eab23ef Mon Sep 17 00:00:00 2001 From: chrox Date: Tue, 3 Mar 2015 18:01:08 +0800 Subject: [PATCH 4/6] use svg badges --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a5b7151d2..8fbf2879b 100644 --- a/README.md +++ b/README.md @@ -253,12 +253,12 @@ http://ccache.samba.org [base-readme]:https://github.com/koreader/koreader-base/blob/master/README.md [nb-script]:https://github.com/koreader/koreader-misc/blob/master/koreader-nightlybuild/koreader-nightlybuild.sh -[travis-badge]:https://travis-ci.org/koreader/koreader.png?branch=master +[travis-badge]:https://travis-ci.org/koreader/koreader.svg?branch=master [travis-link]:https://travis-ci.org/koreader/koreader [travis-conf]:https://github.com/koreader/koreader-base/blob/master/.travis.yml [linux-vm]:http://www.howtogeek.com/howto/11287/how-to-run-ubuntu-in-windows-7-with-vmware-player/ [l10n-readme]:https://github.com/koreader/koreader/blob/master/l10n/README.md [koreader-transifex]:https://www.transifex.com/projects/p/koreader/ -[coverage-badge]:https://coveralls.io/repos/koreader/koreader/badge.png +[coverage-badge]:https://coveralls.io/repos/koreader/koreader/badge.svg [coverage-link]:https://coveralls.io/r/koreader/koreader [licence-badge]:http://img.shields.io/badge/licence-AGPL-brightgreen.svg From 1e7e4017154b2aa9605638b961b56d9c0137753c Mon Sep 17 00:00:00 2001 From: chrox Date: Tue, 3 Mar 2015 18:09:44 +0800 Subject: [PATCH 5/6] fix luarocks install --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bdacbd4b2..837e79b58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,8 @@ before_install: install: - sudo apt-get install libsdl1.2-dev luarocks nasm - - sudo luarocks install busted luacov + - sudo luarocks install busted + - sudo luarocks install luacov - sudo luarocks install luacov-coveralls --server=http://rocks.moonscript.org/dev script: From b4574a735966ba8b3c82a234c11803e47be0839f Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Tue, 3 Mar 2015 21:14:35 +0800 Subject: [PATCH 6/6] fix crash on kindle --- reader.lua | 5 ----- 1 file changed, 5 deletions(-) diff --git a/reader.lua b/reader.lua index 7864b68fd..fa3441d89 100755 --- a/reader.lua +++ b/reader.lua @@ -18,11 +18,6 @@ ffi.cdef[[ ]] if ffi.os == "Windows" then ffi.C._putenv("PATH=libs;common;") -else - ffi.C.putenv("LD_LIBRARY_PATH=" - .. util.realpath("libs") .. ":" - .. util.realpath("common") ..":" - .. ffi.string(ffi.C.getenv("LD_LIBRARY_PATH"))) end local DocSettings = require("docsettings")