diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index de62e197..baa235af 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -409,6 +409,7 @@ add_library( spectro_source.cc sql_commands.cc sql_util.cc + sqlitepp.cc state-extension-functions.cc styling.cc text_format.cc diff --git a/src/Makefile.am b/src/Makefile.am index 7b2899a6..ecbc26d4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -437,6 +437,7 @@ libdiag_a_SOURCES = \ shlex.cc \ spectro_impls.cc \ spectro_source.cc \ + sqlitepp.cc \ sqlite-extension-func.cc \ statusview_curses.cc \ string-extension-functions.cc \ diff --git a/src/archive_manager.cc b/src/archive_manager.cc index 7d2e39ca..bcad728d 100644 --- a/src/archive_manager.cc +++ b/src/archive_manager.cc @@ -54,50 +54,6 @@ namespace fs = ghc::filesystem; namespace archive_manager { -class archive_lock { -public: - class guard { - public: - explicit guard(archive_lock& arc_lock) : g_lock(arc_lock) - { - this->g_lock.lock(); - }; - - ~guard() - { - this->g_lock.unlock(); - }; - - private: - archive_lock& g_lock; - }; - - void lock() const - { - lockf(this->lh_fd, F_LOCK, 0); - }; - - void unlock() const - { - lockf(this->lh_fd, F_ULOCK, 0); - }; - - explicit archive_lock(const fs::path& archive_path) - { - auto lock_path = archive_path; - - lock_path += ".lck"; - auto open_res = lnav::filesystem::create_file(lock_path, O_RDWR, 0600); - if (open_res.isErr()) { - throw std::runtime_error(open_res.unwrapErr()); - } - this->lh_fd = open_res.unwrap(); - this->lh_fd.close_on_exec(); - }; - - auto_fd lh_fd; -}; - #if HAVE_ARCHIVE_H /** * Enables a subset of the supported archive formats to speed up detection, @@ -275,8 +231,8 @@ extract(const std::string& filename, const extract_cb& cb) ec.message())); } - auto arc_lock = archive_lock(tmp_path); - auto lock_guard = archive_lock::guard(arc_lock); + auto arc_lock = lnav::filesystem::file_lock(tmp_path); + auto lock_guard = lnav::filesystem::file_lock::guard(arc_lock); auto done_path = tmp_path; done_path += ".done"; diff --git a/src/base/fs_util.hh b/src/base/fs_util.hh index 9800f500..709f1830 100644 --- a/src/base/fs_util.hh +++ b/src/base/fs_util.hh @@ -78,6 +78,41 @@ Result write_file(const ghc::filesystem::path& path, std::string build_path(const std::vector& paths); +class file_lock { +public: + class guard { + public: + explicit guard(file_lock& arc_lock) : g_lock(arc_lock) + { + this->g_lock.lock(); + }; + + ~guard() { this->g_lock.unlock(); }; + + private: + file_lock& g_lock; + }; + + void lock() const { lockf(this->lh_fd, F_LOCK, 0); } + + void unlock() const { lockf(this->lh_fd, F_ULOCK, 0); } + + explicit file_lock(const ghc::filesystem::path& archive_path) + { + auto lock_path = archive_path; + + lock_path += ".lck"; + auto open_res = lnav::filesystem::create_file( + lock_path, O_RDWR | O_CLOEXEC, 0600); + if (open_res.isErr()) { + throw std::runtime_error(open_res.unwrapErr()); + } + this->lh_fd = open_res.unwrap(); + } + + auto_fd lh_fd; +}; + } // namespace filesystem } // namespace lnav diff --git a/src/base/snippet_highlighters.cc b/src/base/snippet_highlighters.cc index 0222d09c..1fef530f 100644 --- a/src/base/snippet_highlighters.cc +++ b/src/base/snippet_highlighters.cc @@ -39,6 +39,10 @@ namespace snippets { static bool is_bracket(const std::string& str, int index, bool is_lit) { + if (index == 0) { + return true; + } + if (is_lit && str[index - 1] == '\\') { return true; } diff --git a/src/line_buffer.cc b/src/line_buffer.cc index 05fb4143..6b453569 100644 --- a/src/line_buffer.cc +++ b/src/line_buffer.cc @@ -49,13 +49,18 @@ #endif #include "base/auto_pid.hh" +#include "base/fs_util.hh" #include "base/injector.bind.hh" #include "base/injector.hh" #include "base/is_utf8.hh" #include "base/isc.hh" #include "base/math_util.hh" +#include "base/paths.hh" #include "fmtlib/fmt/format.h" #include "line_buffer.hh" +#include "lnav_util.hh" + +using namespace std::chrono_literals; static const ssize_t INITIAL_REQUEST_SIZE = 16 * 1024; static const ssize_t DEFAULT_INCREMENT = 128 * 1024; @@ -438,7 +443,7 @@ line_buffer::ensure_available(file_off_t start, ssize_t max_length) if ((this->lb_file_size != (ssize_t) -1) && (start + this->lb_buffer.capacity() > this->lb_file_size)) { - require(start < this->lb_file_size); + require(start <= this->lb_file_size); /* * If the start is near the end of the file, move the offset back a * bit so we can get more of the file in the cache. @@ -496,7 +501,7 @@ line_buffer::load_next_buffer() // log_debug("BEGIN preload read"); /* ... read in the new data. */ - if (*gi) { + if (!this->lb_cached_fd && *gi) { if (this->lb_file_size != (ssize_t) -1 && this->in_range(start) && this->in_range(this->lb_file_size - 1)) { @@ -523,7 +528,7 @@ line_buffer::load_next_buffer() } } #ifdef HAVE_BZLIB_H - else if (this->lb_bz_file) + else if (!this->lb_cached_fd && this->lb_bz_file) { if (this->lb_file_size != (ssize_t) -1 && (((ssize_t) start >= this->lb_file_size) @@ -547,7 +552,7 @@ line_buffer::load_next_buffer() close(bzfd); throw error(errno); } - if ((bz_file = BZ2_bzdopen(bzfd, "r")) == NULL) { + if ((bz_file = BZ2_bzdopen(bzfd, "r")) == nullptr) { close(bzfd); if (errno == 0) { throw std::bad_alloc(); @@ -571,8 +576,10 @@ line_buffer::load_next_buffer() this->lb_compressed_offset = lseek(bzfd, 0, SEEK_SET); BZ2_bzclose(bz_file); - if (rc != -1 && (rc < (this->lb_alt_buffer.value().available())) && - (start + this->lb_alt_buffer.value().size() + rc > this->lb_file_size)) { + if (rc != -1 && (rc < (this->lb_alt_buffer.value().available())) + && (start + this->lb_alt_buffer.value().size() + rc + > this->lb_file_size)) + { this->lb_file_size = (start + this->lb_alt_buffer.value().size() + rc); } @@ -581,7 +588,8 @@ line_buffer::load_next_buffer() #endif else { - rc = pread(this->lb_fd, + rc = pread(this->lb_cached_fd ? this->lb_cached_fd.value().get() + : this->lb_fd.get(), this->lb_alt_buffer.value().end(), this->lb_alt_buffer.value().available(), start + this->lb_alt_buffer.value().size()); @@ -763,7 +771,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) safe::WriteAccess gi(this->lb_gz_file); /* ... read in the new data. */ - if (*gi) { + if (!this->lb_cached_fd && *gi) { // log_debug("old decomp start"); if (this->lb_file_size != (ssize_t) -1 && this->in_range(start) && this->in_range(this->lb_file_size - 1)) @@ -771,7 +779,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) rc = 0; } else { this->lb_stats.s_decompressions += 1; - if (this->lb_last_line_offset > 0) { + if (false && this->lb_last_line_offset > 0) { this->lb_stats.s_hist[(this->lb_file_offset * 10) / this->lb_last_line_offset] += 1; @@ -793,7 +801,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) #endif } #ifdef HAVE_BZLIB_H - else if (this->lb_bz_file) + else if (!this->lb_cached_fd && this->lb_bz_file) { if (this->lb_file_size != (ssize_t) -1 && (((ssize_t) start >= this->lb_file_size) @@ -852,7 +860,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) else if (this->lb_seekable) { this->lb_stats.s_preads += 1; - if (this->lb_last_line_offset > 0) { + if (false && this->lb_last_line_offset > 0) { this->lb_stats.s_hist[(this->lb_file_offset * 10) / this->lb_last_line_offset] += 1; @@ -862,7 +870,8 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length) this->lb_fd.get(), this->lb_file_offset + this->lb_buffer.size()); #endif - rc = pread(this->lb_fd, + rc = pread(this->lb_cached_fd ? this->lb_cached_fd.value().get() + : this->lb_fd.get(), this->lb_buffer.end(), this->lb_buffer.available(), this->lb_file_offset + this->lb_buffer.size()); @@ -993,7 +1002,7 @@ line_buffer::load_next_line(file_range prev_line) } while (!done) { auto old_retval_size = retval.li_file_range.fr_size; - char *line_start, *lf; + const char *line_start, *lf; /* Find the data in the cache and */ line_start = this->get_range(offset, retval.li_file_range.fr_size); @@ -1059,7 +1068,8 @@ line_buffer::load_next_line(file_range prev_line) && (!this->is_pipe() || request_size > DEFAULT_INCREMENT))) { if ((lf != nullptr) - && ((size_t) (lf - line_start) >= MAX_LINE_BUFFER_SIZE - 1)) { + && ((size_t) (lf - line_start) >= MAX_LINE_BUFFER_SIZE - 1)) + { lf = nullptr; } if (lf != nullptr) { @@ -1134,11 +1144,12 @@ Result line_buffer::read_range(const file_range fr) { shared_buffer_ref retval; - char* line_start; + const char* line_start; file_ssize_t avail; if (this->lb_last_line_offset != -1 - && fr.fr_offset > this->lb_last_line_offset) { + && fr.fr_offset > this->lb_last_line_offset) + { /* * Don't return anything past the last known line. The caller needs * to try reading at the offset of the last line again. @@ -1232,3 +1243,126 @@ line_buffer::quiesce() this->lb_loader_future.wait(); } } + +static ghc::filesystem::path +line_buffer_cache_path() +{ + return lnav::paths::workdir() / "buffer-cache"; +} + +void +line_buffer::enable_cache() +{ + if (!this->lb_compressed || this->lb_cached_fd) { + log_info("%d: skipping cache request (compressed=%d already-cached=%d)", + this->lb_fd.get(), + this->lb_compressed, + (bool) this->lb_cached_fd); + return; + } + + struct stat st; + + if (fstat(this->lb_fd, &st) == -1) { + log_error("failed to fstat(%d) - %d", this->lb_fd.get(), errno); + return; + } + + auto cached_base_name = hasher() + .update(st.st_dev) + .update(st.st_ino) + .update(st.st_size) + .to_string(); + auto cache_dir = line_buffer_cache_path() / cached_base_name.substr(0, 2); + + ghc::filesystem::create_directories(cache_dir); + + auto cached_file_name = fmt::format(FMT_STRING("{}.bin"), cached_base_name); + auto cached_file_path = cache_dir / cached_file_name; + auto cached_done_path + = cache_dir / fmt::format(FMT_STRING("{}.done"), cached_base_name); + + log_info( + "%d:cache file path: %s", this->lb_fd.get(), cached_file_path.c_str()); + + auto fl = lnav::filesystem::file_lock(cached_file_path); + auto guard = lnav::filesystem::file_lock::guard(fl); + + if (ghc::filesystem::exists(cached_done_path)) { + log_info("%d:using existing cache file"); + auto open_res = lnav::filesystem::open_file(cached_file_path, O_RDWR); + if (open_res.isOk()) { + this->lb_cached_fd = open_res.unwrap(); + return; + } + ghc::filesystem::remove(cached_done_path); + } + + auto create_res = lnav::filesystem::create_file( + cached_file_path, O_RDWR | O_TRUNC, 0600); + if (create_res.isErr()) { + log_error("failed to create cache file: %s -- %s", + cached_file_path.c_str(), + create_res.unwrapErr().c_str()); + return; + } + + auto write_fd = create_res.unwrap(); + auto done = false; + + static const ssize_t FILL_LENGTH = 1024 * 1024; + auto off = file_off_t{0}; + while (!done) { + log_debug("%d: caching file content at %d", this->lb_fd.get(), off); + if (!this->fill_range(off, FILL_LENGTH)) { + log_debug("%d: caching finished", this->lb_fd.get()); + done = true; + } else { + file_ssize_t avail; + + const auto* data = this->get_range(off, avail); + auto rc = write(write_fd, data, avail); + if (rc != avail) { + log_error("%d: short write!", this->lb_fd.get()); + return; + } + + off += avail; + } + } + + lnav::filesystem::create_file(cached_done_path, O_WRONLY, 0600); + + this->lb_cached_fd = std::move(write_fd); +} + +void +line_buffer::cleanup_cache() +{ + (void) std::async(std::launch::async, []() { + auto now = std::chrono::system_clock::now(); + auto cache_path = line_buffer_cache_path(); + std::vector to_remove; + + for (const auto& cache_subdir : + ghc::filesystem::directory_iterator(cache_path)) + { + for (const auto& entry : + ghc::filesystem::directory_iterator(cache_subdir)) + { + auto mtime = ghc::filesystem::last_write_time(entry.path()); + auto exp_time = mtime + 1h; + if (now < exp_time) { + continue; + } + + to_remove.emplace_back(entry.path()); + } + } + + for (auto& entry : to_remove) { + log_debug("removing compressed file cache: %s", entry.c_str()); + ghc::filesystem::remove_all(entry); + } + }); +} diff --git a/src/line_buffer.hh b/src/line_buffer.hh index bd06e17a..2ec63035 100644 --- a/src/line_buffer.hh +++ b/src/line_buffer.hh @@ -92,15 +92,9 @@ public: public: gz_indexed(); gz_indexed(gz_indexed&& other) = default; - ~gz_indexed() - { - this->close(); - } + ~gz_indexed() { this->close(); } - inline operator bool() const - { - return this->gz_fd != -1; - } + inline operator bool() const { return this->gz_fd != -1; } uLong get_source_offset() const { @@ -150,38 +144,23 @@ public: void set_fd(auto_fd& fd); /** @return The file descriptor that data should be pulled from. */ - int get_fd() const - { - return this->lb_fd; - } + int get_fd() const { return this->lb_fd; } - time_t get_file_time() const - { - return this->lb_file_time; - } + time_t get_file_time() const { return this->lb_file_time; } /** * @return The size of the file or the amount of data pulled from a pipe. */ - file_ssize_t get_file_size() const - { - return this->lb_file_size; - } + file_ssize_t get_file_size() const { return this->lb_file_size; } - bool is_pipe() const - { - return !this->lb_seekable; - } + bool is_pipe() const { return !this->lb_seekable; } bool is_pipe_closed() const { return !this->lb_seekable && (this->lb_file_size != -1); } - bool is_compressed() const - { - return this->lb_compressed; - } + bool is_compressed() const { return this->lb_compressed; } file_off_t get_read_offset(file_off_t off) const { @@ -259,15 +238,13 @@ public: std::array s_hist{}; }; - struct stats consume_stats() - { - return std::exchange(this->lb_stats, {}); - } + struct stats consume_stats() { return std::exchange(this->lb_stats, {}); } - size_t get_buffer_size() const - { - return this->lb_buffer.size(); - } + size_t get_buffer_size() const { return this->lb_buffer.size(); } + + void enable_cache(); + + static void cleanup_cache(); private: /** @@ -316,15 +293,14 @@ private: * @return A pointer to the start of the cached data in the internal * buffer. */ - char* get_range(file_off_t start, file_ssize_t& avail_out) + const char* get_range(file_off_t start, file_ssize_t& avail_out) const { auto buffer_offset = start - this->lb_file_offset; - char* retval; require(buffer_offset >= 0); require(this->lb_buffer.size() >= buffer_offset); - retval = this->lb_buffer.at(buffer_offset); + const auto* retval = this->lb_buffer.at(buffer_offset); avail_out = this->lb_buffer.size() - buffer_offset; return retval; @@ -367,6 +343,8 @@ private: std::vector lb_line_starts; std::vector lb_line_is_utf; stats lb_stats; + + nonstd::optional lb_cached_fd; }; #endif diff --git a/src/lnav.cc b/src/lnav.cc index e384c57c..d99be33b 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -1167,8 +1167,11 @@ looper() rlc->set_display_next_action(rl_display_next); rlc->set_blur_action(rl_blur); rlc->set_completion_request_action(rl_completion_request); - rlc->set_alt_value(HELP_MSG_2( - e, E, "to move forward/backward through error messages")); + rlc->set_alt_value( + HELP_MSG_2(e, + E, + "to move forward/backward through " ANSI_COLOR( + COLOR_RED) "error" ANSI_NORM " messages")); (void) curs_set(0); @@ -1775,6 +1778,7 @@ UPDATE lnav_views_echo } if (!ran_cleanup) { + line_buffer::cleanup_cache(); archive_manager::cleanup_cache(); tailer::cleanup_cache(); ran_cleanup = true; @@ -2949,6 +2953,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' execute_init_commands(lnav_data.ld_exec_context, cmd_results); archive_manager::cleanup_cache(); tailer::cleanup_cache(); + line_buffer::cleanup_cache(); wait_for_pipers(); isc::to() .send_and_wait( diff --git a/src/log_format.cc b/src/log_format.cc index 0c62dba4..a3680378 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -2166,10 +2166,12 @@ external_log_format::build(std::vector& errors) max_name_width = std::max(max_name_width, pat.p_name.size()); } for (const auto& line_frag : sample_lines) { - auto src_line = line_frag.to_string(); - if (!endswith(src_line, "\n")) { + auto src_line = attr_line_t(line_frag.to_string()); + if (!line_frag.endswith("\n")) { src_line.append("\n"); } + src_line.with_attr_for_all( + VC_ROLE.value(role_t::VCR_QUOTED_CODE)); notes.append(" ").append(src_line); for (auto& part_pair : partial_indexes) { if (part_pair.first >= 0 diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index a2c20dff..9a3ece71 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -394,6 +394,16 @@ vt_open(sqlite3_vtab* p_svt, sqlite3_vtab_cursor** pp_cursor) p_cur->log_cursor.lc_end_line = vis_line_t(p_vt->lss->text_line_count()); p_cur->log_cursor.lc_sub_index = 0; + for (auto& ld : *p_vt->lss) { + auto *lf = ld->get_file_ptr(); + + if (lf == nullptr) { + continue; + } + + lf->enable_cache(); + } + return SQLITE_OK; } diff --git a/src/logfile.hh b/src/logfile.hh index 5704b30d..c92793a5 100644 --- a/src/logfile.hh +++ b/src/logfile.hh @@ -373,6 +373,8 @@ public: void quiesce() { this->lf_line_buffer.quiesce(); } + void enable_cache() { this->lf_line_buffer.enable_cache(); } + void dump_stats(); protected: diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 54561328..aabb5fde 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -212,13 +212,7 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, sbr.share(this->lss_share_manager, (char*) this->lss_token_value.c_str(), this->lss_token_value.size()); - if (this->lss_token_line->is_continued()) { - this->lss_token_attrs.emplace_back( - line_range{0, (int) this->lss_token_value.length()}, - SA_BODY.value()); - } else { - format->annotate(line, this->lss_token_attrs, this->lss_token_values); - } + format->annotate(line, this->lss_token_attrs, this->lss_token_values); if (this->lss_token_line->get_sub_offset() != 0) { this->lss_token_attrs.clear(); } diff --git a/src/shared_buffer.cc b/src/shared_buffer.cc index fcdbc1ae..7e23a657 100644 --- a/src/shared_buffer.cc +++ b/src/shared_buffer.cc @@ -42,7 +42,7 @@ static const bool DEBUG_TRACE = false; void -shared_buffer_ref::share(shared_buffer& sb, char* data, size_t len) +shared_buffer_ref::share(shared_buffer& sb, const char* data, size_t len) { #ifdef HAVE_EXECINFO_H if (DEBUG_TRACE) { @@ -77,7 +77,8 @@ shared_buffer_ref::subset(shared_buffer_ref& other, off_t offset, size_t len) return false; } - memcpy(this->sb_data, &other.sb_data[offset], len); + memcpy( + const_cast(this->sb_data), &other.sb_data[offset], len); } else { this->sb_owner->add_ref(*this); this->sb_data = &other.sb_data[offset]; @@ -134,7 +135,7 @@ shared_buffer_ref::disown() { if (this->sb_owner == nullptr) { if (this->sb_data != nullptr) { - free(this->sb_data); + free(const_cast(this->sb_data)); } } else { this->sb_owner->sb_refs.erase(find(this->sb_owner->sb_refs.begin(), @@ -158,7 +159,8 @@ shared_buffer_ref::copy_ref(const shared_buffer_ref& other) } else { this->sb_owner = nullptr; this->sb_data = (char*) malloc(other.sb_length); - memcpy(this->sb_data, other.sb_data, other.sb_length); + memcpy( + const_cast(this->sb_data), other.sb_data, other.sb_length); this->sb_length = other.sb_length; } } diff --git a/src/shared_buffer.hh b/src/shared_buffer.hh index ace2d20f..c71bab2d 100644 --- a/src/shared_buffer.hh +++ b/src/shared_buffer.hh @@ -111,7 +111,7 @@ public: char* get_writable_data() { if (this->take_ownership()) { - return this->sb_data; + return const_cast(this->sb_data); } return nullptr; @@ -136,12 +136,12 @@ public: }; } - using narrow_result = std::pair; + using narrow_result = std::pair; narrow_result narrow(size_t new_data, size_t new_length); void widen(narrow_result old_data_length); - void share(shared_buffer& sb, char* data, size_t len); + void share(shared_buffer& sb, const char* data, size_t len); bool subset(shared_buffer_ref& other, off_t offset, size_t len); @@ -154,7 +154,7 @@ private: auto_mem sb_backtrace; shared_buffer* sb_owner; - char* sb_data; + const char* sb_data; size_t sb_length; }; diff --git a/src/sql_util.cc b/src/sql_util.cc index ac40da9f..344b2965 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -46,6 +46,7 @@ #include "base/time_util.hh" #include "bound_tags.hh" #include "config.h" +#include "lnav_util.hh" #include "pcrepp/pcrepp.hh" #include "readline_context.hh" #include "readline_highlighters.hh" @@ -345,7 +346,8 @@ walk_sqlite_metadata(sqlite3* db, struct sqlite_metadata_callbacks& smc) } for (auto iter = smc.smc_db_list.begin(); iter != smc.smc_db_list.end(); - ++iter) { + ++iter) + { struct table_list_data tld = {&smc, &iter}; auto_mem query; @@ -836,18 +838,14 @@ sql_execute_script(sqlite3* db, } default: { - attr_line_t sql_content(sqlite3_sql(stmt)); - const char* errmsg; - - errmsg = sqlite3_errmsg(db); - sql_content.with_attr_for_all( - VC_ROLE.value(role_t::VCR_QUOTED_CODE)); - readline_sqlite_highlighter(sql_content, - sql_content.length()); + const auto* sql_str = sqlite3_sql(stmt); + auto sql_content + = annotate_sql_with_error(db, sql_str, nullptr); + errors.emplace_back( lnav::console::user_message::error( "failed to execute SQL statement") - .with_reason(errmsg) + .with_reason(sqlite3_errmsg_to_attr_line(db)) .with_snippet(lnav::console::snippet::from( intern_string::lookup(src_name), sql_content))); done = true; @@ -971,6 +969,24 @@ sqlite_authorizer(void* pUserData, return SQLITE_OK; } +attr_line_t +sqlite3_errmsg_to_attr_line(sqlite3* db) +{ + const auto* errmsg = sqlite3_errmsg(db); + if (startswith(errmsg, sqlitepp::ERROR_PREFIX)) { + auto from_res = lnav::from_json( + &errmsg[strlen(sqlitepp::ERROR_PREFIX)]); + + if (from_res.isOk()) { + return from_res.unwrap().to_attr_line(); + } + + return from_res.unwrapErr()[0].um_message.get_string(); + } + + return attr_line_t(errmsg); +} + std::string sql_keyword_re() { @@ -1064,7 +1080,8 @@ annotate_sql_statement(attr_line_t& al) int start = 0; while ((iter = find_string_attr(sa, &SQL_IDENTIFIER_ATTR, start)) - != sa.end()) { + != sa.end()) + { string_attrs_t::const_iterator piter; bool found_open = false; ssize_t lpc; @@ -1177,7 +1194,8 @@ find_sql_help_for_line(const attr_line_t& al, size_t x) if (help_count > 1 && name != func_pair.first->second->ht_name) { while (func_pair.first != func_pair.second) { if (find(kw.begin(), kw.end(), func_pair.first->second->ht_name) - == kw.end()) { + == kw.end()) + { ++func_pair.first; } else { func_pair.second = next(func_pair.first); diff --git a/src/sql_util.hh b/src/sql_util.hh index 81bae60a..e115c995 100644 --- a/src/sql_util.hh +++ b/src/sql_util.hh @@ -120,6 +120,8 @@ int guess_type_from_pcre(const std::string& pattern, std::string& collator); const char* sqlite3_type_to_string(int type); +attr_line_t sqlite3_errmsg_to_attr_line(sqlite3* db); + attr_line_t annotate_sql_with_error(sqlite3* db, const char* sql, const char* tail); diff --git a/src/sqlitepp.cc b/src/sqlitepp.cc new file mode 100644 index 00000000..906d248f --- /dev/null +++ b/src/sqlitepp.cc @@ -0,0 +1,36 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "sqlitepp.hh" + +namespace sqlitepp { + +const char* ERROR_PREFIX = "lnav-error:"; + +} diff --git a/src/sqlitepp.hh b/src/sqlitepp.hh index 489abef0..7ebb798b 100644 --- a/src/sqlitepp.hh +++ b/src/sqlitepp.hh @@ -34,7 +34,10 @@ #include +#include + #include "base/auto_mem.hh" +#include "optional.hpp" /* XXX figure out how to do this with the template */ void sqlite_close_wrapper(void* mem); @@ -55,6 +58,8 @@ quote(const nonstd::optional& str) return retval; } +extern const char* ERROR_PREFIX; + } // namespace sqlitepp #endif diff --git a/src/tailer/tailer.looper.cc b/src/tailer/tailer.looper.cc index e7664444..e2e4c80c 100644 --- a/src/tailer/tailer.looper.cc +++ b/src/tailer/tailer.looper.cc @@ -598,7 +598,8 @@ tailer::looper::host_tailer::loop_body() auto finished_child = std::move(conn).close(); if (finished_child.exit_status() != 0 - && !this->ht_error_queue.empty()) { + && !this->ht_error_queue.empty()) + { report_error(this->ht_netloc, this->ht_error_queue.back()); } @@ -632,7 +633,8 @@ tailer::looper::host_tailer::loop_body() auto child_iter = conn.c_child_paths.find(pe.pe_path); if (child_iter != conn.c_child_paths.end() - && !child_iter->second.loo_tail) { + && !child_iter->second.loo_tail) + { conn.c_child_paths.erase(child_iter); } } @@ -1029,7 +1031,8 @@ tailer::looper::child_finished(std::shared_ptr child) auto child_tailer = std::static_pointer_cast(child); for (auto iter = this->l_remotes.begin(); iter != this->l_remotes.end(); - ++iter) { + ++iter) + { if (iter->second != child_tailer) { continue; } @@ -1101,12 +1104,13 @@ tailer::cleanup_cache() (void) std::async(std::launch::async, []() { auto now = std::chrono::system_clock::now(); auto cache_path = remote_cache_path(); - auto& cfg = injector::get(); + const auto& cfg = injector::get(); std::vector to_remove; log_debug("cache-ttl %d", cfg.c_cache_ttl.count()); for (const auto& entry : - ghc::filesystem::directory_iterator(cache_path)) { + ghc::filesystem::directory_iterator(cache_path)) + { auto mtime = ghc::filesystem::last_write_time(entry.path()); auto exp_time = mtime + cfg.c_cache_ttl; if (now < exp_time) { diff --git a/src/textview_curses.hh b/src/textview_curses.hh index c8e67e51..db649e9d 100644 --- a/src/textview_curses.hh +++ b/src/textview_curses.hh @@ -577,7 +577,11 @@ public: return retval; } - void grep_quiesce() { this->tc_sub_source->quiesce(); } + void grep_quiesce() { + if (this->tc_sub_source != nullptr) { + this->tc_sub_source->quiesce(); + } + } void grep_begin(grep_proc& gp, vis_line_t start, diff --git a/src/view_helpers.cc b/src/view_helpers.cc index 66c4a535..b81f9caf 100644 --- a/src/view_helpers.cc +++ b/src/view_helpers.cc @@ -155,7 +155,8 @@ public: if (parent_node != nullptr) { for (const auto& sibling : - parent_node->hn_named_children) { + parent_node->hn_named_children) + { retval.template emplace_back(sibling.first); } } @@ -306,7 +307,8 @@ open_pretty_view() bool first_line = true; for (vis_line_t vl = log_tc->get_top(); vl <= log_tc->get_bottom(); - ++vl) { + ++vl) + { content_line_t cl = lss.at(vl); auto lf = lss.find(cl); auto ll = lf->begin() + cl; @@ -409,7 +411,8 @@ open_pretty_view() for (vis_line_t vl = text_tc->get_top(); vl <= text_tc->get_bottom(); - ++vl) { + ++vl) + { auto ll = lf->begin() + vl; shared_buffer_ref sbr; @@ -796,7 +799,8 @@ execute_examples() dos.list_value_for_overlay(db_tc, 0, 1, 0_vl, al); result.append(al); for (int lpc = 0; lpc < (int) dls.text_line_count(); - lpc++) { + lpc++) + { al.clear(); dls.text_value_for_line( db_tc, lpc, al.get_string(), false); @@ -864,6 +868,10 @@ toggle_view(textview_curses* toggle_tc) rebuild_hist(); } else if (toggle_tc == &lnav_data.ld_views[LNV_HELP]) { build_all_help_text(); + if (lnav_data.ld_rl_view != nullptr) { + lnav_data.ld_rl_view->set_alt_value( + HELP_MSG_1(q, "to return to the previous view")); + } } lnav_data.ld_last_view = nullptr; lnav_data.ld_view_stack.push_back(toggle_tc); diff --git a/src/views_vtab.cc b/src/views_vtab.cc index 68a01921..3326623b 100644 --- a/src/views_vtab.cc +++ b/src/views_vtab.cc @@ -194,15 +194,9 @@ CREATE TABLE lnav_views ( using iterator = textview_curses*; - iterator begin() - { - return std::begin(lnav_data.ld_views); - } + iterator begin() { return std::begin(lnav_data.ld_views); } - iterator end() - { - return std::end(lnav_data.ld_views); - } + iterator end() { return std::end(lnav_data.ld_views); } int get_column(cursor& vc, sqlite3_context* ctx, int col) { @@ -417,15 +411,9 @@ CREATE TABLE lnav_view_stack ( ); )"; - iterator begin() - { - return lnav_data.ld_view_stack.begin(); - } + iterator begin() { return lnav_data.ld_view_stack.begin(); } - iterator end() - { - return lnav_data.ld_view_stack.end(); - } + iterator end() { return lnav_data.ld_view_stack.end(); } int get_column(cursor& vc, sqlite3_context* ctx, int col) { @@ -535,10 +523,7 @@ struct lnav_view_filter_base { return ++retval; } - iterator end() - { - return {LNV__MAX, -1}; - } + iterator end() { return {LNV__MAX, -1}; } sqlite_int64 get_rowid(iterator iter) { @@ -731,7 +716,7 @@ CREATE TABLE lnav_view_filters ( if (set_res.isErr()) { tab->zErrMsg = sqlite3_mprintf( "%s%s", - LNAV_SQLITE_ERROR_PREFIX, + sqlitepp::ERROR_PREFIX, lnav::to_json(set_res.unwrapErr()).c_str()); return SQLITE_ERROR; } @@ -832,7 +817,7 @@ CREATE TABLE lnav_view_filters ( if (set_res.isErr()) { tab->zErrMsg = sqlite3_mprintf( "%s%s", - LNAV_SQLITE_ERROR_PREFIX, + sqlitepp::ERROR_PREFIX, lnav::to_json(set_res.unwrapErr()).c_str()); return SQLITE_ERROR; } diff --git a/src/vtab_module.cc b/src/vtab_module.cc index 66f488e3..725b0c91 100644 --- a/src/vtab_module.cc +++ b/src/vtab_module.cc @@ -31,18 +31,17 @@ #include "config.h" #include "lnav_util.hh" +#include "sqlitepp.hh" std::string vtab_module_schemas; std::map vtab_module_ddls; -const char* LNAV_SQLITE_ERROR_PREFIX = "lnav-error:"; - void to_sqlite(sqlite3_context* ctx, const lnav::console::user_message& um) { auto errmsg = fmt::format( - FMT_STRING("{}{}"), LNAV_SQLITE_ERROR_PREFIX, lnav::to_json(um)); + FMT_STRING("{}{}"), sqlitepp::ERROR_PREFIX, lnav::to_json(um)); sqlite3_result_error(ctx, errmsg.c_str(), errmsg.size()); } @@ -51,9 +50,9 @@ lnav::console::user_message sqlite3_error_to_user_message(sqlite3* db) { const auto* errmsg = sqlite3_errmsg(db); - if (startswith(errmsg, LNAV_SQLITE_ERROR_PREFIX)) { + if (startswith(errmsg, sqlitepp::ERROR_PREFIX)) { auto from_res = lnav::from_json( - &errmsg[strlen(LNAV_SQLITE_ERROR_PREFIX)]); + &errmsg[strlen(sqlitepp::ERROR_PREFIX)]); if (from_res.isOk()) { return from_res.unwrap(); diff --git a/src/vtab_module.hh b/src/vtab_module.hh index 2e72bf9b..95811eae 100644 --- a/src/vtab_module.hh +++ b/src/vtab_module.hh @@ -48,8 +48,6 @@ #include "shlex.resolver.hh" #include "sqlite-extension-func.hh" -extern const char* LNAV_SQLITE_ERROR_PREFIX; - lnav::console::user_message sqlite3_error_to_user_message(sqlite3*); struct from_sqlite_conversion_error : std::exception { diff --git a/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err b/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err index 341d629b..55f0cda8 100644 --- a/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err +++ b/test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err @@ -88,7 +88,7 @@ reason: sample does not match any patterns  --> {test_dir}/bad-config/formats/invalid-sample/format.json:41  = note: the following shows how each pattern matched this sample: - 1428634687123; foo bar + 1428634687123; foo bar ^ bad-time matched up to here ^ semi matched up to here ^ std matched up to here @@ -125,7 +125,8 @@  | CREATE TALE invalid (x y z);   |  ^ near "TALE": syntax error  ✘ error: failed to execute SQL statement - reason: lnav-error:{"level":"error","message":{"str":"call to regexp_match(re, str) failed","attrs":[{"start":8,"end":20,"type":"role","value":46},{"start":21,"end":23,"type":"role","value":45},{"start":25,"end":28,"type":"role","value":45},{"start":8,"end":29,"type":"role","value":59}]},"reason":{"str":"missing )","attrs":[]},"snippets":[],"help":{"str":"","attrs":[]}} + reason: ✘ error: call to regexp_match(re, str) failed + |  reason: missing )  --> {test_dir}/bad-config/formats/invalid-sql/init2.sql  | SELECT regexp_match('abc(', '123')   | FROM sqlite_master;  diff --git a/test/expected/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out b/test/expected/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out index 4bc8ab86..a11ff066 100644 --- a/test/expected/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out +++ b/test/expected/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out @@ -38,13 +38,13 @@  ... 33 common frames omitted  @version: 1 - logger_name: org.apache.jasper.runtime.JspFactoryImpl + logger_name: org.apache.jasper.runtime.JspFactoryImpl  thread_name: http-bio-0.0.0.0-8081-exec-198  level: ERROR  customer: foobaz 2016-08-03T12:06:31.009 - ;Exception initializing page context;   @version: 1 - logger_name: org.apache.jasper.runtime.JspFactoryImpl + logger_name: org.apache.jasper.runtime.JspFactoryImpl  thread_name: http-bio-0.0.0.0-8081-exec-198  level: ERROR  customer: foobaz