From 884e2df6adc0a175cf24ccc43f8133ed9e622f1e Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Tue, 29 Aug 2023 22:26:00 -0700 Subject: [PATCH] [:eval] treat the argument like the contents of a file with multiple commands --- NEWS.md | 10 +- docs/schemas/config-v1.schema.json | 2 +- src/CMakeLists.txt | 1 + src/base/intern_string.cc | 4 +- src/base/math_util.hh | 4 + src/command_executor.cc | 160 ++++++---- src/command_executor.hh | 22 +- src/formats/access_log.json | 3 + src/formats/formats.am | 1 + src/formats/pcap_log.json | 1 - src/formats/redis_log.json | 51 ++++ src/internals/cmd-ref.rst | 5 +- src/listview_curses.cc | 25 ++ src/lnav.cc | 2 + src/lnav_commands.cc | 237 +++++++++++---- src/lnav_config.cc | 2 +- src/log_format.cc | 11 +- src/log_format.hh | 1 + src/logfile.cc | 10 +- src/logfile_sub_source.cc | 4 +- src/ptimec.hh | 5 +- src/readline_callbacks.cc | 3 +- src/readline_curses.cc | 159 +++++----- src/readline_highlighters.cc | 90 +++--- src/root-config.json | 3 + src/scripts/docker-compose-url-handler.lnav | 15 - src/scripts/docker-url-handler.lnav | 36 ++- src/scripts/pcap_log-converter.sh | 2 +- src/scripts/scripts.am | 1 - src/shlex.cc | 285 +++++++++++------- src/shlex.hh | 52 +++- src/sql_commands.cc | 18 +- src/textfile_sub_source.cc | 10 +- src/time_formats.am | 1 + src/top_status_source.cc | 3 +- test/drive_shlexer.cc | 58 ++-- test/expected/expected.am | 2 + ...3639753916f71254e8c9cce4ebb8bfd9978d3e.out | 3 + ...06341dd560f927512e92c7c0985ed8b25827ae.out | 3 +- ...a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out | 8 +- ...d0ff9b68adc17136329f457fe52d5addcb12c0.err | 6 + ...d0ff9b68adc17136329f457fe52d5addcb12c0.out | 0 ...dd967cb2af90899c9e5e45d00b676b5a3163aa.out | 5 +- ...81f5dd570580cbe746ad91b58a28b8371283b3.out | 5 +- ...f44d06fc137a77bc230be86376ccad23a2806b.out | 2 +- ...58e530a8ecb77cbaec1a7507768dd5a1942ac9.out | 3 +- ...31e16ea2469da7a4328c93c7bcc8e109f84d2f.out | 3 +- ...eebcdef56edd783579eaaddaff7c5cc127bb86.out | 3 +- ...9addb0e5b6f4254d81dd89ecf12783109644bb.out | 3 +- ...961e6728e96d0a44535a6c9907cc990c10316c.out | 3 +- ...c4e861804a5434900fdb4d67b149d1baa2edf4.out | 3 +- ...fe5f6b8fc9ba00539fad0fa0bfb08319d8b04b.out | 3 +- ...d46422a913e3a06ddbd262933ef5352c30e68f.out | 7 +- ...599f0b53d1bd27af767113853f8e84291f137d.out | 3 +- ...fa2239ab17e7563d0c524f5400a79d6ff8bfda.out | 3 +- test/formats/jsontest/lnav-logstash.json | 56 ++-- test/lnav_doctests.cc | 61 ++++ test/test_cmds.sh | 4 + 58 files changed, 1010 insertions(+), 476 deletions(-) create mode 100644 src/formats/redis_log.json delete mode 100755 src/scripts/docker-compose-url-handler.lnav create mode 100644 test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err create mode 100644 test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.out diff --git a/NEWS.md b/NEWS.md index 36674fc6..996412ea 100644 --- a/NEWS.md +++ b/NEWS.md @@ -32,9 +32,12 @@ Features: lnav script. Schemes can be defined under `/tuning/url-schemes`. See the main docs for more details. * Added a `docker://` URL scheme that can be used to tail - the logs for a container (e.g. `docker://my-container`) or + the logs for containers (e.g. `docker://my-container`) or files within a container (e.g. - `docker://my-serv/var/log/dpkg.log`). + `docker://my-serv/var/log/dpkg.log`). Containers mentioned + in a "Compose" configuration file can be tailed by using + `compose` as the host name with the path to the configuration + file (e.g. `docker://compose/compose.yaml`). * Added an `:annotate` command that can trigger a call-out to a script to analyze a log message and generate an annotation that is attached to the message. The script @@ -85,6 +88,9 @@ Features: are now recognized and styled as appropriate. * Added a `data` column to the `fstat()` table-valued- function so the contents of a file can be read. +* Added a log format for Redis. +* The `:eval` command will now treat its argument(s) as a + script, allowing multiple commands to be executed. Bug Fixes: * Binary data piped into stdin should now be treated the same diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index dde28bf8..603b56c4 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -214,7 +214,7 @@ "title": "/tuning/url-scheme", "type": "object", "patternProperties": { - "(\\w+)": { + "([a-z][\\w\\-\\+\\.]+)": { "description": "Definition of a custom URL scheme", "title": "/tuning/url-scheme/", "type": "object", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd288c16..9800eab9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,6 +96,7 @@ set(TIME_FORMATS "%m/%e/%Y %l:%M:%S%p" "%m/%d/%y %H:%M:%S" "%m/%d/%Y %H:%M:%S" + "%d/%b/%Y %H:%M:%S" "%d/%b/%y %H:%M:%S" "%m%d %H:%M:%S" "%Y%m%d %H:%M:%S" diff --git a/src/base/intern_string.cc b/src/base/intern_string.cc index 5d6838a8..a175eb6f 100644 --- a/src/base/intern_string.cc +++ b/src/base/intern_string.cc @@ -225,7 +225,9 @@ string_fragment::split_lines() const start = index + 1; } } - retval.emplace_back(this->sf_string, start, this->sf_end); + if (retval.empty() || start < this->sf_end) { + retval.emplace_back(this->sf_string, start, this->sf_end); + } return retval; } diff --git a/src/base/math_util.hh b/src/base/math_util.hh index 57ac4535..fcd507cc 100644 --- a/src/base/math_util.hh +++ b/src/base/math_util.hh @@ -124,6 +124,10 @@ public: bool try_consume(T rhs) { + if (rhs == 0) { + return false; + } + if (this->c_value - rhs > this->c_min) { this->c_value -= rhs; return true; diff --git a/src/command_executor.cc b/src/command_executor.cc index 73853d51..6d6e3138 100644 --- a/src/command_executor.cc +++ b/src/command_executor.cc @@ -106,7 +106,7 @@ sql_progress_finished() static Result execute_from_file( exec_context& ec, - const ghc::filesystem::path& path, + const std::string& src, int line_number, const std::string& cmdline); @@ -499,6 +499,7 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg) if (lnav_data.ld_flags & LNF_HEADLESS) { if (ec.ec_local_vars.size() == 1) { + lnav_data.ld_views[LNV_DB].reload_data(); ensure_view(&lnav_data.ld_views[LNV_DB]); } } @@ -509,13 +510,73 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg) return Ok(retval); } +Result +multiline_executor::push_back(string_fragment line) +{ + this->me_line_number += 1; + + if (line.trim().empty()) { + if (this->me_cmdline) { + this->me_cmdline = this->me_cmdline.value() + "\n"; + } + return Ok(); + } + if (line[0] == '#') { + return Ok(); + } + + switch (line[0]) { + case ':': + case '/': + case ';': + case '|': + if (this->me_cmdline) { + this->me_last_result + = TRY(execute_from_file(this->me_exec_context, + this->me_source, + this->me_starting_line_number, + trim(this->me_cmdline.value()))); + } + + this->me_starting_line_number = this->me_line_number; + this->me_cmdline = line.to_string(); + break; + default: + if (this->me_cmdline) { + this->me_cmdline = fmt::format( + FMT_STRING("{}{}"), this->me_cmdline.value(), line); + } else { + this->me_last_result = TRY( + execute_from_file(this->me_exec_context, + this->me_source, + this->me_line_number, + fmt::format(FMT_STRING(":{}"), line))); + } + break; + } + + return Ok(); +} + +Result +multiline_executor::final() +{ + if (this->me_cmdline) { + this->me_last_result + = TRY(execute_from_file(this->me_exec_context, + this->me_source, + this->me_starting_line_number, + trim(this->me_cmdline.value()))); + } + + return Ok(this->me_last_result); +} + static Result -execute_file_contents(exec_context& ec, - const ghc::filesystem::path& path, - bool multiline) +execute_file_contents(exec_context& ec, const ghc::filesystem::path& path) { - static ghc::filesystem::path stdin_path("-"); - static ghc::filesystem::path dev_stdin_path("/dev/stdin"); + static const ghc::filesystem::path stdin_path("-"); + static const ghc::filesystem::path dev_stdin_path("/dev/stdin"); std::string retval; FILE* file; @@ -529,59 +590,18 @@ execute_file_contents(exec_context& ec, return ec.make_error("unable to open file"); } - int line_number = 0, starting_line_number = 0; auto_mem line; size_t line_max_size; ssize_t line_size; - nonstd::optional cmdline; + multiline_executor me(ec, path.string()); ec.ec_path_stack.emplace_back(path.parent_path()); exec_context::output_guard og(ec); while ((line_size = getline(line.out(), &line_max_size, file)) != -1) { - line_number += 1; - - if (trim(line.in()).empty()) { - if (multiline && cmdline) { - cmdline = cmdline.value() + "\n"; - } - continue; - } - if (line[0] == '#') { - continue; - } - - switch (line[0]) { - case ':': - case '/': - case ';': - case '|': - if (cmdline) { - retval = TRY(execute_from_file( - ec, path, starting_line_number, trim(cmdline.value()))); - } - - starting_line_number = line_number; - cmdline = std::string(line); - break; - default: - if (multiline && cmdline) { - cmdline = fmt::format( - FMT_STRING("{}{}"), cmdline.value(), line.in()); - } else { - retval = TRY(execute_from_file( - ec, - path, - line_number, - fmt::format(FMT_STRING(":{}"), line.in()))); - } - break; - } + TRY(me.push_back(string_fragment::from_bytes(line.in(), line_size))); } - if (cmdline) { - retval = TRY(execute_from_file( - ec, path, starting_line_number, trim(cmdline.value()))); - } + retval = TRY(me.final()); if (file == stdin) { if (isatty(STDOUT_FILENO)) { @@ -596,25 +616,35 @@ execute_file_contents(exec_context& ec, } Result -execute_file(exec_context& ec, const std::string& path_and_args, bool multiline) +execute_file(exec_context& ec, const std::string& path_and_args) { + static const intern_string_t SRC = intern_string::lookup("cmdline"); + available_scripts scripts; - std::vector split_args; std::string retval, msg; shlex lexer(path_and_args); log_info("Executing file: %s", path_and_args.c_str()); - if (!lexer.split(split_args, scoped_resolver{&ec.ec_local_vars.top()})) { - return ec.make_error("unable to parse path"); + auto split_args_res = lexer.split(scoped_resolver{&ec.ec_local_vars.top()}); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um = lnav::console::user_message::error( + "unable to parse script command-line") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); } + auto split_args = split_args_res.unwrap(); if (split_args.empty()) { return ec.make_error("no script specified"); } ec.ec_local_vars.push({}); - auto script_name = split_args[0]; + auto script_name = split_args[0].se_value; auto& vars = ec.ec_local_vars.top(); char env_arg_name[32]; std::string star, open_error = "file not found"; @@ -627,13 +657,13 @@ execute_file(exec_context& ec, const std::string& path_and_args, bool multiline) vars["#"] = env_arg_name; for (size_t lpc = 0; lpc < split_args.size(); lpc++) { snprintf(env_arg_name, sizeof(env_arg_name), "%lu", lpc); - vars[env_arg_name] = split_args[lpc]; + vars[env_arg_name] = split_args[lpc].se_value; } for (size_t lpc = 1; lpc < split_args.size(); lpc++) { if (lpc > 1) { star.append(" "); } - star.append(split_args[lpc]); + star.append(split_args[lpc].se_value); } vars["__all__"] = star; @@ -674,8 +704,7 @@ execute_file(exec_context& ec, const std::string& path_and_args, bool multiline) if (!paths_to_exec.empty()) { for (auto& path_iter : paths_to_exec) { - retval - = TRY(execute_file_contents(ec, path_iter.sm_path, multiline)); + retval = TRY(execute_file_contents(ec, path_iter.sm_path)); } } ec.ec_local_vars.pop(); @@ -690,13 +719,13 @@ execute_file(exec_context& ec, const std::string& path_and_args, bool multiline) Result execute_from_file(exec_context& ec, - const ghc::filesystem::path& path, + const std::string& src, int line_number, const std::string& cmdline) { std::string retval, alt_msg; - auto _sg = ec.enter_source( - intern_string::lookup(path.string()), line_number, cmdline); + auto _sg + = ec.enter_source(intern_string::lookup(src), line_number, cmdline); switch (cmdline[0]) { case ':': @@ -717,10 +746,8 @@ execute_from_file(exec_context& ec, break; } - log_info("%s:%d:execute result -- %s", - path.c_str(), - line_number, - retval.c_str()); + log_info( + "%s:%d:execute result -- %s", src.c_str(), line_number, retval.c_str()); return Ok(retval); } @@ -842,6 +869,7 @@ execute_init_commands( } if (dls.dls_rows.size() > 1 && lnav_data.ld_view_stack.size() == 1) { + lnav_data.ld_views[LNV_DB].reload_data(); ensure_view(LNV_DB); } } diff --git a/src/command_executor.hh b/src/command_executor.hh index 42db4c95..8ba313ab 100644 --- a/src/command_executor.hh +++ b/src/command_executor.hh @@ -284,8 +284,28 @@ Result execute_command( Result execute_sql( exec_context& ec, const std::string& sql, std::string& alt_msg); + +class multiline_executor { +public: + exec_context& me_exec_context; + std::string me_source; + nonstd::optional me_cmdline; + int me_line_number{0}; + int me_starting_line_number{0}; + std::string me_last_result; + + multiline_executor(exec_context& ec, std::string src) + : me_exec_context(ec), me_source(src) + { + } + + Result push_back(string_fragment line); + + Result final(); +}; + Result execute_file( - exec_context& ec, const std::string& path_and_args, bool multiline = true); + exec_context& ec, const std::string& path_and_args); Result execute_any( exec_context& ec, const std::string& cmdline); void execute_init_commands( diff --git a/src/formats/access_log.json b/src/formats/access_log.json index 6a5b0201..b71d2101 100644 --- a/src/formats/access_log.json +++ b/src/formats/access_log.json @@ -111,6 +111,9 @@ { "line": "10.112.2.3 - - [16/Sep/2022:00:53:14 +0200] \"POST /api/v4/jobs/request HTTP/1.1\" 204 0 \"\" \"gitlab-runner 15.3.0 (15-3-stable; go1.19; linux/amd64)\" -", "level": "info" + }, + { + "line": "172.18.0.1 - - [29/Aug/2023 22:02:58] \"GET / HTTP/1.1\" 200 -" } ] } diff --git a/src/formats/formats.am b/src/formats/formats.am index 66b0ab31..3449f3a7 100644 --- a/src/formats/formats.am +++ b/src/formats/formats.am @@ -27,6 +27,7 @@ FORMAT_FILES = \ $(srcdir)/%reldir%/papertrail_log.json \ $(srcdir)/%reldir%/pcap_log.json \ $(srcdir)/%reldir%/procstate_log.json \ + $(srcdir)/%reldir%/redis_log.json \ $(srcdir)/%reldir%/snaplogic_log.json \ $(srcdir)/%reldir%/sssd_log.json \ $(srcdir)/%reldir%/strace_log.json \ diff --git a/src/formats/pcap_log.json b/src/formats/pcap_log.json index 93b110ec..a9e86efb 100644 --- a/src/formats/pcap_log.json +++ b/src/formats/pcap_log.json @@ -4,7 +4,6 @@ "json": true, "title": "Packet Capture", "description": "Internal format for pcap files", - "multiline": false, "convert-to-local-time": true, "converter": { "header": { diff --git a/src/formats/redis_log.json b/src/formats/redis_log.json new file mode 100644 index 00000000..e07de332 --- /dev/null +++ b/src/formats/redis_log.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "redis_log": { + "url": [ + "https://redis.com", + "https://build47.com/redis-log-format-levels/" + ], + "description": "The Redis database", + "regex": { + "v3.x": { + "pattern": "(?\\d+):(?[XCSM])\\s+(?\\d{1,2} [a-zA-Z]{3} \\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{3})\\s+(?[\\.\\0\\*\\#])\\s+(?.*)" + }, + "sig": { + "pattern": "(?\\d+):signal-handler \\((?\\d+)\\) (?.*)" + } + }, + "timestamp-format": [ + "%s", + "%d %b %Y %H:%M:%S.%L" + ], + "level": { + "debug": "^\\.$", + "trace": "^-$", + "notice": "^\\*$", + "warning": "^#$" + }, + "value": { + "level": { + "kind": "string" + }, + "pid": { + "kind": "string", + "identifier": true + }, + "role": { + "kind": "string" + }, + "timestamp": { + "kind": "string" + } + }, + "sample": [ + { + "line": "1:M 29 Aug 2023 13:47:38.984 * monotonic clock: POSIX clock_gettime" + }, + { + "line": "1:signal-handler (1693279182) Received SIGTERM scheduling shutdown..." + } + ] + } +} diff --git a/src/internals/cmd-ref.rst b/src/internals/cmd-ref.rst index 7df65cb8..d58b1e35 100644 --- a/src/internals/cmd-ref.rst +++ b/src/internals/cmd-ref.rst @@ -1209,12 +1209,13 @@ .. _sh: -:sh *cmdline* -^^^^^^^^^^^^^ +:sh *--name=* *cmdline* +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Execute the given command-line and display the captured output **Parameters** + * **--name=\*** --- The name to give to the captured output * **cmdline\*** --- The command-line to execute. **See Also** diff --git a/src/listview_curses.cc b/src/listview_curses.cc index 53258189..c4648d6c 100644 --- a/src/listview_curses.cc +++ b/src/listview_curses.cc @@ -255,6 +255,14 @@ listview_curses::handle_key(int ch) case 'g': case KEY_HOME: + if (this->lv_overlay_focused) { + this->lv_focused_overlay_top = 0_vl; + this->lv_focused_overlay_selection = 0_vl; + this->lv_source->listview_selection_changed(*this); + this->set_needs_update(); + break; + } + if (this->is_selectable()) { this->set_selection(0_vl); } else { @@ -264,6 +272,23 @@ listview_curses::handle_key(int ch) case 'G': case KEY_END: { + if (this->lv_overlay_focused) { + std::vector overlay_content; + this->lv_overlay_source->list_value_for_overlay( + *this, this->get_selection(), overlay_content); + auto overlay_height + = this->get_overlay_height(overlay_content.size(), height); + auto ov_top_for_last = vis_line_t{ + static_cast(overlay_content.size() - overlay_height)}; + + this->lv_focused_overlay_top = ov_top_for_last; + this->lv_focused_overlay_selection + = vis_line_t(overlay_content.size() - 1); + this->lv_source->listview_selection_changed(*this); + this->set_needs_update(); + break; + } + vis_line_t last_line(this->get_inner_height() - 1); vis_line_t tail_bottom(this->get_top_for_last_row()); diff --git a/src/lnav.cc b/src/lnav.cc index ccd115c7..38159d68 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -2242,6 +2242,8 @@ main(int argc, char* argv[]) SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' )"; + lnav_data.ld_child_pollers.clear(); + for (auto& lf : lnav_data.ld_active_files.fc_files) { lf->close(); } diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 6a3f3032..d4f86eed 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -273,7 +273,10 @@ com_unix_time(exec_context& ec, char* rest; u_time = time(nullptr); - log_time = *localtime(&u_time); + if (localtime_r(&u_time, &log_time) == nullptr) { + return ec.make_error( + "invalid epoch time: {} -- {}", u_time, strerror(errno)); + } log_time.tm_isdst = -1; @@ -294,7 +297,10 @@ com_unix_time(exec_context& ec, u_time = mktime(&log_time); parsed = true; } else if (sscanf(args[1].c_str(), "%ld", &u_time)) { - log_time = *localtime(&u_time); + if (localtime_r(&u_time, &log_time) == nullptr) { + return ec.make_error( + "invalid epoch time: {} -- {}", args[1], strerror(errno)); + } parsed = true; } @@ -305,7 +311,7 @@ com_unix_time(exec_context& ec, strftime(ftime, sizeof(ftime), "%a %b %d %H:%M:%S %Y %z %Z", - localtime(&u_time)); + localtime_r(&u_time, &log_time)); len = strlen(ftime); snprintf(ftime + len, sizeof(ftime) - len, " -- %ld", u_time); retval = std::string(ftime); @@ -1026,6 +1032,8 @@ com_save_to(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("path"); + FILE *outfile = nullptr, *toclose = nullptr; const char* mode = ""; std::string fn, retval; @@ -1040,13 +1048,22 @@ com_save_to(exec_context& ec, fn = trim(remaining_args(cmdline, args)); - std::vector split_args; shlex lexer(fn); - if (!lexer.split(split_args, ec.create_resolver())) { - return ec.make_error("unable to parse arguments"); + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um + = lnav::console::user_message::error("unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); } + auto split_args = split_args_res.unwrap() + | lnav::itertools::map([](const auto& elem) { return elem.se_value; }); auto anon_iter = std::find(split_args.begin(), split_args.end(), "--anonymize"); if (anon_iter != split_args.end()) { @@ -1724,6 +1741,8 @@ com_redirect_to(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("path"); + if (args.empty()) { args.emplace_back("filename"); return Ok(std::string()); @@ -1739,16 +1758,21 @@ com_redirect_to(exec_context& ec, } std::string fn = trim(remaining_args(cmdline, args)); - std::vector split_args; shlex lexer(fn); - scoped_resolver scopes = { - &ec.ec_local_vars.top(), - &ec.ec_global_vars, - }; - if (!lexer.split(split_args, scopes)) { - return ec.make_error("unable to parse arguments"); + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um + = lnav::console::user_message::error("unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); } + auto split_args = split_args_res.unwrap() + | lnav::itertools::map([](const auto& elem) { return elem.se_value; }); if (split_args.size() > 1) { return ec.make_error("more than one file name was matched"); } @@ -2521,6 +2545,7 @@ com_session(exec_context& ec, static Result com_open(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("path"); std::string retval; if (args.empty()) { @@ -2542,17 +2567,22 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) pat = trim(remaining_args(cmdline, args)); - std::vector split_args; shlex lexer(pat); - scoped_resolver scopes = { - &ec.ec_local_vars.top(), - &ec.ec_global_vars, - }; + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um + = lnav::console::user_message::error("unable to parse file names") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); - if (!lexer.split(split_args, scopes)) { - return ec.make_error("unable to parse arguments"); + return Err(um); } + auto split_args = split_args_res.unwrap() + | lnav::itertools::map([](const auto& elem) { return elem.se_value; }); + std::vector> files_to_front; std::vector closed_files; logfile_open_options loo; @@ -2871,6 +2901,7 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) static Result com_close(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("path"); std::string retval; if (args.empty()) { @@ -2885,7 +2916,21 @@ com_close(exec_context& ec, std::string cmdline, std::vector& args) if (args.size() > 1) { auto lexer = shlex(cmdline); - lexer.split(args, ec.create_resolver()); + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um = lnav::console::user_message::error( + "unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); + } + + auto args = split_args_res.unwrap() + | lnav::itertools::map( + [](const auto& elem) { return elem.se_value; }); args.erase(args.begin()); for (const auto& lf : lnav_data.ld_active_files.fc_files) { @@ -2976,6 +3021,7 @@ com_file_visibility(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("path"); bool only_this_file = false; bool make_visible; std::string retval; @@ -3031,7 +3077,21 @@ com_file_visibility(exec_context& ec, int text_file_count = 0, log_file_count = 0; auto lexer = shlex(cmdline); - lexer.split(args, ec.create_resolver()); + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um = lnav::console::user_message::error( + "unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); + } + + auto args = split_args_res.unwrap() + | lnav::itertools::map( + [](const auto& elem) { return elem.se_value; }); args.erase(args.begin()); for (const auto& lf : lnav_data.ld_active_files.fc_files) { @@ -4263,6 +4323,8 @@ com_rebuild(exec_context& ec, static Result com_cd(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("path"); + if (args.empty()) { args.emplace_back("dirname"); return Ok(std::string()); @@ -4277,17 +4339,22 @@ com_cd(exec_context& ec, std::string cmdline, std::vector& args) pat = trim(remaining_args(cmdline, args)); - std::vector split_args; shlex lexer(pat); - scoped_resolver scopes = { - &ec.ec_local_vars.top(), - &ec.ec_global_vars, - }; + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um + = lnav::console::user_message::error("unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); - if (!lexer.split(split_args, scopes)) { - return ec.make_error("unable to parse arguments"); + return Err(um); } + auto split_args = split_args_res.unwrap() + | lnav::itertools::map([](const auto& elem) { return elem.se_value; }); + if (split_args.size() != 1) { return ec.make_error("expecting a single argument"); } @@ -4325,7 +4392,24 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) static size_t EXEC_COUNT = 0; if (!ec.ec_dry_run) { - auto carg = trim(cmdline.substr(args[0].size())); + nonstd::optional name_flag; + + shlex lexer(cmdline); + auto cmd_start = args[0].size(); + auto split_res = lexer.split(ec.create_resolver()); + if (split_res.isOk()) { + auto flags = split_res.unwrap(); + if (flags.size() >= 2) { + static const char* NAME_FLAG = "--name="; + + if (startswith(flags[1].se_value, NAME_FLAG)) { + name_flag = flags[1].se_value.substr(strlen(NAME_FLAG)); + cmd_start = flags[1].se_origin.sf_end; + } + } + } + + auto carg = trim(cmdline.substr(cmd_start)); log_info("executing: %s", carg.c_str()); @@ -4386,10 +4470,36 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) _exit(EXIT_FAILURE); } - auto display_name = ec.get_provenance() - .value_or(exec_context::file_open{fmt::format( - FMT_STRING("[{}] {}"), EXEC_COUNT++, carg)}) - .fo_name; + std::string display_name; + auto open_prov = ec.get_provenance(); + if (open_prov) { + if (name_flag) { + display_name = fmt::format( + FMT_STRING("{}/{}"), open_prov->fo_name, name_flag.value()); + } else { + display_name = open_prov->fo_name; + } + } else if (name_flag) { + display_name = name_flag.value(); + } else { + display_name + = fmt::format(FMT_STRING("[{}] {}"), EXEC_COUNT++, carg); + } + + auto name_base = display_name; + size_t name_counter = 0; + + while (true) { + auto fn_iter + = lnav_data.ld_active_files.fc_file_names.find(display_name); + if (fn_iter == lnav_data.ld_active_files.fc_file_names.end()) { + break; + } + name_counter += 1; + display_name + = fmt::format(FMT_STRING("{} [{}]"), name_base, name_counter); + } + auto create_piper_res = lnav::piper::create_looper(display_name, std::move(child_fds[0].read_end()), @@ -4398,7 +4508,7 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) if (create_piper_res.isErr()) { auto um = lnav::console::user_message::error("unable to create piper") - .with_reason(child_res.unwrapErr()); + .with_reason(create_piper_res.unwrapErr()); ec.add_error_context(um); return Err(um); } @@ -4591,26 +4701,12 @@ com_eval(exec_context& ec, std::string cmdline, std::vector& args) } auto src_guard = ec.enter_source(EVAL_SRC, 1, expanded_cmd); - std::string alt_msg; - switch (expanded_cmd[0]) { - case ':': - return execute_command(ec, expanded_cmd.substr(1)); - case ';': - return execute_sql(ec, expanded_cmd.substr(1), alt_msg); - case '|': - return execute_file(ec, expanded_cmd.substr(1)); - case '/': { - auto search_cmd = expanded_cmd.substr(1); - lnav_data.ld_view_stack.top() | - [&search_cmd](auto tc) { tc->execute_search(search_cmd); }; - break; - } - default: - return ec.make_error( - "expecting argument to start with ':', ';', '/', " - "or '|' to signify a command, SQL query, or script to " - "execute"); + auto content = string_fragment::from_str(expanded_cmd); + multiline_executor me(ec, ":eval"); + for (auto line : content.split_lines()) { + TRY(me.push_back(line)); } + retval = TRY(me.final()); } else { return ec.make_error("expecting a command or query to evaluate"); } @@ -5193,25 +5289,40 @@ com_prompt(exec_context& ec, if (args.empty()) { } else if (!ec.ec_dry_run) { - args.clear(); + static const intern_string_t SRC = intern_string::lookup("flags"); auto lexer = shlex(cmdline); - lexer.split(args, ec.create_resolver()); + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um = lnav::console::user_message::error( + "unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); + } + + auto split_args = split_args_res.unwrap() + | lnav::itertools::map( + [](const auto& elem) { return elem.se_value; }); - auto alt_flag = std::find(args.begin(), args.end(), "--alt"); + auto alt_flag + = std::find(split_args.begin(), split_args.end(), "--alt"); auto is_alt = false; - if (alt_flag != args.end()) { - args.erase(alt_flag); + if (alt_flag != split_args.end()) { + split_args.erase(alt_flag); is_alt = true; } - auto prompter = PROMPT_TYPES.find(args[1]); + auto prompter = PROMPT_TYPES.find(split_args[1]); if (prompter == PROMPT_TYPES.end()) { - return ec.make_error("Unknown prompt type: {}", args[1]); + return ec.make_error("Unknown prompt type: {}", split_args[1]); } - prompter->second(args); + prompter->second(split_args); lnav_data.ld_rl_view->set_alt_focus(is_alt); } return Ok(std::string()); @@ -6058,6 +6169,8 @@ readline_context::command_t STD_COMMANDS[] = { help_text(":sh") .with_summary("Execute the given command-line and display the " "captured output") + .with_parameter(help_text( + "--name=", "The name to give to the captured output")) .with_parameter( help_text("cmdline", "The command-line to execute.")) .with_tags({"scripting"}), diff --git a/src/lnav_config.cc b/src/lnav_config.cc index efc9802a..85fdeb5b 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -1364,7 +1364,7 @@ static const struct json_path_container url_scheme_handlers = { }; static const struct json_path_container url_handlers = { - yajlpp::pattern_property_handler(R"((?\w+))") + yajlpp::pattern_property_handler(R"((?[a-z][\w\-\+\.]+))") .with_description("Definition of a custom URL scheme") .with_obj_provider( [](const yajlpp_provider_context& ypc, _lnav_config* root) { diff --git a/src/log_format.cc b/src/log_format.cc index e2116dc0..a3f8075f 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -1301,7 +1301,7 @@ external_log_format::scan(logfile& lf, } } - log_debug("%s:%d:date-time re-locked to %d", + log_debug("%s:%d: date-time re-locked to %d", lf.get_unique_path().c_str(), dst.size(), this->lf_date_time.dts_fmt_lock); @@ -1474,10 +1474,12 @@ external_log_format::scan(logfile& lf, if (orig_lock != curr_fmt) { uint32_t lock_line; - log_debug("%zu: changing pattern lock %d -> %d", + log_debug("%s:%zu: changing pattern lock %d -> (%d)%s", + lf.get_unique_path().c_str(), dst.size() - 1, orig_lock, - curr_fmt); + curr_fmt, + this->elf_pattern_order[curr_fmt]->p_name.c_str()); if (this->lf_pattern_locks.empty()) { lock_line = 0; } else { @@ -2648,6 +2650,8 @@ external_log_format::build(std::vector& errors) .with_snippets(this->get_snippets())); } if (this->elf_type == elf_type_t::ELF_TYPE_JSON) { + this->lf_multiline = true; + this->lf_structured = true; this->jlf_parse_context = std::make_shared(this->elf_name); this->jlf_yajl_handle.reset( @@ -2658,7 +2662,6 @@ external_log_format::build(std::vector& errors) yajl_config( this->jlf_yajl_handle.get(), yajl_dont_validate_strings, 1); } - } else { if (this->elf_patterns.empty()) { errors.emplace_back(lnav::console::user_message::error( diff --git a/src/log_format.hh b/src/log_format.hh index 262d53b6..0cec50af 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -542,6 +542,7 @@ public: std::string lf_description; uint8_t lf_mod_index{0}; bool lf_multiline{true}; + bool lf_structured{false}; date_time_scanner lf_date_time; date_time_scanner lf_time_scanner; std::vector lf_pattern_locks; diff --git a/src/logfile.cc b/src/logfile.cc index 4c7095ab..991a1276 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -389,10 +389,16 @@ logfile::process_prefix(shared_buffer_ref& sbr, for (size_t lpc = 0; lpc < this->lf_index.size() - 1; lpc++) { if (this->lf_format->lf_multiline) { + if (this->lf_format->lf_structured) { + this->lf_index[lpc].set_ignore(true); + } else { + this->lf_index[lpc].set_time(last_line.get_time()); + this->lf_index[lpc].set_millis(last_line.get_millis()); + } + } else { this->lf_index[lpc].set_time(last_line.get_time()); this->lf_index[lpc].set_millis(last_line.get_millis()); - } else { - this->lf_index[lpc].set_ignore(true); + this->lf_index[lpc].set_level(LEVEL_INVALID); } } diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 0a9797e7..e5faad54 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -728,6 +728,7 @@ logfile_sub_source::rebuild_index( if (force) { log_debug("forced to full rebuild"); retval = rebuild_result::rr_full_rebuild; + full_sort = true; } std::vector file_order(this->lss_files.size()); @@ -765,6 +766,7 @@ logfile_sub_source::rebuild_index( ld.ld_file_index); force = true; retval = rebuild_result::rr_full_rebuild; + full_sort = true; } } else { if (time_left && deadline && ui_clock::now() > deadline.value()) { @@ -850,7 +852,7 @@ logfile_sub_source::rebuild_index( if (this->lss_index.reserve(total_lines)) { // The index array was reallocated, just do a full sort/rebuild since - // its been cleared out. + // it's been cleared out. log_debug("expanding index capacity %zu", this->lss_index.ba_capacity); force = true; retval = rebuild_result::rr_full_rebuild; diff --git a/src/ptimec.hh b/src/ptimec.hh index 0a44af04..8af6594e 100644 --- a/src/ptimec.hh +++ b/src/ptimec.hh @@ -282,10 +282,13 @@ ptime_S(struct exttm* dst, const char* str, off_t& off_inout, ssize_t len) } dst->et_tm.tm_sec = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0'); + if (dst->et_tm.tm_sec < 0 || dst->et_tm.tm_sec >= 60) { + return false; + } dst->et_flags |= ETF_SECOND_SET; }); - return (dst->et_tm.tm_sec >= 0 && dst->et_tm.tm_sec <= 59); + return true; } inline void diff --git a/src/readline_callbacks.cc b/src/readline_callbacks.cc index 32fdb70b..ba5ec1e0 100644 --- a/src/readline_callbacks.cc +++ b/src/readline_callbacks.cc @@ -773,13 +773,14 @@ rl_callback_int(readline_curses* rc, bool is_alt) } } + tm current_tm; struct stat st; if (fstat(fd_copy, &st) != -1 && st.st_size > 0) { strftime(timestamp, sizeof(timestamp), "%a %b %d %H:%M:%S %Z", - localtime(¤t_time)); + localtime_r(¤t_time, ¤t_tm)); snprintf(desc, sizeof(desc), "Output of %s (%s)", diff --git a/src/readline_curses.cc b/src/readline_curses.cc index 7fc47bf9..3bd4957d 100644 --- a/src/readline_curses.cc +++ b/src/readline_curses.cc @@ -56,6 +56,7 @@ #include "base/ansi_scrubber.hh" #include "base/auto_mem.hh" +#include "base/itertools.hh" #include "base/lnav_log.hh" #include "base/paths.hh" #include "base/string_util.hh" @@ -374,7 +375,6 @@ readline_context::attempted_completion(const char* text, int start, int end) } else { char* space; std::string cmd; - std::vector prefix; int point = rl_point; while (point > 0 && rl_line_buffer[point] != ' ') { point -= 1; @@ -384,7 +384,11 @@ readline_context::attempted_completion(const char* text, int start, int end) arg_possibilities = nullptr; rl_completion_append_character = 0; - if (lexer.split(prefix, scoped_resolver{&scope})) { + auto split_res = lexer.split(scoped_resolver{&scope}); + if (split_res.isOk()) { + auto prefix = split_res.unwrap() + | lnav::itertools::map( + [](const auto& elem) { return elem.se_value; }); auto prefix2 = fmt::format(FMT_STRING("{}"), fmt::join(prefix, "\x1f")); auto prefix_iter = loaded_context->rc_prefixes.find(prefix2); @@ -422,88 +426,101 @@ readline_context::attempted_completion(const char* text, int start, int end) arg_possibilities = nullptr; } else if (proto[0] == "dirname") { shlex fn_lexer(rl_line_buffer, rl_point); - std::vector fn_list; + auto split_res = fn_lexer.split(scoped_resolver{&scope}); + if (split_res.isOk()) { + auto fn_list = split_res.unwrap(); + const auto& last_fn = fn_list.size() <= 1 + ? "" + : fn_list.back().se_value; - fn_lexer.split(fn_list, scoped_resolver{&scope}); + static std::set dir_name_set; - const auto& last_fn = fn_list.size() <= 1 ? "" - : fn_list.back(); - - static std::set dir_name_set; - - dir_name_set.clear(); - auto_mem completed_fn; - int fn_state = 0; + dir_name_set.clear(); + auto_mem completed_fn; + int fn_state = 0; - while ((completed_fn = rl_filename_completion_function( - last_fn.c_str(), fn_state)) - != nullptr) - { - dir_name_set.insert(completed_fn.in()); - fn_state += 1; + while ((completed_fn = rl_filename_completion_function( + last_fn.c_str(), fn_state)) + != nullptr) + { + dir_name_set.insert(completed_fn.in()); + fn_state += 1; + } + arg_possibilities = &dir_name_set; + arg_needs_shlex = true; + } else { + arg_possibilities = nullptr; } - arg_possibilities = &dir_name_set; - arg_needs_shlex = true; } else if (proto[0] == "filename") { shlex fn_lexer(rl_line_buffer, rl_point); - std::vector fn_list; int found = 0; - fn_lexer.split(fn_list, scoped_resolver{&scope}); - - const auto& last_fn = fn_list.size() <= 1 ? "" - : fn_list.back(); - - if (last_fn.find(':') != std::string::npos) { - auto rp_iter = loaded_context->rc_possibilities.find( - "remote-path"); - if (rp_iter != loaded_context->rc_possibilities.end()) { - for (const auto& poss : rp_iter->second) { - if (startswith(poss, last_fn.c_str())) { - found += 1; + auto split_res = fn_lexer.split(scoped_resolver{&scope}); + if (split_res.isOk()) { + auto fn_list = split_res.unwrap(); + const auto& last_fn = fn_list.size() <= 1 + ? "" + : fn_list.back().se_value; + + if (last_fn.find(':') != std::string::npos) { + auto rp_iter + = loaded_context->rc_possibilities.find( + "remote-path"); + if (rp_iter + != loaded_context->rc_possibilities.end()) + { + for (const auto& poss : rp_iter->second) { + if (startswith(poss, last_fn.c_str())) { + found += 1; + } + } + if (found) { + arg_possibilities = &rp_iter->second; + arg_needs_shlex = false; } } - if (found) { - arg_possibilities = &rp_iter->second; - arg_needs_shlex = false; + if (!found + || (endswith(last_fn, "/") && found == 1)) + { + char msg[2048]; + + snprintf( + msg, sizeof(msg), "\t:%s", last_fn.c_str()); + sendstring(child_this->rc_command_pipe[1], + msg, + strlen(msg)); } } - if (!found || (endswith(last_fn, "/") && found == 1)) { - char msg[2048]; - - snprintf( - msg, sizeof(msg), "\t:%s", last_fn.c_str()); - sendstring(child_this->rc_command_pipe[1], - msg, - strlen(msg)); - } - } - if (!found) { - static std::set file_name_set; - - file_name_set.clear(); - auto_mem completed_fn; - int fn_state = 0; - auto recent_netlocs_iter - = loaded_context->rc_possibilities.find( - "recent-netlocs"); - - if (recent_netlocs_iter - != loaded_context->rc_possibilities.end()) - { - file_name_set.insert( - recent_netlocs_iter->second.begin(), - recent_netlocs_iter->second.end()); - } - while ((completed_fn = rl_filename_completion_function( - last_fn.c_str(), fn_state)) - != nullptr) - { - file_name_set.insert(completed_fn.in()); - fn_state += 1; + if (!found) { + static std::set file_name_set; + + file_name_set.clear(); + auto_mem completed_fn; + int fn_state = 0; + auto recent_netlocs_iter + = loaded_context->rc_possibilities.find( + "recent-netlocs"); + + if (recent_netlocs_iter + != loaded_context->rc_possibilities.end()) + { + file_name_set.insert( + recent_netlocs_iter->second.begin(), + recent_netlocs_iter->second.end()); + } + while ( + (completed_fn = rl_filename_completion_function( + last_fn.c_str(), fn_state)) + != nullptr) + { + file_name_set.insert(completed_fn.in()); + fn_state += 1; + } + arg_possibilities = &file_name_set; + arg_needs_shlex = true; } - arg_possibilities = &file_name_set; - arg_needs_shlex = true; + } else { + arg_possibilities = nullptr; } } else { arg_possibilities diff --git a/src/readline_highlighters.cc b/src/readline_highlighters.cc index 58c537c5..e18278f6 100644 --- a/src/readline_highlighters.cc +++ b/src/readline_highlighters.cc @@ -306,63 +306,77 @@ readline_shlex_highlighter_int(attr_line_t& al, int x, line_range sub) { attr_line_builder alb(al); const auto& str = al.get_string(); - string_fragment cap; - shlex_token_t token; nonstd::optional quote_start; shlex lexer(string_fragment{al.al_string.data(), sub.lr_start, sub.lr_end}); + bool done = false; + + while (!done) { + auto tokenize_res = lexer.tokenize(); + if (tokenize_res.isErr()) { + auto te = tokenize_res.unwrapErr(); + + alb.overlay_attr(line_range(sub.lr_start + te.te_source.sf_begin, + sub.lr_start + te.te_source.sf_end), + VC_STYLE.value(text_attrs{A_REVERSE})); + alb.overlay_attr(line_range(sub.lr_start + te.te_source.sf_begin, + sub.lr_start + te.te_source.sf_end), + VC_ROLE.value(role_t::VCR_ERROR)); + return; + } - while (lexer.tokenize(cap, token)) { - switch (token) { - case shlex_token_t::ST_ERROR: - alb.overlay_attr(line_range(sub.lr_start + cap.sf_begin, - sub.lr_start + cap.sf_end), - VC_STYLE.value(text_attrs{A_REVERSE})); - alb.overlay_attr(line_range(sub.lr_start + cap.sf_begin, - sub.lr_start + cap.sf_end), - VC_ROLE.value(role_t::VCR_ERROR)); + auto token = tokenize_res.unwrap(); + switch (token.tr_token) { + case shlex_token_t::eof: + done = true; break; - case shlex_token_t::ST_TILDE: - case shlex_token_t::ST_ESCAPE: - alb.overlay_attr(line_range(sub.lr_start + cap.sf_begin, - sub.lr_start + cap.sf_end), - VC_ROLE.value(role_t::VCR_SYMBOL)); + case shlex_token_t::tilde: + case shlex_token_t::escape: + alb.overlay_attr( + line_range(sub.lr_start + token.tr_frag.sf_begin, + sub.lr_start + token.tr_frag.sf_end), + VC_ROLE.value(role_t::VCR_SYMBOL)); break; - case shlex_token_t::ST_DOUBLE_QUOTE_START: - case shlex_token_t::ST_SINGLE_QUOTE_START: - quote_start = sub.lr_start + cap.sf_begin; + case shlex_token_t::double_quote_start: + case shlex_token_t::single_quote_start: + quote_start = sub.lr_start + token.tr_frag.sf_begin; break; - case shlex_token_t::ST_DOUBLE_QUOTE_END: - case shlex_token_t::ST_SINGLE_QUOTE_END: + case shlex_token_t::double_quote_end: + case shlex_token_t::single_quote_end: alb.overlay_attr( - line_range(quote_start.value(), sub.lr_start + cap.sf_end), + line_range(quote_start.value(), + sub.lr_start + token.tr_frag.sf_end), VC_ROLE.value(role_t::VCR_STRING)); quote_start = nonstd::nullopt; break; - case shlex_token_t::ST_VARIABLE_REF: - case shlex_token_t::ST_QUOTED_VARIABLE_REF: { - int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1; - auto ident = str.substr(sub.lr_start + cap.sf_begin + 1 + extra, - cap.length() - 1 - extra * 2); + case shlex_token_t::variable_ref: + case shlex_token_t::quoted_variable_ref: { + int extra = token.tr_token == shlex_token_t::variable_ref ? 0 + : 1; + auto ident = str.substr( + sub.lr_start + token.tr_frag.sf_begin + 1 + extra, + token.tr_frag.length() - 1 - extra * 2); alb.overlay_attr( - line_range(sub.lr_start + cap.sf_begin, - sub.lr_start + cap.sf_begin + 1 + extra), + line_range( + sub.lr_start + token.tr_frag.sf_begin, + sub.lr_start + token.tr_frag.sf_begin + 1 + extra), VC_ROLE.value(role_t::VCR_SYMBOL)); alb.overlay_attr( - line_range(sub.lr_start + cap.sf_begin + 1 + extra, - sub.lr_start + cap.sf_end - extra), - VC_ROLE.value( - x == sub.lr_start + cap.sf_end - || (cap.sf_begin <= x && x < cap.sf_end) - ? role_t::VCR_SYMBOL - : role_t::VCR_IDENTIFIER)); + line_range( + sub.lr_start + token.tr_frag.sf_begin + 1 + extra, + sub.lr_start + token.tr_frag.sf_end - extra), + VC_ROLE.value(x == sub.lr_start + token.tr_frag.sf_end + || (token.tr_frag.sf_begin <= x + && x < token.tr_frag.sf_end) + ? role_t::VCR_SYMBOL + : role_t::VCR_IDENTIFIER)); if (extra) { alb.overlay_attr_for_char( - sub.lr_start + cap.sf_end - 1, + sub.lr_start + token.tr_frag.sf_end - 1, VC_ROLE.value(role_t::VCR_SYMBOL)); } break; } - case shlex_token_t::ST_WHITESPACE: + case shlex_token_t::whitespace: break; } } diff --git a/src/root-config.json b/src/root-config.json index 6b359a28..07910363 100644 --- a/src/root-config.json +++ b/src/root-config.json @@ -96,6 +96,9 @@ "docker": { "handler": "docker-url-handler" }, + "docker-compose": { + "handler": "docker-compose-url-handler" + }, "piper": { "handler": "piper-url-handler" } diff --git a/src/scripts/docker-compose-url-handler.lnav b/src/scripts/docker-compose-url-handler.lnav deleted file mode 100755 index 67ac37dc..00000000 --- a/src/scripts/docker-compose-url-handler.lnav +++ /dev/null @@ -1,15 +0,0 @@ -# -# @synopsis: docker-compose-url-handler -# @description: Internal script to handle opening docker-compose URLs -# - -;SELECT st_name FROM fstat('compose.yml') - UNION - SELECT st_name FROM fstat('compose.yaml') - UNION - SELECT st_name FROM fstat('docker-compose.yml') - UNION - SELECT st_name FROM fstat('docker-compose.yaml') - -;SELECT group_concat(compose_services.key, ' ') - FROM fstat($st_name), json_each(yaml_to_json(data), '$.services') as compose_services diff --git a/src/scripts/docker-url-handler.lnav b/src/scripts/docker-url-handler.lnav index d250b5c4..588cf62b 100755 --- a/src/scripts/docker-url-handler.lnav +++ b/src/scripts/docker-url-handler.lnav @@ -3,15 +3,29 @@ # @description: Internal script to handle opening docker URLs # -;SELECT CASE path - WHEN '/' THEN - 'docker logs -f ' || hostname - ELSE - 'docker exec ' || hostname || ' tail -n +0 -F "' || path || '"' - END AS cmd - FROM (SELECT - jget(url, '/host') AS hostname, - jget(url, '/path') AS path - FROM (SELECT parse_url($1) AS url)) +;SELECT jget(url, '/host') AS docker_hostname, + jget(url, '/path') AS docker_path + FROM (SELECT parse_url($1) AS url) -:sh eval $cmd +;SELECT CASE + $docker_hostname + WHEN 'compose' THEN ( + SELECT group_concat( + printf( + ':sh --name=%s docker compose logs --no-log-prefix -f %s', + compose_services.key, + compose_services.key + ), + char(10) + ) AS cmds + FROM fstat(substr($docker_path, 2)), + json_each(yaml_to_json(data), '$.services') as compose_services + ) + ELSE CASE + $docker_path + WHEN '/' THEN ':sh docker logs -f ' || $docker_hostname + ELSE ':sh docker exec ' || $docker_hostname || ' tail -n +0 -F "' || $docker_path || '"' + END + END AS cmds + +:eval ${cmds} diff --git a/src/scripts/pcap_log-converter.sh b/src/scripts/pcap_log-converter.sh index 94ef5379..80148e9c 100755 --- a/src/scripts/pcap_log-converter.sh +++ b/src/scripts/pcap_log-converter.sh @@ -1,7 +1,7 @@ #!/bin/bash # Check that tshark is installed and return a nice message. -if ! command -v tshark; then +if ! command -v tshark > /dev/null; then echo "pcap support requires 'tshark' v3+ to be installed" > /dev/stderr exit 1 fi diff --git a/src/scripts/scripts.am b/src/scripts/scripts.am index 267cbc4e..0c06ff2b 100644 --- a/src/scripts/scripts.am +++ b/src/scripts/scripts.am @@ -2,7 +2,6 @@ BUILTIN_LNAVSCRIPTS = \ $(srcdir)/scripts/dhclient-summary.lnav \ $(srcdir)/scripts/docker-url-handler.lnav \ - $(srcdir)/scripts/docker-compose-url-handler.lnav \ $(srcdir)/scripts/lnav-pop-view.lnav \ $(srcdir)/scripts/partition-by-boot.lnav \ $(srcdir)/scripts/piper-url-handler.lnav \ diff --git a/src/shlex.cc b/src/shlex.cc index 4bdbdc3a..1f2f28f1 100644 --- a/src/shlex.cc +++ b/src/shlex.cc @@ -36,40 +36,59 @@ #include "config.h" #include "shlex.hh" -bool -shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) +using namespace lnav::roles::literals; + +attr_line_t +shlex::to_attr_line(const shlex::tokenize_error_t& te) const { + return attr_line_t() + .append(string_fragment::from_bytes(this->s_str, this->s_len)) + .append("\n") + .pad_to(te.te_source.sf_begin) + .append("^"_snippet_border); +} + +Result +shlex::tokenize() +{ + tokenize_result_t retval; + + retval.tr_frag.sf_string = this->s_str; while (this->s_index < this->s_len) { switch (this->s_str[this->s_index]) { case '\\': - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; if (this->s_index + 1 < this->s_len) { - token_out = shlex_token_t::ST_ESCAPE; + retval.tr_token = shlex_token_t::escape; this->s_index += 2; - cap_out.sf_end = this->s_index; + retval.tr_frag.sf_end = this->s_index; } else { this->s_index += 1; - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_ERROR; + retval.tr_frag.sf_end = this->s_index; + + return Err(tokenize_error_t{ + "invalid escape", + retval.tr_frag, + }); } - return true; + return Ok(retval); case '\"': if (!this->s_ignore_quotes) { switch (this->s_state) { case state_t::STATE_NORMAL: - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; this->s_index += 1; - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_DOUBLE_QUOTE_START; + retval.tr_frag.sf_end = this->s_index; + retval.tr_token = shlex_token_t::double_quote_start; this->s_state = state_t::STATE_IN_DOUBLE_QUOTE; - return true; + return Ok(retval); case state_t::STATE_IN_DOUBLE_QUOTE: - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; this->s_index += 1; - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_DOUBLE_QUOTE_END; + retval.tr_frag.sf_end = this->s_index; + retval.tr_token = shlex_token_t::double_quote_end; this->s_state = state_t::STATE_NORMAL; - return true; + return Ok(retval); default: break; } @@ -79,19 +98,19 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) if (!this->s_ignore_quotes) { switch (this->s_state) { case state_t::STATE_NORMAL: - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; this->s_index += 1; - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_SINGLE_QUOTE_START; + retval.tr_frag.sf_end = this->s_index; + retval.tr_token = shlex_token_t::single_quote_start; this->s_state = state_t::STATE_IN_SINGLE_QUOTE; - return true; + return Ok(retval); case state_t::STATE_IN_SINGLE_QUOTE: - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; this->s_index += 1; - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_SINGLE_QUOTE_END; + retval.tr_frag.sf_end = this->s_index; + retval.tr_token = shlex_token_t::single_quote_end; this->s_state = state_t::STATE_NORMAL; - return true; + return Ok(retval); default: break; } @@ -100,9 +119,10 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) case '$': switch (this->s_state) { case state_t::STATE_NORMAL: - case state_t::STATE_IN_DOUBLE_QUOTE: - this->scan_variable_ref(cap_out, token_out); - return true; + case state_t::STATE_IN_DOUBLE_QUOTE: { + auto rc = TRY(this->scan_variable_ref()); + return Ok(rc); + } default: break; } @@ -110,7 +130,7 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) case '~': switch (this->s_state) { case state_t::STATE_NORMAL: - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; this->s_index += 1; while (this->s_index < this->s_len && (isalnum(this->s_str[this->s_index]) @@ -119,9 +139,9 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) { this->s_index += 1; } - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_TILDE; - return true; + retval.tr_frag.sf_end = this->s_index; + retval.tr_token = shlex_token_t::tilde; + return Ok(retval); default: break; } @@ -130,13 +150,13 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) case '\t': switch (this->s_state) { case state_t::STATE_NORMAL: - cap_out.sf_begin = this->s_index; + retval.tr_frag.sf_begin = this->s_index; while (isspace(this->s_str[this->s_index])) { this->s_index += 1; } - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_WHITESPACE; - return true; + retval.tr_frag.sf_end = this->s_index; + retval.tr_token = shlex_token_t::whitespace; + return Ok(retval); default: break; } @@ -148,29 +168,47 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out) this->s_index += 1; } - return false; + if (this->s_state != state_t::STATE_NORMAL) { + retval.tr_frag.sf_begin = this->s_index; + retval.tr_frag.sf_end = this->s_len; + return Err(tokenize_error_t{ + "non-terminated string", + retval.tr_frag, + }); + } + + retval.tr_frag.sf_begin = this->s_len; + retval.tr_frag.sf_end = this->s_len; + retval.tr_token = shlex_token_t::eof; + return Ok(retval); } -void -shlex::scan_variable_ref(string_fragment& cap_out, shlex_token_t& token_out) +Result +shlex::scan_variable_ref() { - cap_out.sf_begin = this->s_index; + tokenize_result_t retval; + + retval.tr_frag.sf_string = this->s_str; + + retval.tr_frag.sf_begin = this->s_index; this->s_index += 1; if (this->s_index >= this->s_len) { - cap_out.sf_end = this->s_index; - token_out = shlex_token_t::ST_ERROR; - return; + retval.tr_frag.sf_end = this->s_index; + return Err(tokenize_error_t{ + "invalid variable reference", + retval.tr_frag, + }); } if (this->s_str[this->s_index] == '{') { - token_out = shlex_token_t::ST_QUOTED_VARIABLE_REF; + retval.tr_token = shlex_token_t::quoted_variable_ref; this->s_index += 1; } else { - token_out = shlex_token_t::ST_VARIABLE_REF; + retval.tr_token = shlex_token_t::variable_ref; } while (this->s_index < this->s_len) { - if (token_out == shlex_token_t::ST_VARIABLE_REF) { + if (retval.tr_token == shlex_token_t::variable_ref) { if (isalnum(this->s_str[this->s_index]) || this->s_str[this->s_index] == '#' || this->s_str[this->s_index] == '_') @@ -188,14 +226,19 @@ shlex::scan_variable_ref(string_fragment& cap_out, shlex_token_t& token_out) } } - cap_out.sf_end = this->s_index; - if (token_out == shlex_token_t::ST_QUOTED_VARIABLE_REF + retval.tr_frag.sf_end = this->s_index; + if (retval.tr_token == shlex_token_t::quoted_variable_ref && this->s_str[this->s_index - 1] != '}') { - cap_out.sf_begin += 1; - cap_out.sf_end = cap_out.sf_begin + 1; - token_out = shlex_token_t::ST_ERROR; + retval.tr_frag.sf_begin += 1; + retval.tr_frag.sf_end = retval.tr_frag.sf_begin + 1; + return Err(tokenize_error_t{ + "missing closing curly-brace in variable reference", + retval.tr_frag, + }); } + + return Ok(retval); } void @@ -222,27 +265,36 @@ shlex::eval(std::string& result, const scoped_resolver& vars) { result.clear(); - string_fragment cap; - shlex_token_t token; int last_index = 0; + bool done = false; + + while (!done) { + auto tokenize_res = this->tokenize(); + if (tokenize_res.isErr()) { + return false; + } + auto token = tokenize_res.unwrap(); - while (this->tokenize(cap, token)) { - result.append(&this->s_str[last_index], cap.sf_begin - last_index); - switch (token) { - case shlex_token_t::ST_ERROR: - return false; - case shlex_token_t::ST_ESCAPE: - result.append(1, this->s_str[cap.sf_begin + 1]); + result.append(&this->s_str[last_index], + token.tr_frag.sf_begin - last_index); + switch (token.tr_token) { + case shlex_token_t::eof: + done = true; break; - case shlex_token_t::ST_WHITESPACE: - result.append(&this->s_str[cap.sf_begin], cap.length()); + case shlex_token_t::escape: + result.append(1, this->s_str[token.tr_frag.sf_begin + 1]); break; - case shlex_token_t::ST_VARIABLE_REF: - case shlex_token_t::ST_QUOTED_VARIABLE_REF: { - int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1; + case shlex_token_t::whitespace: + result.append(&this->s_str[token.tr_frag.sf_begin], + token.tr_frag.length()); + break; + case shlex_token_t::variable_ref: + case shlex_token_t::quoted_variable_ref: { + int extra = token.tr_token == shlex_token_t::variable_ref ? 0 + : 1; const std::string var_name( - &this->s_str[cap.sf_begin + 1 + extra], - cap.length() - 1 - extra * 2); + &this->s_str[token.tr_frag.sf_begin + 1 + extra], + token.tr_frag.length() - 1 - extra * 2); auto local_var = vars.find(var_name); const char* var_value = getenv(var_name.c_str()); @@ -253,21 +305,21 @@ shlex::eval(std::string& result, const scoped_resolver& vars) } break; } - case shlex_token_t::ST_TILDE: - this->resolve_home_dir(result, cap); + case shlex_token_t::tilde: + this->resolve_home_dir(result, token.tr_frag); break; - case shlex_token_t::ST_DOUBLE_QUOTE_START: - case shlex_token_t::ST_DOUBLE_QUOTE_END: + case shlex_token_t::double_quote_start: + case shlex_token_t::double_quote_end: result.append("\""); break; - case shlex_token_t::ST_SINGLE_QUOTE_START: - case shlex_token_t::ST_SINGLE_QUOTE_END: + case shlex_token_t::single_quote_start: + case shlex_token_t::single_quote_end: result.append("'"); break; default: break; } - last_index = cap.sf_end; + last_index = token.tr_frag.sf_end; } result.append(&this->s_str[last_index], this->s_len - last_index); @@ -275,66 +327,89 @@ shlex::eval(std::string& result, const scoped_resolver& vars) return true; } -bool -shlex::split(std::vector& result, const scoped_resolver& vars) +Result, shlex::tokenize_error_t> +shlex::split(const scoped_resolver& vars) { - result.clear(); - - string_fragment cap; - shlex_token_t token; + std::vector retval; int last_index = 0; bool start_new = true; + bool done = false; - while (isspace(this->s_str[this->s_index])) { + while (this->s_index < this->s_len && isspace(this->s_str[this->s_index])) { this->s_index += 1; } - while (this->tokenize(cap, token)) { + if (this->s_index == this->s_len) { + return Ok(retval); + } + while (!done) { + auto tokenize_res = TRY(this->tokenize()); + if (start_new) { - result.emplace_back(""); + retval.emplace_back(split_element_t{ + string_fragment::from_byte_range( + this->s_str, last_index, tokenize_res.tr_frag.sf_begin), + "", + }); start_new = false; + } else if (tokenize_res.tr_token != shlex_token_t::whitespace) { + retval.back().se_origin.sf_end = tokenize_res.tr_frag.sf_end; + } else { + retval.back().se_origin.sf_end = tokenize_res.tr_frag.sf_begin; } - result.back().append(&this->s_str[last_index], - cap.sf_begin - last_index); - switch (token) { - case shlex_token_t::ST_ERROR: - return false; - case shlex_token_t::ST_ESCAPE: - result.back().append(1, this->s_str[cap.sf_begin + 1]); + retval.back().se_value.append( + &this->s_str[last_index], + tokenize_res.tr_frag.sf_begin - last_index); + switch (tokenize_res.tr_token) { + case shlex_token_t::eof: + done = true; + break; + case shlex_token_t::escape: + retval.back().se_value.append( + 1, this->s_str[tokenize_res.tr_frag.sf_begin + 1]); break; - case shlex_token_t::ST_WHITESPACE: + case shlex_token_t::whitespace: start_new = true; break; - case shlex_token_t::ST_VARIABLE_REF: - case shlex_token_t::ST_QUOTED_VARIABLE_REF: { - int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1; - std::string var_name(&this->s_str[cap.sf_begin + 1 + extra], - cap.length() - 1 - extra * 2); + case shlex_token_t::variable_ref: + case shlex_token_t::quoted_variable_ref: { + int extra = tokenize_res.tr_token == shlex_token_t::variable_ref + ? 0 + : 1; + std::string var_name( + &this->s_str[tokenize_res.tr_frag.sf_begin + 1 + extra], + tokenize_res.tr_frag.length() - 1 - extra * 2); auto local_var = vars.find(var_name); const char* var_value = getenv(var_name.c_str()); if (local_var != vars.end()) { - result.back().append(fmt::to_string(local_var->second)); + retval.back().se_value.append( + fmt::to_string(local_var->second)); } else if (var_value != nullptr) { - result.back().append(var_value); + retval.back().se_value.append(var_value); } break; } - case shlex_token_t::ST_TILDE: - this->resolve_home_dir(result.back(), cap); + case shlex_token_t::tilde: + this->resolve_home_dir(retval.back().se_value, + tokenize_res.tr_frag); break; default: break; } - last_index = cap.sf_end; + last_index = tokenize_res.tr_frag.sf_end; } if (last_index < this->s_len) { - if (start_new || result.empty()) { - result.emplace_back(""); + if (start_new || retval.empty()) { + retval.emplace_back(split_element_t{ + string_fragment::from_byte_range( + this->s_str, last_index, this->s_len), + "", + }); } - result.back().append(&this->s_str[last_index], - this->s_len - last_index); + retval.back().se_value.append(&this->s_str[last_index], + this->s_len - last_index); } - return true; + return Ok(retval); } diff --git a/src/shlex.hh b/src/shlex.hh index a4644537..42b53bcc 100644 --- a/src/shlex.hh +++ b/src/shlex.hh @@ -32,32 +32,34 @@ #ifndef LNAV_SHLEX_HH_H #define LNAV_SHLEX_HH_H +#include #include #include #include #include +#include "base/attr_line.hh" #include "base/intern_string.hh" #include "base/opt_util.hh" #include "shlex.resolver.hh" enum class shlex_token_t { - ST_ERROR, - ST_WHITESPACE, - ST_ESCAPE, - ST_DOUBLE_QUOTE_START, - ST_DOUBLE_QUOTE_END, - ST_SINGLE_QUOTE_START, - ST_SINGLE_QUOTE_END, - ST_VARIABLE_REF, - ST_QUOTED_VARIABLE_REF, - ST_TILDE, + eof, + whitespace, + escape, + double_quote_start, + double_quote_end, + single_quote_start, + single_quote_end, + variable_ref, + quoted_variable_ref, + tilde, }; class shlex { public: - shlex(const char* str, size_t len) : s_str(str), s_len(len){}; + shlex(const char* str, size_t len) : s_str(str), s_len(len) {} explicit shlex(const string_fragment& sf) : s_str(sf.data()), s_len(sf.length()) @@ -65,7 +67,9 @@ public: } explicit shlex(const std::string& str) - : s_str(str.c_str()), s_len(str.size()){}; + : s_str(str.c_str()), s_len(str.size()) + { + } shlex& with_ignore_quotes(bool val) { @@ -73,11 +77,27 @@ public: return *this; } - bool tokenize(string_fragment& cap_out, shlex_token_t& token_out); + struct tokenize_result_t { + shlex_token_t tr_token; + string_fragment tr_frag; + }; + + struct tokenize_error_t { + const char* te_msg{nullptr}; + string_fragment te_source; + }; + + Result tokenize(); bool eval(std::string& result, const scoped_resolver& vars); - bool split(std::vector& result, const scoped_resolver& vars); + struct split_element_t { + string_fragment se_origin; + std::string se_value; + }; + + Result, tokenize_error_t> split( + const scoped_resolver& vars); void reset() { @@ -85,10 +105,12 @@ public: this->s_state = state_t::STATE_NORMAL; } - void scan_variable_ref(string_fragment& cap_out, shlex_token_t& token_out); + Result scan_variable_ref(); void resolve_home_dir(std::string& result, string_fragment cap) const; + attr_line_t to_attr_line(const tokenize_error_t& te) const; + enum class state_t { STATE_NORMAL, STATE_IN_DOUBLE_QUOTE, diff --git a/src/sql_commands.cc b/src/sql_commands.cc index e1453316..bbb3b88f 100644 --- a/src/sql_commands.cc +++ b/src/sql_commands.cc @@ -30,6 +30,7 @@ #include "base/auto_mem.hh" #include "base/fs_util.hh" #include "base/injector.bind.hh" +#include "base/itertools.hh" #include "base/lnav_log.hh" #include "bound_tags.hh" #include "command_executor.hh" @@ -88,6 +89,7 @@ sql_cmd_read(exec_context& ec, std::string cmdline, std::vector& args) { + static const intern_string_t SRC = intern_string::lookup("cmdline"); static auto& lnav_db = injector::get(); static auto& lnav_flags = injector::get(); @@ -102,13 +104,23 @@ sql_cmd_read(exec_context& ec, return ec.make_error("{} -- unavailable in secure mode", args[0]); } - std::vector split_args; shlex lexer(cmdline); - if (!lexer.split(split_args, ec.create_resolver())) { - return ec.make_error("unable to parse arguments"); + auto split_args_res = lexer.split(ec.create_resolver()); + if (split_args_res.isErr()) { + auto split_err = split_args_res.unwrapErr(); + auto um + = lnav::console::user_message::error("unable to parse file name") + .with_reason(split_err.te_msg) + .with_snippet(lnav::console::snippet::from( + SRC, lexer.to_attr_line(split_err))); + + return Err(um); } + auto split_args = split_args_res.unwrap() + | lnav::itertools::map([](const auto& elem) { return elem.se_value; }); + for (size_t lpc = 1; lpc < split_args.size(); lpc++) { auto read_res = lnav::filesystem::read_file(split_args[lpc]); diff --git a/src/textfile_sub_source.cc b/src/textfile_sub_source.cc index 2f78e3cc..004b02ae 100644 --- a/src/textfile_sub_source.cc +++ b/src/textfile_sub_source.cc @@ -636,6 +636,11 @@ textfile_sub_source::rescan_files( continue; } + if (!this->tss_completed_last_scan && lf->size() > 0) { + ++iter; + continue; + } + try { const auto& st = lf->get_stat(); uint32_t old_size = lf->size(); @@ -650,11 +655,6 @@ textfile_sub_source::rescan_files( continue; } - if (!this->tss_completed_last_scan && lf->size() > 0) { - ++iter; - continue; - } - bool new_data = false; switch (new_text_data) { case logfile::rebuild_result_t::NEW_LINES: diff --git a/src/time_formats.am b/src/time_formats.am index 780c9d5c..7147a0c8 100644 --- a/src/time_formats.am +++ b/src/time_formats.am @@ -68,6 +68,7 @@ TIME_FORMATS = \ "%m/%e/%Y %l:%M:%S%p" \ "%m/%d/%y %H:%M:%S" \ "%m/%d/%Y %H:%M:%S" \ + "%d/%b/%Y %H:%M:%S" \ "%d/%b/%y %H:%M:%S" \ "%m%d %H:%M:%S" \ "%Y%m%d %H:%M:%S" \ diff --git a/src/top_status_source.cc b/src/top_status_source.cc index 4be57908..71001942 100644 --- a/src/top_status_source.cc +++ b/src/top_status_source.cc @@ -64,12 +64,13 @@ top_status_source::update_time(const timeval& current_time) { auto& sf = this->tss_fields[TSF_TIME]; char buffer[32]; + tm current_tm; buffer[0] = ' '; strftime(&buffer[1], sizeof(buffer) - 1, this->tss_config.tssc_clock_format.c_str(), - localtime(¤t_time.tv_sec)); + localtime_r(¤t_time.tv_sec, ¤t_tm)); sf.set_value(buffer); } diff --git a/test/drive_shlexer.cc b/test/drive_shlexer.cc index 42ad1b9d..ffbfc32c 100644 --- a/test/drive_shlexer.cc +++ b/test/drive_shlexer.cc @@ -35,7 +35,7 @@ using namespace std; const char* ST_TOKEN_NAMES[] = { - "err", + "eof", "wsp", "esc", "dst", @@ -47,6 +47,22 @@ const char* ST_TOKEN_NAMES[] = { "til", }; +static void +put_underline(FILE* file, string_fragment frag) +{ + for (int lpc = 0; lpc < frag.sf_end; lpc++) { + if (lpc == frag.sf_begin) { + fputc('^', stdout); + } else if (lpc == (frag.sf_end - 1)) { + fputc('^', stdout); + } else if (lpc > frag.sf_begin) { + fputc('-', stdout); + } else { + fputc(' ', stdout); + } + } +} + int main(int argc, char* argv[]) { @@ -56,25 +72,26 @@ main(int argc, char* argv[]) } shlex lexer(argv[1], strlen(argv[1])); - string_fragment cap; - shlex_token_t token; + bool done = false; printf(" %s\n", argv[1]); - while (lexer.tokenize(cap, token)) { - int lpc; + while (!done) { + auto tokenize_res = lexer.tokenize(); + if (tokenize_res.isErr()) { + auto te = tokenize_res.unwrapErr(); + + printf("err "); + put_underline(stdout, te.te_source); + printf(" -- %s\n", te.te_msg); + break; + } - printf("%s ", ST_TOKEN_NAMES[(int) token]); - for (lpc = 0; lpc < cap.sf_end; lpc++) { - if (lpc == cap.sf_begin) { - fputc('^', stdout); - } else if (lpc == (cap.sf_end - 1)) { - fputc('^', stdout); - } else if (lpc > cap.sf_begin) { - fputc('-', stdout); - } else { - fputc(' ', stdout); - } + auto tr = tokenize_res.unwrap(); + if (tr.tr_token == shlex_token_t::eof) { + done = true; } + printf("%s ", ST_TOKEN_NAMES[(int) tr.tr_token]); + put_underline(stdout, tr.tr_frag); printf("\n"); } @@ -85,11 +102,14 @@ main(int argc, char* argv[]) printf("eval -- %s\n", result.c_str()); } lexer.reset(); - std::vector sresult; - if (lexer.split(sresult, scoped_resolver{&vars})) { + auto split_res = lexer.split(scoped_resolver{&vars}); + if (split_res.isOk()) { + auto sresult = split_res.unwrap(); printf("split:\n"); for (size_t lpc = 0; lpc < sresult.size(); lpc++) { - printf(" %zu -- %s\n", lpc, sresult[lpc].c_str()); + printf("% 3zu ", lpc); + put_underline(stdout, sresult[lpc].se_origin); + printf(" -- %s\n", sresult[lpc].se_value.c_str()); } } diff --git a/test/expected/expected.am b/test/expected/expected.am index c7a4cf1f..05ad065b 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -206,6 +206,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_cmds.sh_ca66660c973f76a3c2a147c7f5035bcb4e8a8bbc.out \ $(srcdir)/%reldir%/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.err \ $(srcdir)/%reldir%/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out \ + $(srcdir)/%reldir%/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err \ + $(srcdir)/%reldir%/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.out \ $(srcdir)/%reldir%/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.err \ $(srcdir)/%reldir%/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.out \ $(srcdir)/%reldir%/test_cmds.sh_d3b69abdfb39e4bfa5828c2f9593e2b2b7ed4d5d.err \ diff --git a/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out b/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out index e0f69bb2..37708715 100644 --- a/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out +++ b/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out @@ -105,6 +105,9 @@ "docker": { "handler": "docker-url-handler" }, + "docker-compose": { + "handler": "docker-compose-url-handler" + }, "hw": { "handler": "hw-url-handler" }, diff --git a/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out b/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out index 7e95380c..2bc893da 100644 --- a/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out +++ b/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out @@ -48,9 +48,10 @@ /tuning/remote/ssh/config/ConnectTimeout -> root-config.json:35 /tuning/remote/ssh/start-command -> root-config.json:37 /tuning/remote/ssh/transfer-command -> root-config.json:38 +/tuning/url-scheme/docker-compose/handler -> root-config.json:100 /tuning/url-scheme/docker/handler -> root-config.json:97 /tuning/url-scheme/hw/handler -> {test_dir}/configs/installed/hw-url-handler.json:6 -/tuning/url-scheme/piper/handler -> root-config.json:100 +/tuning/url-scheme/piper/handler -> root-config.json:103 /ui/clock-format -> root-config.json:4 /ui/default-colors -> root-config.json:6 /ui/dim-text -> root-config.json:5 diff --git a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out index 5a47da2b..dd945991 100644 --- a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out +++ b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out @@ -1444,11 +1444,13 @@ For support questions, email: -:sh cmdline +:sh --name= cmdline ══════════════════════════════════════════════════════════════════════ Execute the given command-line and display the captured output -Parameter - cmdline The command-line to execute. +Parameters + --name= The name to give to the captured + output + cmdline The command-line to execute. See Also :alt-msg, :cd, :echo, :eval, :export-session-to, :rebuild, :redirect-to, :write-csv-to, :write-json-to, :write-jsonlines-to, diff --git a/test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err b/test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err new file mode 100644 index 00000000..a55e4f8e --- /dev/null +++ b/test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err @@ -0,0 +1,6 @@ +✘ error: invalid epoch time: 16120724091612072409 -- Value too large to be stored in data type + --> command-option:1 + | :unix-time 16120724091612072409  + = help: :unix-time seconds + ══════════════════════════════════════════════════════════════════════ + Convert epoch time to a human-readable form diff --git a/test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.out b/test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.out new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.out b/test/expected/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.out index fd072bcd..0dd40801 100644 --- a/test/expected/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.out +++ b/test/expected/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.out @@ -1,7 +1,8 @@ ~ foo til ^ wsp ^ +eof ^ eval -- ../test foo split: - 0 -- ../test - 1 -- foo + 0 ^ -- ../test + 1 ^-^ -- foo diff --git a/test/expected/test_shlexer.sh_2781f5dd570580cbe746ad91b58a28b8371283b3.out b/test/expected/test_shlexer.sh_2781f5dd570580cbe746ad91b58a28b8371283b3.out index b7ba9e89..2081fa3f 100644 --- a/test/expected/test_shlexer.sh_2781f5dd570580cbe746ad91b58a28b8371283b3.out +++ b/test/expected/test_shlexer.sh_2781f5dd570580cbe746ad91b58a28b8371283b3.out @@ -1,7 +1,8 @@ ~nonexistent/bar baz til ^----------^ wsp ^ +eof ^ eval -- ~nonexistent/bar baz split: - 0 -- ~nonexistent/bar - 1 -- baz + 0 ^--------------^ -- ~nonexistent/bar + 1 ^-^ -- baz diff --git a/test/expected/test_shlexer.sh_2af44d06fc137a77bc230be86376ccad23a2806b.out b/test/expected/test_shlexer.sh_2af44d06fc137a77bc230be86376ccad23a2806b.out index 85cca78c..8d6bcc08 100644 --- a/test/expected/test_shlexer.sh_2af44d06fc137a77bc230be86376ccad23a2806b.out +++ b/test/expected/test_shlexer.sh_2af44d06fc137a77bc230be86376ccad23a2806b.out @@ -1,2 +1,2 @@ \ -err ^ +err ^ -- invalid escape diff --git a/test/expected/test_shlexer.sh_6858e530a8ecb77cbaec1a7507768dd5a1942ac9.out b/test/expected/test_shlexer.sh_6858e530a8ecb77cbaec1a7507768dd5a1942ac9.out index f677b170..6869e6dc 100644 --- a/test/expected/test_shlexer.sh_6858e530a8ecb77cbaec1a7507768dd5a1942ac9.out +++ b/test/expected/test_shlexer.sh_6858e530a8ecb77cbaec1a7507768dd5a1942ac9.out @@ -1,5 +1,6 @@ ${FOO} qrf ^----^ +eof ^ eval -- bar split: - 0 -- bar + 0 ^----^ -- bar diff --git a/test/expected/test_shlexer.sh_7f31e16ea2469da7a4328c93c7bcc8e109f84d2f.out b/test/expected/test_shlexer.sh_7f31e16ea2469da7a4328c93c7bcc8e109f84d2f.out index 630eb1c5..46009ecd 100644 --- a/test/expected/test_shlexer.sh_7f31e16ea2469da7a4328c93c7bcc8e109f84d2f.out +++ b/test/expected/test_shlexer.sh_7f31e16ea2469da7a4328c93c7bcc8e109f84d2f.out @@ -2,6 +2,7 @@ dst ^ qrf ^----^ den ^ +eof ^ eval -- "abc xyz 123" split: - 0 -- abc xyz 123 + 0 ^--------------^ -- abc xyz 123 diff --git a/test/expected/test_shlexer.sh_8aeebcdef56edd783579eaaddaff7c5cc127bb86.out b/test/expected/test_shlexer.sh_8aeebcdef56edd783579eaaddaff7c5cc127bb86.out index 759e75bf..5eda1e56 100644 --- a/test/expected/test_shlexer.sh_8aeebcdef56edd783579eaaddaff7c5cc127bb86.out +++ b/test/expected/test_shlexer.sh_8aeebcdef56edd783579eaaddaff7c5cc127bb86.out @@ -1,6 +1,7 @@ 'abc $DEF 123' sst ^ sen ^ +eof ^ eval -- 'abc $DEF 123' split: - 0 -- abc $DEF 123 + 0 ^------------^ -- abc $DEF 123 diff --git a/test/expected/test_shlexer.sh_8e9addb0e5b6f4254d81dd89ecf12783109644bb.out b/test/expected/test_shlexer.sh_8e9addb0e5b6f4254d81dd89ecf12783109644bb.out index 2cbee1a2..6ea7ec2b 100644 --- a/test/expected/test_shlexer.sh_8e9addb0e5b6f4254d81dd89ecf12783109644bb.out +++ b/test/expected/test_shlexer.sh_8e9addb0e5b6f4254d81dd89ecf12783109644bb.out @@ -2,6 +2,7 @@ dst ^ ref ^--^ den ^ +eof ^ eval -- "abc xyz 123" split: - 0 -- abc xyz 123 + 0 ^------------^ -- abc xyz 123 diff --git a/test/expected/test_shlexer.sh_90961e6728e96d0a44535a6c9907cc990c10316c.out b/test/expected/test_shlexer.sh_90961e6728e96d0a44535a6c9907cc990c10316c.out index 9a2b2a74..1fc686bf 100644 --- a/test/expected/test_shlexer.sh_90961e6728e96d0a44535a6c9907cc990c10316c.out +++ b/test/expected/test_shlexer.sh_90961e6728e96d0a44535a6c9907cc990c10316c.out @@ -1,6 +1,7 @@ "def" dst ^ den ^ +eof ^ eval -- "def" split: - 0 -- def + 0 ^---^ -- def diff --git a/test/expected/test_shlexer.sh_95c4e861804a5434900fdb4d67b149d1baa2edf4.out b/test/expected/test_shlexer.sh_95c4e861804a5434900fdb4d67b149d1baa2edf4.out index 30a7791d..465073bf 100644 --- a/test/expected/test_shlexer.sh_95c4e861804a5434900fdb4d67b149d1baa2edf4.out +++ b/test/expected/test_shlexer.sh_95c4e861804a5434900fdb4d67b149d1baa2edf4.out @@ -1,5 +1,6 @@ $FOO ref ^--^ +eof ^ eval -- bar split: - 0 -- bar + 0 ^--^ -- bar diff --git a/test/expected/test_shlexer.sh_d7fe5f6b8fc9ba00539fad0fa0bfb08319d8b04b.out b/test/expected/test_shlexer.sh_d7fe5f6b8fc9ba00539fad0fa0bfb08319d8b04b.out index a2ae7ff1..8feba4e0 100644 --- a/test/expected/test_shlexer.sh_d7fe5f6b8fc9ba00539fad0fa0bfb08319d8b04b.out +++ b/test/expected/test_shlexer.sh_d7fe5f6b8fc9ba00539fad0fa0bfb08319d8b04b.out @@ -1,6 +1,7 @@ 'abc' sst ^ sen ^ +eof ^ eval -- 'abc' split: - 0 -- abc + 0 ^---^ -- abc diff --git a/test/expected/test_shlexer.sh_d9d46422a913e3a06ddbd262933ef5352c30e68f.out b/test/expected/test_shlexer.sh_d9d46422a913e3a06ddbd262933ef5352c30e68f.out index def9d5c3..5f35534d 100644 --- a/test/expected/test_shlexer.sh_d9d46422a913e3a06ddbd262933ef5352c30e68f.out +++ b/test/expected/test_shlexer.sh_d9d46422a913e3a06ddbd262933ef5352c30e68f.out @@ -2,8 +2,9 @@ wsp ^ ref ^--^ wsp ^^ +eof ^ eval -- abc xyz 123 split: - 0 -- abc - 1 -- xyz - 2 -- 123 + 0 ^-^ -- abc + 1 ^--^ -- xyz + 2 ^-^ -- 123 diff --git a/test/expected/test_shlexer.sh_e0599f0b53d1bd27af767113853f8e84291f137d.out b/test/expected/test_shlexer.sh_e0599f0b53d1bd27af767113853f8e84291f137d.out index 9b849990..eece1b63 100644 --- a/test/expected/test_shlexer.sh_e0599f0b53d1bd27af767113853f8e84291f137d.out +++ b/test/expected/test_shlexer.sh_e0599f0b53d1bd27af767113853f8e84291f137d.out @@ -1,6 +1,7 @@ '"' sst ^ sen ^ +eof ^ eval -- '"' split: - 0 -- " + 0 ^-^ -- " diff --git a/test/expected/test_shlexer.sh_e8fa2239ab17e7563d0c524f5400a79d6ff8bfda.out b/test/expected/test_shlexer.sh_e8fa2239ab17e7563d0c524f5400a79d6ff8bfda.out index a668d4db..41a0f641 100644 --- a/test/expected/test_shlexer.sh_e8fa2239ab17e7563d0c524f5400a79d6ff8bfda.out +++ b/test/expected/test_shlexer.sh_e8fa2239ab17e7563d0c524f5400a79d6ff8bfda.out @@ -1,6 +1,7 @@ "'" dst ^ den ^ +eof ^ eval -- "'" split: - 0 -- ' + 0 ^-^ -- ' diff --git a/test/formats/jsontest/lnav-logstash.json b/test/formats/jsontest/lnav-logstash.json index 27f92398..b821168b 100644 --- a/test/formats/jsontest/lnav-logstash.json +++ b/test/formats/jsontest/lnav-logstash.json @@ -7,40 +7,48 @@ "json": true, "hide-extra": false, "file-pattern": "\\.clog.*", - "multiline": false, "line-format": [ - { "field" : "@timestamp" }, + { + "field": "@timestamp" + }, " ", - { "field" : "ipaddress" }, + { + "field": "ipaddress" + }, " ", - { "field" : "message" }, + { + "field": "message" + }, " ", - { "field" : "stack_trace", "default-value" : "" } + { + "field": "stack_trace", + "default-value": "" + } ], - "timestamp-field" : "@timestamp", - "body-field" : "message", - "level-field" : "level", - "level" : { - "trace" : "TRACE", - "debug" : "DEBUG", - "info" : "INFO", - "error" : "ERROR", - "warning" : "WARN" + "timestamp-field": "@timestamp", + "body-field": "message", + "level-field": "level", + "level": { + "trace": "TRACE", + "debug": "DEBUG", + "info": "INFO", + "error": "ERROR", + "warning": "WARN" }, - "value" : { - "logger_name" : { - "kind" : "string", - "identifier" : true + "value": { + "logger_name": { + "kind": "string", + "identifier": true }, - "ipaddress" : { - "kind" : "string", - "identifier" : true + "ipaddress": { + "kind": "string", + "identifier": true }, - "level_value" : { + "level_value": { "hidden": true }, - "stack_trace" : { - "kind" : "string" + "stack_trace": { + "kind": "string" } } } diff --git a/test/lnav_doctests.cc b/test/lnav_doctests.cc index a97b6e51..3241f9d1 100644 --- a/test/lnav_doctests.cc +++ b/test/lnav_doctests.cc @@ -37,6 +37,7 @@ #include "lnav_util.hh" #include "ptimec.hh" #include "relative_time.hh" +#include "shlex.hh" #include "unique_path.hh" using namespace std; @@ -61,6 +62,66 @@ TEST_CASE("overwritten-logfile") { } #endif +TEST_CASE("shlex::eval") +{ + std::string cmdline1 = "${semantic_highlight_color}"; + + shlex lexer(cmdline1); + + std::map vars = { + {"semantic_highlight_color", "foo"}, + }; + + std::string out; + auto rc = lexer.eval(out, scoped_resolver{&vars}); + CHECK(rc); + CHECK(out == "foo"); +} + +TEST_CASE("shlex::split") +{ + { + std::string cmdline1 = ""; + + std::map vars; + shlex lexer(cmdline1); + auto split_res = lexer.split(scoped_resolver{&vars}); + CHECK(split_res.isOk()); + auto args = split_res.unwrap(); + CHECK(args.empty()); + } + { + std::string cmdline1 = ":sh --name=\"foo $BAR\" echo Hello!"; + + std::map vars; + shlex lexer(cmdline1); + auto split_res = lexer.split(scoped_resolver{&vars}); + CHECK(split_res.isOk()); + auto args = split_res.unwrap(); + for (const auto& se : args) { + printf(" range %d:%d -- %s\n", + se.se_origin.sf_begin, + se.se_origin.sf_end, + se.se_value.c_str()); + } + } + { + std::string cmdline1 = "abc def $FOO ghi"; + + std::map vars; + shlex lexer(cmdline1); + auto split_res = lexer.split(scoped_resolver{&vars}); + CHECK(split_res.isOk()); + auto args = split_res.unwrap(); + for (const auto& se : args) { + printf(" range %d:%d -- %s\n", + se.se_origin.sf_begin, + se.se_origin.sf_end, + se.se_value.c_str()); + } + } +} + TEST_CASE("byte_array") { using my_array_t = byte_array<8>; diff --git a/test/test_cmds.sh b/test/test_cmds.sh index 8dccf623..ca232f82 100644 --- a/test/test_cmds.sh +++ b/test/test_cmds.sh @@ -48,6 +48,10 @@ run_cap_test env TZ=UTC ${lnav_test} -n \ -c ":unix-time 1612072409" \ "${test_dir}/logfile_access_log.*" +run_cap_test env TZ=UTC ${lnav_test} -n \ + -c ":unix-time 16120724091612072409" \ + "${test_dir}/logfile_access_log.*" + run_cap_test env TZ=UTC ${lnav_test} -n \ -c ":current-time" \ "${test_dir}/logfile_access_log.*"