diff --git a/NEWS b/NEWS index 3f340fd9..01ffe02d 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ lnav v0.7.3: * Add 'pipe-to' and 'pipe-line-to' commands that pipe the currently marked lines or the current log message to a shell command, respectively. + * Added a "pretty-print" view (P hotkey) that tries to reformat log + messages so that they are easier to read. lnav v0.7.2: * Added log formats for vdsm, openstack, and the vmkernel. diff --git a/docs/source/hotkeys.rst b/docs/source/hotkeys.rst index e3cb2fd7..b5d76883 100644 --- a/docs/source/hotkeys.rst +++ b/docs/source/hotkeys.rst @@ -174,6 +174,8 @@ Display - View/leave builtin help * - |ks| q |ke| - Return to the previous view/quit + * - |ks| Shift |ke| + |ks| p |ke| + - Switch to/from the pretty-printed view of the top log message * - |ks| Shift |ke| + |ks| t |ke| - Display elapsed time between lines * - |ks| t |ke| diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 24c36e54..263115f0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -84,6 +84,7 @@ set(diag_STAT_SRCS log_data_table.hh log_format_impls.cc logfile_stats.hh + pretty_printer.hh ptimec.hh sequence_sink.hh status_controllers.hh diff --git a/src/data_parser.hh b/src/data_parser.hh index 85e77df9..4876b40a 100644 --- a/src/data_parser.hh +++ b/src/data_parser.hh @@ -680,6 +680,9 @@ private: case DT_IPV6_ADDRESS: case DT_MAC_ADDRESS: case DT_HEX_DUMP: + case DT_XML_OPEN_TAG: + case DT_XML_CLOSE_TAG: + case DT_XML_EMPTY_TAG: case DT_UUID: case DT_URL: case DT_PATH: diff --git a/src/data_scanner.cc b/src/data_scanner.cc index 8b44e892..6208fb0e 100644 --- a/src/data_scanner.cc +++ b/src/data_scanner.cc @@ -63,6 +63,17 @@ static struct { { "hexd", pcrepp( "\\A([0-9a-fA-F][0-9a-fA-F](?::[0-9a-fA-F][0-9a-fA-F])+)"), }, + { "xmlt", pcrepp( + "\\A(<\\??[\\w:]+\\s*(?:[\\w:]+(?:\\s*=\\s*" + "(?:\"((?:\\\\.|[^\"])+)\"|'((?:\\\\.|[^'])+)'|[^>]+)" + "))*\\s*(?:/|\\?)>)"), }, + { "xmlo", pcrepp( + "\\A(<[\\w:]+\\s*(?:[\\w:]+(?:\\s*=\\s*" + "(?:\"((?:\\\\.|[^\"])+)\"|'((?:\\\\.|[^'])+)'|[^>]+)" + "))*\\s*>)"), }, + + { "xmlc", pcrepp("\\A()"), }, + { "coln", pcrepp("\\A(:)"), }, { "eq", pcrepp("\\A(=)"), diff --git a/src/data_scanner.hh b/src/data_scanner.hh index 63a14bcb..6c67eeb7 100644 --- a/src/data_scanner.hh +++ b/src/data_scanner.hh @@ -46,6 +46,9 @@ enum data_token_t { DT_TIME, DT_IPV6_ADDRESS, DT_HEX_DUMP, + DT_XML_EMPTY_TAG, + DT_XML_OPEN_TAG, + DT_XML_CLOSE_TAG, /* DT_QUALIFIED_NAME, */ DT_COLON, @@ -116,8 +119,9 @@ public: }; data_scanner(shared_buffer_ref &line, size_t off = 0, size_t len = (size_t) -1) - : ds_sbr(line), ds_pcre_input(line.get_data(), off, len) + : ds_sbr(line), ds_pcre_input(line.get_data(), off, len == -1 ? line.length() : len) { + require(len == -1 || len <= line.length()); if (line.length() > 0 && line.get_data()[line.length() - 1] == '.') { this->ds_pcre_input.pi_length -= 1; } @@ -130,6 +134,7 @@ public: private: std::string ds_line; shared_buffer_ref ds_sbr; - pcre_input ds_pcre_input; + pcre_input ds_pcre_input; }; + #endif diff --git a/src/default-log-formats.json b/src/default-log-formats.json index 65a47a94..227a11de 100644 --- a/src/default-log-formats.json +++ b/src/default-log-formats.json @@ -11,7 +11,7 @@ "pattern" : "^(?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?[^ ]+) (?[^ ]+) (?[A-Z]+) \"(?[^ \\?]+)(?:\\?(?[^ ]*))?\" (?:-1|\\d+) (?\\d+) \\d+" }, "std" : { - "pattern" : "^(?[\\w\\.:\\-]+) [\\w\\.\\-]+ (?\\S+) \\[(?[^\\]]+)\\] \"(?:\\-|(?\\w+) (?[^ \\?]+)(?:\\?(?[^ ]*))? (?[\\w/\\.]+))\" (?\\d+) (?\\d+|-)(?: \"(?[^\"]+)\" \"(?[^\"]+)\")?.*" + "pattern" : "^(?[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?\\S+)\\s+\\[(?[^\\]]+)\\] \"(?:\\-|(?\\w+) (?[^ \\?]+)(?:\\?(?[^ ]*))? (?[\\w/\\.]+))\" (?\\d+) (?\\d+|-)(?: \"(?[^\"]+)\" \"(?[^\"]+)\")?(?.*)" } }, "level-field": "sc_status", @@ -62,6 +62,9 @@ "sample" : [ { "line" : "10.112.72.172 - - [11/Feb/2013:06:43:36 +0000] \"GET /client/ HTTP/1.1\" 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\"" + }, + { + "line" : "10.1.10.51 - - [23/Dec/2014:21:20:35 +0000] \"POST /api/1/rest/foo/bar HTTP/1.1\" 200 - \"-\" \"-\" 293" } ] }, diff --git a/src/help.txt b/src/help.txt index dd33236b..dd7f85ee 100644 --- a/src/help.txt +++ b/src/help.txt @@ -161,6 +161,10 @@ through the file. >/< Move horizontally to the next/previous search hit. + P Switch to/from the pretty-printed view of the top log + message. In this view, structured data, such as XML, + will be reformatted to make it easier to read. + t Switch to/from the text file view. The text file view is for any files that are not recognized as log files. diff --git a/src/lnav.cc b/src/lnav.cc index 71131762..3ee14bc0 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -115,6 +115,7 @@ #include "log_data_helper.hh" #include "readline_highlighters.hh" #include "environ_vtab.hh" +#include "pretty_printer.hh" #include "yajlpp.hh" @@ -152,6 +153,7 @@ const char *lnav_view_strings[LNV__MAX + 1] = { "db", "example", "schema", + "pretty", NULL }; @@ -165,6 +167,7 @@ static const char *view_titles[LNV__MAX] = { "DB", "EXAMPLE", "SCHEMA", + "PRETTY", }; static bool rescan_files(bool required = false); @@ -948,8 +951,8 @@ public: } }; - plain_text_source(const vector &lines) { - this->tds_lines = lines; + plain_text_source(const vector &text_lines) { + this->tds_lines = text_lines; }; size_t text_line_count() @@ -1108,6 +1111,52 @@ static void update_view_name(void) A_REVERSE | view_colors::ansi_color_pair(COLOR_BLUE, COLOR_WHITE))); } +static void open_schema_view(void) +{ + textview_curses *schema_tc = &lnav_data.ld_views[LNV_SCHEMA]; + string schema; + + dump_sqlite_schema(lnav_data.ld_db, schema); + + schema += "\n\n-- Virtual Table Definitions --\n\n"; + schema += ENVIRON_CREATE_STMT; + for (log_vtab_manager::iterator vtab_iter = + lnav_data.ld_vtab_manager->begin(); + vtab_iter != lnav_data.ld_vtab_manager->end(); + ++vtab_iter) { + schema += vtab_iter->second->get_table_statement(); + } + + if (schema_tc->get_sub_source() != NULL) { + delete schema_tc->get_sub_source(); + } + + schema_tc->set_sub_source(new plain_text_source(schema)); +} + +static void open_pretty_view(void) +{ + textview_curses *log_tc = &lnav_data.ld_views[LNV_LOG]; + textview_curses *pretty_tc = &lnav_data.ld_views[LNV_PRETTY]; + logfile_sub_source &lss = lnav_data.ld_log_source; + if (lss.text_line_count() > 0) { + content_line_t cl = lss.at(log_tc->get_top()); + logfile *lf = lss.find(cl); + logfile::iterator ll = lf->message_start(lf->begin() + cl); + shared_buffer_ref sbr; + + lf->read_full_message(ll, sbr); + data_scanner ds(sbr); + pretty_printer pp(&ds); + + if (pretty_tc->get_sub_source() != NULL) { + delete pretty_tc->get_sub_source(); + } + string pretty_text = pp.print(); + pretty_tc->set_sub_source(new plain_text_source(pretty_text)); + } +} + bool toggle_view(textview_curses *toggle_tc) { textview_curses *tc = lnav_data.ld_view_stack.top(); @@ -1117,6 +1166,12 @@ bool toggle_view(textview_curses *toggle_tc) lnav_data.ld_view_stack.pop(); } else { + if (toggle_tc == &lnav_data.ld_views[LNV_SCHEMA]) { + open_schema_view(); + } + else if (toggle_tc == &lnav_data.ld_views[LNV_PRETTY]) { + open_pretty_view(); + } lnav_data.ld_view_stack.push(toggle_tc); retval = true; } @@ -1145,29 +1200,6 @@ void redo_search(lnav_view_t view_index) lnav_data.ld_scroll_broadcaster.invoke(tc); } -static void open_schema_view(void) -{ - textview_curses *schema_tc = &lnav_data.ld_views[LNV_SCHEMA]; - string schema; - - dump_sqlite_schema(lnav_data.ld_db, schema); - - schema += "\n\n-- Virtual Table Definitions --\n\n"; - schema += ENVIRON_CREATE_STMT; - for (log_vtab_manager::iterator vtab_iter = - lnav_data.ld_vtab_manager->begin(); - vtab_iter != lnav_data.ld_vtab_manager->end(); - ++vtab_iter) { - schema += vtab_iter->second->get_table_statement(); - } - - if (schema_tc->get_sub_source() != NULL) { - delete schema_tc->get_sub_source(); - } - - schema_tc->set_sub_source(new plain_text_source(schema)); -} - /** * Ensure that the view is on the top of the view stack. * @@ -1178,9 +1210,6 @@ void ensure_view(textview_curses *expected_tc) textview_curses *tc = lnav_data.ld_view_stack.top(); if (tc != expected_tc) { - if (expected_tc == &lnav_data.ld_views[LNV_SCHEMA]) { - open_schema_view(); - } toggle_view(expected_tc); } @@ -1968,6 +1997,16 @@ static void handle_paging_key(int ch) tc->reload_data(); break; + case 'P': + if (tc == &lnav_data.ld_views[LNV_PRETTY] || + (lss && lss->text_line_count() > 0)) { + toggle_view(&lnav_data.ld_views[LNV_PRETTY]); + } + else { + lnav_data.ld_rl_view->set_value("Pretty-printed only works with log messages"); + } + break; + case 't': if (lnav_data.ld_text_source.current_file() == NULL) { flash(); @@ -4264,6 +4303,7 @@ int main(int argc, char *argv[]) setup_highlights(lnav_data.ld_views[LNV_LOG].get_highlights()); setup_highlights(lnav_data.ld_views[LNV_TEXT].get_highlights()); setup_highlights(lnav_data.ld_views[LNV_SCHEMA].get_highlights()); + setup_highlights(lnav_data.ld_views[LNV_PRETTY].get_highlights()); } { diff --git a/src/lnav.hh b/src/lnav.hh index 6b95fb06..52439635 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -108,6 +108,7 @@ typedef enum { LNV_DB, LNV_EXAMPLE, LNV_SCHEMA, + LNV_PRETTY, LNV__MAX } lnav_view_t; diff --git a/src/pretty_printer.hh b/src/pretty_printer.hh new file mode 100644 index 00000000..6f50abbb --- /dev/null +++ b/src/pretty_printer.hh @@ -0,0 +1,170 @@ +/** + * Copyright (c) 2015, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __pretty_printer_hh +#define __pretty_printer_hh + +#include +#include + +#include "data_scanner.hh" + +class pretty_printer { + +public: + + struct element { + element(data_token_t token, pcre_context &pc) + : e_token(token), e_capture(*pc.all()) { + + }; + + data_token_t e_token; + pcre_context::capture_t e_capture; + }; + + pretty_printer(data_scanner *ds) + : pp_depth(0), pp_line_length(0), pp_scanner(ds) { + + }; + + std::string print() { + pcre_context_static<30> pc; + data_token_t dt; + + while (this->pp_scanner->tokenize(pc, dt)) { + element el(dt, pc); + + switch (dt) { + case DT_XML_EMPTY_TAG: + this->start_new_line(); + this->pp_values.push_back(el); + this->start_new_line(); + continue; + case DT_XML_OPEN_TAG: + this->start_new_line(); + this->write_element(el); + this->pp_depth += 1; + continue; + case DT_XML_CLOSE_TAG: + this->flush_values(); + this->pp_depth -= 1; + this->write_element(el); + this->start_new_line(); + continue; + case DT_LCURLY: + case DT_LSQUARE: + this->flush_values(true); + this->pp_values.push_back(el); + this->pp_depth += 1; + continue; + case DT_RCURLY: + case DT_RSQUARE: + this->flush_values(); + this->pp_depth -= 1; + this->write_element(el); + continue; + case DT_COMMA: + this->flush_values(true); + this->write_element(el); + this->start_new_line(); + continue; + } + this->pp_values.push_back(el); + } + this->flush_values(); + this->pp_stream << std::endl << std::ends; + + std::string retval = this->pp_stream.str(); + this->pp_stream.freeze(false); + return retval; + }; + +private: + + void start_new_line() { + bool has_output; + + if (this->pp_line_length > 0) { + this->pp_stream << std::endl; + } + has_output = this->flush_values(); + if (has_output) { + this->pp_stream << std::endl; + } + this->pp_line_length = 0; + } + + bool flush_values(bool start_on_depth = false) { + bool retval = false; + + while (!this->pp_values.empty()) { + { + element &el = this->pp_values.front(); + this->write_element(this->pp_values.front()); + if (start_on_depth && + (el.e_token == DT_LSQUARE || + el.e_token == DT_LCURLY)) { + this->pp_stream << std::endl; + this->pp_line_length = 0; + } + } + this->pp_values.pop_front(); + retval = true; + } + return retval; + } + + void append_indent() { + for (int lpc = 0; lpc < this->pp_depth; lpc++) { + this->pp_stream << " "; + } + } + + void write_element(const element &el) { + if (this->pp_line_length == 0 && el.e_token == DT_WHITE) { + return; + } + pcre_input &pi = this->pp_scanner->get_input(); + if (this->pp_line_length == 0) { + this->append_indent(); + } + this->pp_stream << pi.get_substr(&el.e_capture); + this->pp_line_length += el.e_capture.length(); + } + + int pp_depth; + int pp_line_length; + data_scanner *pp_scanner; + std::strstream pp_stream; + std::deque pp_values; + +}; + +#endif diff --git a/test/Makefile.am b/test/Makefile.am index 26d8ff23..2c14074a 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -200,6 +200,7 @@ dist_noinst_DATA = \ datafile_simple.11 \ datafile_simple.12 \ datafile_simple.13 \ + datafile_xml.0 \ listview_output.0 \ listview_output.1 \ listview_output.2 \ diff --git a/test/Makefile.in b/test/Makefile.in index cb45ca33..1e80f025 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -794,6 +794,7 @@ dist_noinst_DATA = \ datafile_simple.11 \ datafile_simple.12 \ datafile_simple.13 \ + datafile_xml.0 \ listview_output.0 \ listview_output.1 \ listview_output.2 \ diff --git a/test/datafile_xml.0 b/test/datafile_xml.0 new file mode 100644 index 00000000..bb839b90 --- /dev/null +++ b/test/datafile_xml.0 @@ -0,0 +1,21 @@ + + key 0:0 +xmlo 0:9 ^-------^ +pair 0:9 ^-------^ + key 12:12 ^ +xmlo 12:40 ^--------------------------^ +pair 12:40 ^--------------------------^ + key 42:42 ^ +xmlc 42:49 ^-----^ +pair 42:49 ^-----^ + key 51:51 ^ +xmlt 51:61 ^--------^ +pair 51:61 ^--------^ + +-- + + + + + + diff --git a/test/drive_data_scanner.cc b/test/drive_data_scanner.cc index e03556c2..96a0f01a 100644 --- a/test/drive_data_scanner.cc +++ b/test/drive_data_scanner.cc @@ -41,7 +41,8 @@ #include "data_parser.hh" #include "log_format.hh" #include "log_format_loader.hh" -#include "../src/shared_buffer.hh" +#include "pretty_printer.hh" +#include "shared_buffer.hh" using namespace std; @@ -50,7 +51,7 @@ const char *TMP_NAME = "scanned.tmp"; int main(int argc, char *argv[]) { int c, retval = EXIT_SUCCESS; - bool prompt = false, is_log = false; + bool prompt = false, is_log = false, pretty_print = false; { std::vector paths, errors; @@ -58,12 +59,16 @@ int main(int argc, char *argv[]) load_formats(paths, errors); } - while ((c = getopt(argc, argv, "pl")) != -1) { + while ((c = getopt(argc, argv, "pPl")) != -1) { switch (c) { case 'p': prompt = true; break; + case 'P': + pretty_print = true; + break; + case 'l': is_log = true; break; @@ -159,6 +164,14 @@ int main(int argc, char *argv[]) dp.parse(); dp.print(out, dp.dp_pairs); + + if (pretty_print) { + data_scanner ds2(sub_line, body.lr_start, sub_line.length()); + pretty_printer pp(&ds2); + + string pretty_out = pp.print(); + fprintf(out, "\n--\n%s", pretty_out.c_str()); + } fclose(out); sprintf(cmd, "diff -u %s %s", argv[lpc], TMP_NAME); diff --git a/test/test_data_parser.sh b/test/test_data_parser.sh index 2bf3271b..26f8163f 100644 --- a/test/test_data_parser.sh +++ b/test/test_data_parser.sh @@ -5,6 +5,11 @@ for fn in ${top_srcdir}/test/datafile_simple.*; do on_error_fail_with "$fn does not match" done +for fn in ${top_srcdir}/test/datafile_xml.*; do + run_test ./drive_data_scanner -P $fn + on_error_fail_with "$fn does not match" +done + for fn in ${top_srcdir}/test/log-samples/*.txt; do run_test ./drive_data_scanner -l $fn on_error_fail_with "$fn does not match"