[formats] fix error message if pattern does not fully match sample

src/base/string_attr_type.hh

@@ -477,6 +477,20 @@ h6(S str)
 
 namespace literals {
 
+inline std::pair<std::string, string_attr_pair> operator"" _ok(const char* str,
+                                                               std::size_t len)
+{
+    return std::make_pair(std::string(str, len),
+                          VC_ROLE.template value(role_t::VCR_OK));
+}
+
+inline std::pair<std::string, string_attr_pair> operator"" _error(
+    const char* str, std::size_t len)
+{
+    return std::make_pair(std::string(str, len),
+                          VC_ROLE.template value(role_t::VCR_ERROR));
+}
+
 inline std::pair<std::string, string_attr_pair> operator"" _info(
     const char* str, std::size_t len)
 {

src/log_format.cc

@@ -1945,7 +1945,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
              pat_iter != this->elf_pattern_order.end() && !found;
              ++pat_iter)
         {
-            pattern& pat = *(*pat_iter);
+            auto& pat = *(*pat_iter);
 
             if (!pat.p_pcre) {
                 continue;
@@ -2094,22 +2094,57 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
                 pcre_context_static<128> pc_full;
                 pcre_input pi_full(elf_sample.s_line.pp_value);
 
-                if (!pat.p_pcre->match(pc_full, pi_full)
-                    || static_cast<size_t>(pc_full.all()->length())
-                        != elf_sample.s_line.pp_value.length())
-                {
+                if (!pat.p_pcre->match(pc_full, pi_full)) {
+                    attr_line_t regex_al = pat.p_pcre->get_pattern();
+                    lnav::snippets::regex_highlighter(
+                        regex_al, -1, line_range{0, (int) regex_al.length()});
                     errors.emplace_back(
                         lnav::console::user_message::error(
                             attr_line_t("invalid pattern: ")
                                 .append_quoted(lnav::roles::symbol(
                                     pat.p_name.to_string())))
                             .with_reason("pattern does not match entire "
-                                         "multiline message")
+                                         "multiline sample message")
                             .with_snippet(elf_sample.s_line.to_snippet())
-                            .with_help(attr_line_t("using ")
-                                           .append_quoted(".*")
-                                           .append(" when capturing the body "
-                                                   "will match new-lines")));
+                            .with_note(attr_line_t()
+                                           .append(lnav::roles::symbol(
+                                               pat.p_name.to_string()))
+                                           .append(" = ")
+                                           .append(regex_al))
+                            .with_help(
+                                attr_line_t("use ").append_quoted(".*").append(
+                                    " to match new-lines")));
+                } else if (static_cast<size_t>(pc_full.all()->length())
+                           != elf_sample.s_line.pp_value.length())
+                {
+                    attr_line_t regex_al = pat.p_pcre->get_pattern();
+                    lnav::snippets::regex_highlighter(
+                        regex_al, -1, line_range{0, (int) regex_al.length()});
+                    auto match_length
+                        = static_cast<size_t>(pc_full.all()->length());
+                    attr_line_t sample_al = elf_sample.s_line.pp_value;
+                    sample_al.append("\n")
+                        .append(match_length, ' ')
+                        .append("^ matched up to here"_error)
+                        .with_attr_for_all(
+                            VC_ROLE.value(role_t::VCR_QUOTED_CODE));
+                    auto sample_snippet = lnav::console::snippet::from(
+                        elf_sample.s_line.pp_location, sample_al);
+                    errors.emplace_back(
+                        lnav::console::user_message::error(
+                            attr_line_t("invalid pattern: ")
+                                .append_quoted(lnav::roles::symbol(
+                                    pat.p_name.to_string())))
+                            .with_reason("pattern does not match entire "
+                                         "message")
+                            .with_snippet(sample_snippet)
+                            .with_note(attr_line_t()
+                                           .append(lnav::roles::symbol(
+                                               pat.p_name.to_string()))
+                                           .append(" = ")
+                                           .append(regex_al))
+                            .with_help("update the regular expression to fully "
+                                       "capture the sample message"));
                 }
             }
         }

src/yajlpp/yajlpp.cc

@@ -125,7 +125,8 @@ yajl_cleanup_tree(yajl_val val)
 
             yajl_cleanup_tree(child_val);
             if (YAJL_IS_OBJECT(child_val)
-                && YAJL_GET_OBJECT(child_val)->len == 0) {
+                && YAJL_GET_OBJECT(child_val)->len == 0)
+            {
                 free((char*) val_as_obj->keys[lpc]);
                 yajl_tree_free(val_as_obj->values[lpc]);
                 val_as_obj->len -= 1;
@@ -312,7 +313,8 @@ json_path_handler_base::gen_schema(yajlpp_gen_context& ygc) const
                 schema.gen("type");
                 if (this->jph_is_array) {
                     if (this->jph_regex->p_pattern.find("#?")
-                        == std::string::npos) {
+                        == std::string::npos)
+                    {
                         schema.gen("array");
                     } else {
                         yajlpp_array type_array(ygc.ygc_handle);
@@ -610,7 +612,8 @@ yajlpp_parse_context::map_start(void* ctx)
     ypc->ypc_path_index_stack.push_back(ypc->ypc_path.size() - 1);
 
     if (ypc->ypc_path.size() > 1
-        && ypc->ypc_path[ypc->ypc_path.size() - 2] == '#') {
+        && ypc->ypc_path[ypc->ypc_path.size() - 2] == '#')
+    {
         ypc->ypc_array_index.back() += 1;
     }
 
@@ -695,7 +698,8 @@ yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers,
         std::string curr_path(&this->ypc_path[0], this->ypc_path.size() - 1);
 
         if (this->ypc_active_paths.find(curr_path)
-            == this->ypc_active_paths.end()) {
+            == this->ypc_active_paths.end())
+        {
             return;
         }
     }
@@ -736,14 +740,16 @@ yajlpp_parse_context::update_callbacks(const json_path_container* orig_handlers,
                 this->ypc_handler_stack.emplace_back(&jph);
 
                 if (1 + child_start + cap->c_end
-                    != (int) this->ypc_path.size() - 1) {
+                    != (int) this->ypc_path.size() - 1)
+                {
                     this->update_callbacks(jph.jph_children,
                                            1 + child_start + cap->c_end);
                     return;
                 }
             } else {
                 if (1 + child_start + cap->c_end
-                    != (int) this->ypc_path.size() - 1) {
+                    != (int) this->ypc_path.size() - 1)
+                {
                     continue;
                 }
 
@@ -1437,17 +1443,18 @@ json_path_handler_base::report_regex_value_error(
     pcre_error_content.append("\n")
         .append(pcre_error.ce_offset, ' ')
         .append(lnav::roles::error("^ "))
-        .append(lnav::roles::error(pcre_error.ce_msg));
+        .append(lnav::roles::error(pcre_error.ce_msg))
+        .with_attr_for_all(VC_ROLE.value(role_t::VCR_QUOTED_CODE));
     ypc->report_error(lnav::console::user_message::error(
                           attr_line_t()
                               .append_quoted(value)
                               .append(" is not a valid regular expression for "
                                       "property ")
-                              .append_quoted(
-                    lnav::roles::symbol(ypc->get_full_path().to_string())))
-            .with_reason(pcre_error.ce_msg)
-            .with_snippet(ypc->get_snippet())
-            .with_snippet(lnav::console::snippet::from(
+                              .append_quoted(lnav::roles::symbol(
+                                  ypc->get_full_path().to_string())))
+                          .with_reason(pcre_error.ce_msg)
+                          .with_snippet(ypc->get_snippet())
+                          .with_snippet(lnav::console::snippet::from(
                               ypc->get_full_path(), pcre_error_content))
                           .with_help(this->get_help_text(ypc)));
 }

test/bad-config/formats/invalid-regex/format.json

@@ -4,12 +4,15 @@
         "regex": {
             "std": {
                 "pattern": "^(?<timestamp>\\d+: (?<body>.*)$"
+            },
+            "incomplete-match": {
+                "pattern": "^(?<timestamp>\\d+);"
             }
         },
-        "level" : {
-            "error" : "(foo"
+        "level": {
+            "error": "(foo"
         },
-        "timestamp-format" : [
+        "timestamp-format": [
             "%i"
         ],
         "sample": [

test/expected/expected.am

@@ -232,6 +232,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_events.sh_ed8dc44add223341c03ccb7b3e18371bdb42b710.out \
     $(srcdir)/%reldir%/test_format_loader.sh_15e861d2327512a721fd42ae51dc5427689e0bb6.err \
     $(srcdir)/%reldir%/test_format_loader.sh_15e861d2327512a721fd42ae51dc5427689e0bb6.out \
+    $(srcdir)/%reldir%/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err \
+    $(srcdir)/%reldir%/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.out \
     $(srcdir)/%reldir%/test_format_loader.sh_a47f2b090a5d8a226783835c7ff7d1c8821f11ed.err \
     $(srcdir)/%reldir%/test_format_loader.sh_a47f2b090a5d8a226783835c7ff7d1c8821f11ed.out \
     $(srcdir)/%reldir%/test_json_format.sh_168cac40c27f547044c89d39eb0ff2ef81da4b21.err \

test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err

@@ -0,0 +1,131 @@
+✘ error: “abc(def” is not a valid regular expression for property “/invalid_props_log/search-table/bad_table_regex/pattern”
+ reason: missing )
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:24
+ |                 "pattern": "abc(def"    
+ --> /invalid_props_log/search-table/bad_table_regex/pattern
+ | abc(def                                 
+ |        ^ missing )                      
+ = help: Property Synopsis
+           /invalid_props_log/search-table/bad_table_regex/pattern <regex>
+         Description
+           The regular expression for this search table.
+✘ error: “^(?<timestamp>\d+: (?<body>.*)$” is not a valid regular expression for property “/bad_regex_log/regex/std/pattern”
+ reason: missing )
+ --> {test_dir}/bad-config/formats/invalid-regex/format.json:6
+ |                 "pattern": "^(?<timestamp>\\d+: (?<body>.*)$"
+ --> /bad_regex_log/regex/std/pattern
+ | ^(?<timestamp>\d+: (?<body>.*)$           
+ |                                ^ missing )
+ = help: Property Synopsis
+           /bad_regex_log/regex/std/pattern <message-regex>
+         Description
+           The regular expression to match a log message and capture fields.
+✘ error: “(foo” is not a valid regular expression for property “/bad_regex_log/level/error”
+ reason: missing )
+ --> {test_dir}/bad-config/formats/invalid-regex/format.json:13
+ |             "error": "(foo"             
+ --> /bad_regex_log/level/error
+ | (foo                                    
+ |     ^ missing )                         
+ = help: Property Synopsis
+           /bad_regex_log/level/error <pattern|integer>
+         Description
+           The regular expression used to match the log text for this level.  For JSON logs with numeric levels, this should be the number for the corresponding level.
+✘ error: “abc(” is not a valid regular expression for property “/bad_regex_log/highlights/foobar/pattern”
+ reason: missing )
+ --> {test_dir}/bad-config/formats/invalid-regex/format.json:25
+ |                 "pattern": "abc("       
+ --> /bad_regex_log/highlights/foobar/pattern
+ | abc(                                    
+ |     ^ missing )                         
+ = help: Property Synopsis
+           /bad_regex_log/highlights/foobar/pattern <regex>
+         Description
+           A regular expression to highlight in logs of this format.
+✘ error: “foo” is not a valid value for option “/bad_sample_log/value/pid/kind”
+ --> {test_dir}/bad-config/formats/invalid-sample/format.json:24
+ |                 "kind": "foo"           
+ = help: Property Synopsis
+           /bad_sample_log/value/pid/kind <data-type>
+         Description
+           The type of data in the field
+         Allowed Values
+           string, integer, float, boolean, json, struct, quoted, xml
+✘ error: 'bad' is not a supported log format $schema version
+ --> {test_dir}/bad-config/formats/invalid-schema/format.json:2
+ |     "$schema": "bad"                    
+ = note: expecting one of the following $schema values:
+           https://lnav.org/schemas/format-v1.schema.json
+ = help: Property Synopsis
+           /$schema The URI of the schema for this file
+         Description
+           Specifies the type of this file
+✘ error: invalid pattern: “incomplete-match”
+ reason: pattern does not match entire message
+ --> {test_dir}/bad-config/formats/invalid-regex/format.json:20
+ | 1428634687123; foo                      
+ |               ^ matched up to here      
+ = note: incomplete-match = ^(?<timestamp>\d+);
+ = help: update the regular expression to fully capture the sample message
+✘ error: invalid sample log message: "abc: foo"
+ reason: unrecognized timestamp -- abc
+ --> {test_dir}/bad-config/formats/invalid-sample/format.json:30
+ = note: the following custom formats were tried:
+           abc
+           ^ “%i” matched up to here
+ = help: If the timestamp format is not supported by default, you can add a custom format with the “timestamp-format” property
+✘ error: invalid sample log message: "1428634687123| debug hello"
+ reason: “debug” does not match the expected level of “info”
+ --> {test_dir}/bad-config/formats/invalid-sample/format.json:33
+ = note: matched regex = with-level
+         captured level = “debug”
+✘ error: invalid pattern: “with-level”
+ reason: pattern does not match entire multiline sample message
+ --> {test_dir}/bad-config/formats/invalid-sample/format.json:37
+ = note: with-level = ^(?<timestamp>\d+)\| (?<level>\w+) (?<body>\w+)$
+ = help: use “.*” to match new-lines
+✘ error: invalid sample log message: "1428634687123; foo bar"
+ reason: sample does not match any patterns
+ --> {test_dir}/bad-config/formats/invalid-sample/format.json:41
+ = note: the following shows how each pattern matched this sample:
+            1428634687123; foo bar
+                         ^ bad-time matched up to here
+                              ^ semi matched up to here
+                         ^ std matched up to here
+                         ^ with-level matched up to here
+ = note: bad-time   = “^(?<timestamp>\w+): (?<body>\w+)$”
+         semi       = “^(?<timestamp>\d+); (?<body>\w+)$”
+         std        = “^(?<timestamp>\d+): (?<pid>\w+) (?<body>.*)$”
+         with-level = “^(?<timestamp>\d+)\| (?<level>\w+) (?<body>\w+)$”
+         
+✘ error: invalid value for property “/invalid_props_log/timestamp-field”
+ reason: “ts” was not found in the pattern at /invalid_props_log/regex/std
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:4
+ = note: the following captures are available:
+           body, pid, timestamp
+✘ error: “not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/color”
+ reason: Unknown color: 'not a color'.  See https://jonasjacek.github.io/colors/ for a list of supported color names
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:18
+✘ error: “also not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/background-color”
+ reason: Unknown color: 'also not a color'.  See https://jonasjacek.github.io/colors/ for a list of supported color names
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:19
+✘ error: “no_regexes_log” is not a valid log format
+ reason: no regexes specified
+ --> {test_dir}/bad-config/formats/no-regexes/format.json:4
+✘ error: “no_regexes_log” is not a valid log format
+ reason: log message samples must be included in a format definition
+ --> {test_dir}/bad-config/formats/no-regexes/format.json:4
+✘ error: “no_sample_log” is not a valid log format
+ reason: log message samples must be included in a format definition
+ --> {test_dir}/bad-config/formats/no-samples/format.json:4
+✘ error: failed to compile SQL statement
+ reason: near "TALE": syntax error
+ --> {test_dir}/bad-config/formats/invalid-sql/init.sql:4
+ | -- comment test                         
+ | CREATE TALE invalid (x y z);            
+ |        ^ near "TALE": syntax error      
+✘ error: failed to execute SQL statement
+ reason: lnav-error:{"level":"error","message":{"str":"call to regexp_match(re, str) failed","attrs":[{"start":8,"end":20,"type":"role","value":46},{"start":21,"end":23,"type":"role","value":45},{"start":25,"end":28,"type":"role","value":45},{"start":8,"end":29,"type":"role","value":59}]},"reason":{"str":"missing )","attrs":[]},"snippets":[],"help":{"str":"","attrs":[]}}
+ --> {test_dir}/bad-config/formats/invalid-sql/init2.sql
+ | SELECT regexp_match('abc(', '123')      
+ | FROM sqlite_master;                     

test/expected/test_format_loader.sh_a47f2b090a5d8a226783835c7ff7d1c8821f11ed.err

@@ -9,8 +9,8 @@
  --> {test_dir}/bad-config-json/formats/invalid-key/format.json:4
  |         "level-pointer": "abc(",        
  --> /invalid_key_log/level-pointer
- | abc(                                    
- |     ^ missing )                         
+ | abc(                                    
+ |     ^ missing )                         
  = help: Property Synopsis
            /invalid_key_log/level-pointer 
          Description
@@ -20,8 +20,8 @@
  --> {test_dir}/bad-config-json/formats/invalid-key/format.json:5
  |         "file-pattern": "def[ghi",      
  --> /invalid_key_log/file-pattern
- | def[ghi                                           
- |        ^ missing terminating ] for character class
+ | def[ghi                                           
+ |        ^ missing terminating ] for character class
  = help: Property Synopsis
            /invalid_key_log/file-pattern 
          Description

test/test_format_loader.sh

@@ -5,7 +5,7 @@ export YES_COLOR=1
 run_cap_test ${lnav_test} -C \
     -I ${test_dir}/bad-config-json
 
-if x"$HAVE_SQLITE3_ERROR_OFFSET" != x""; then
+if test x"$HAVE_SQLITE3_ERROR_OFFSET" != x""; then
     run_cap_test env LC_ALL=C ${lnav_test} -C \
         -I ${test_dir}/bad-config
 fi