Archives
/
lnav
mirror of https://github.com/tstack/lnav
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[sections] fix some issues with doc sections in logs

Browse Source
pull/1170/head
Tim Stack 12 months ago
parent e77ef5c8cd
commit 8118c5dbe4
10 changed files with 158 additions and 20 deletions
Show Stats Download Patch File Download Diff File

2
src/base/intern_string.cc
Unescape Escape View File

@ -219,7 +219,7 @@ string_fragment::split_lines() const
int start = this->sf_begin;
for (auto index = start; index < this->sf_end; index++) {
if ((*this)[index] == '\n') {
if (this->sf_string[index] == '\n') {
retval.emplace_back(this->sf_string, start, index + 1);
start = index + 1;
}

8
src/base/intern_string.tests.cc
Unescape Escape View File

@ -56,6 +56,14 @@ TEST_CASE("split_lines")
CHECK(in1 == split[0].to_string());
}
{
auto sf = string_fragment::from_str_range(in1, 7, -1);
auto split = sf.split_lines();
CHECK(1 == split.size());
CHECK("World!" == split[0].to_string());
}
{
auto sf = string_fragment(in2);
auto split = sf.split_lines();

8
src/document.sections.cc
Unescape Escape View File

@ -283,14 +283,18 @@ public:
case DT_XML_CLOSE_TAG: {
auto term = this->flush_values();
if (this->sw_depth > 0) {
this->sw_depth -= 1;
this->append_child_node(term);
if (term) {
this->append_child_node(term);
}
this->sw_interval_state.pop_back();
this->sw_hier_stage
= std::move(this->sw_hier_nodes.back());
this->sw_hier_nodes.pop_back();
}
this->append_child_node(el.e_capture);
if (this->sw_depth > 0) {
this->sw_depth -= 1;
}
this->flush_values();
break;
}

40
src/logfile.cc
Unescape Escape View File

@ -227,16 +227,18 @@ logfile::process_prefix(shared_buffer_ref& sbr,
const auto& root_formats = log_format::get_root_formats();
nonstd::optional<std::pair<log_format*, log_format::scan_match>>
best_match;
size_t scan_count = 0;
/*
* Try each scanner until we get a match. Fortunately, the formats
* tend to be sufficiently different that there are few ambiguities...
*/
log_info("logfile[%s]: scanning line %d (offset: %d; size: %d)",
this->lf_filename.c_str(),
this->lf_index.size(),
li.li_file_range.fr_offset,
li.li_file_range.fr_size);
log_trace("logfile[%s]: scanning line %d (offset: %d; size: %d)",
this->lf_filename.c_str(),
this->lf_index.size(),
li.li_file_range.fr_offset,
li.li_file_range.fr_size);
size_t prev_index_size = this->lf_index.size();
for (const auto& curr : root_formats) {
if (this->lf_index.size()
>= curr->lf_max_unrecognized_lines.value_or(
@ -268,13 +270,13 @@ logfile::process_prefix(shared_buffer_ref& sbr,
continue;
}
scan_count += 1;
curr->clear();
this->set_format_base_time(curr.get());
std::vector<logline> scan_backup = this->lf_index;
auto scan_res = curr->scan(*this, this->lf_index, li, sbr, sbc);
scan_res.match(
[this, &curr, &best_match, &scan_backup](
[this, &curr, &best_match, &prev_index_size](
const log_format::scan_match& sm) {
if (!best_match
|| sm.sm_quality > best_match->second.sm_quality)
@ -283,29 +285,43 @@ logfile::process_prefix(shared_buffer_ref& sbr,
" scan with format (%s) matched with quality (%d)",
curr->get_name().c_str(),
sm.sm_quality);
if (best_match) {
auto last = this->lf_index.begin();
std::advance(last, prev_index_size);
this->lf_index.erase(this->lf_index.begin(), last);
}
best_match = std::make_pair(curr.get(), sm);
prev_index_size = this->lf_index.size();
} else {
log_info(
" scan with format (%s) matched, but "
"is low quality (%d)",
curr->get_name().c_str(),
sm.sm_quality);
this->lf_index = std::move(scan_backup);
while (this->lf_index.size() > prev_index_size) {
this->lf_index.pop_back();
}
}
},
[curr](const log_format::scan_incomplete& si) {
log_info(
log_trace(
" scan with format (%s) is incomplete, "
"more data required",
curr->get_name().c_str());
},
[curr](const log_format::scan_no_match& snm) {
log_info(" scan with format (%s) does not match -- %s",
curr->get_name().c_str(),
snm.snm_reason);
log_trace(" scan with format (%s) does not match -- %s",
curr->get_name().c_str(),
snm.snm_reason);
});
}
if (!scan_count) {
log_info("%s: no formats available to scan, no longer detecting",
this->lf_filename.c_str());
this->lf_options.loo_detect_format = false;
}
if (best_match) {
auto winner = best_match.value();
auto* curr = winner.first;

9
src/logfile_sub_source.cc
Unescape Escape View File

@ -2275,10 +2275,10 @@ logfile_sub_source::text_crumbs_for_line(int line,
auto sf = sbr.to_string_fragment();
auto body_opt = get_string_attr(al.get_attrs(), SA_BODY);
auto sf_lines = sf.split_lines();
auto nl_pos_opt = sf.find('\n');
auto msg_line_number = std::distance(msg_start_iter, line_pair.second);
auto line_from_top = line - msg_line_number;
if (sf_lines.size() > 1 && body_opt) {
if (body_opt && nl_pos_opt) {
if (this->lss_token_meta_line != file_line_number
|| this->lss_token_meta_size != sf.length())
{
@ -2289,11 +2289,14 @@ logfile_sub_source::text_crumbs_for_line(int line,
}
const auto initial_size = crumbs.size();
auto sf_body
= sf.sub_range(body_opt->saw_string_attr->sa_range.lr_start,
body_opt->saw_string_attr->sa_range.lr_end);
file_off_t line_offset = 0;
file_off_t line_end_offset = sf.length();
size_t line_number = 0;
for (const auto& sf_line : sf_lines) {
for (const auto& sf_line : sf_body.split_lines()) {
if (line_number >= msg_line_number) {
line_end_offset = line_offset + sf_line.length();
break;

2
test/expected/expected.am
Unescape Escape View File

@ -962,6 +962,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_977cdf5d396522194d6b9e945169ff8073b4296b.out \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_9a5be90921256e90428c77753eca5ea0d31bd910.err \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_9a5be90921256e90428c77753eca5ea0d31bd910.out \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_a1e6ee4f098d525330d5f58a9d71cbbd816d51bb.err \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_a1e6ee4f098d525330d5f58a9d71cbbd816d51bb.out \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_a2c0f0e51b3f85ea2a05ecdcacaad962b4fe5d4f.err \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_a2c0f0e51b3f85ea2a05ecdcacaad962b4fe5d4f.out \
$(srcdir)/%reldir%/test_sql_views_vtab.sh_ac1f6e9a88608ef8939f9c2f7061a25a86742d46.err \

2
test/expected/test_json_format.sh_80959e2bb6a7fdf938c2e4dbd7d7c81eb84fa072.out
Unescape Escape View File

@ -1,5 +1,3 @@
2013-09-06T20:00:48.124 TRACE trace test
@fields: { "lvl": "TRACE", "msg": "trace test"}
2013-09-06T20:00:48.124 TRACE trace test
@fields: { "lvl": "TRACE", "msg": "trace test"}
2013-09-06T20:00:49.124 INFO Starting up service

0
test/expected/test_sql_views_vtab.sh_a1e6ee4f098d525330d5f58a9d71cbbd816d51bb.err
Unescape Escape View File

101
test/expected/test_sql_views_vtab.sh_a1e6ee4f098d525330d5f58a9d71cbbd816d51bb.out
Unescape Escape View File

@ -0,0 +1,101 @@
[
{
"top_meta": {
"time": "2020-12-10 06:56:41.092",
"file": "{test_dir}/logfile_xml_msg.0",
"breadcrumbs": [
{
"display_value": "2020-12-10T06:56:41.092",
"search_placeholder": "(Enter an absolute or relative time)",
"possibilities": [
{
"display_value": "-1 day"
},
{
"display_value": "-1h"
},
{
"display_value": "-30m"
},
{
"display_value": "-15m"
},
{
"display_value": "-5m"
},
{
"display_value": "-1m"
},
{
"display_value": "+1m"
},
{
"display_value": "+5m"
},
{
"display_value": "+15m"
},
{
"display_value": "+30m"
},
{
"display_value": "+1h"
},
{
"display_value": "+1 day"
}
]
},
{
"display_value": "generic_log",
"search_placeholder": "",
"possibilities": [
{
"display_value": "generic_log"
}
]
},
{
"display_value": "logfile_xml_msg.0[2]",
"search_placeholder": "",
"possibilities": [
{
"display_value": "logfile_xml_msg.0"
}
]
},
{
"display_value": "<a-request>",
"search_placeholder": "",
"possibilities": [
{
"display_value": "<a-request>"
}
]
},
{
"display_value": "<head>",
"search_placeholder": "",
"possibilities": [
{
"display_value": "<head>"
},
{
"display_value": "<request id=\"1\">"
},
{
"display_value": "<source>"
}
]
},
{
"display_value": "⋯",
"search_placeholder": "",
"possibilities": [
]
}
]
}
}
]

6
test/test_sql_views_vtab.sh
Unescape Escape View File

@ -171,3 +171,9 @@ run_cap_test ${lnav_test} -n \
run_cap_test ${lnav_test} -n \
-c ";UPDATE lnav_views SET top_meta = json_object('anchor', '#build') WHERE name = 'text'" \
${top_srcdir}/README.md
run_cap_test ${lnav_test} -n \
-c ":goto 5" \
-c ";SELECT top_meta FROM lnav_top_view" \
-c ":write-json-to -" \
${test_dir}/logfile_xml_msg.0

Write Preview
Loading…
Cancel
Save