Archives
/
lnav
mirror of https://github.com/tstack/lnav
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[json-logs] automatically remove trailing line feeds

Browse Source 
Fixes #969
pull/1170/head
Tim Stack 12 months ago
parent e0811eb629
commit 7bf42e7d2f
8 changed files with 57 additions and 32 deletions
Show Stats Download Patch File Download Diff File

4
NEWS.md
Unescape Escape View File

@ -19,6 +19,10 @@ Features:
string can optionally be prepended/appended if the value
is not empty.
Changes:
* For JSON-lines logs, line-feeds at the end of a value are
automatically stripped.
Bug Fixes:
* Hidden values in JSON logs are now hidden by default.
* Text with ANSI-escapes is now filtered properly.

2
docs/source/formats.rst
Unescape Escape View File

@ -146,6 +146,8 @@ object with the following fields:
[ { "field": "ts" }, " ", { "field": "msg" } ]
.. note:: Line-feeds at the end of a value are automatically stripped.
:field: The name or `JSON-Pointer <https://tools.ietf.org/html/rfc6901>`_
of the message field that should be inserted at this point in the
message. The special :code:`__timestamp__` field name can be used to

7
src/base/intern_string.hh
Unescape Escape View File

@ -170,6 +170,13 @@ struct string_fragment {
char back() const { return this->sf_string[this->sf_end - 1]; }
void pop_back()
{
if (!this->empty()) {
this->sf_end -= 1;
}
}
iterator begin() const { return &this->sf_string[this->sf_begin]; }
iterator end() const { return &this->sf_string[this->sf_end]; }

11
src/log_format.cc
Unescape Escape View File

@ -1549,6 +1549,9 @@ external_log_format::get_subline(const logline& ll,
logline_value_cmp(&jfe.jfe_value.pp_value));
if (lv_iter != this->jlf_line_values.lvv_values.end()) {
auto str = lv_iter->to_string();
while (endswith(str, "\n")) {
str.pop_back();
}
size_t nl_pos = str.find('\n');
if (!jfe.jfe_prefix.empty()) {
@ -1758,7 +1761,10 @@ external_log_format::get_subline(const logline& ll,
continue;
}
const std::string str = lv.to_string();
auto str = lv.to_string();
while (endswith(str, "\n")) {
str.pop_back();
}
lv.lv_sub_offset = sub_offset;
lv.lv_origin.lr_start = this->jlf_cached_line.size() + 2
@ -2964,6 +2970,9 @@ external_log_format::value_line_count(const intern_string_t ist,
value_line_count_result retval;
if (str != nullptr) {
auto frag = string_fragment::from_bytes(str, len);
while (frag.endswith("\n")) {
frag.pop_back();
}
while (!frag.empty()) {
auto utf_res = is_utf8(frag, '\n');
if (!utf_res.is_valid()) {

26
test/expected/test_json_format.sh_168cac40c27f547044c89d39eb0ff2ef81da4b21.out
Unescape Escape View File

@ -1,13 +1,13 @@
{"ts": "2013-09-06T20:00:48.124817Z", "lvl": "TRACE", "msg": "trace test"}
{"ts": "2013-09-06T20:00:49.124817Z", "lvl": "INFO", "msg": "Starting up \u001B[0;32mservice\u001B[0m"}
{"ts": "2013-09-06T22:00:49.124817Z", "lvl": "INFO", "msg": "Shutting down service", "user": "steve@example.com"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG5", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG4", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG3", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG2", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG", "msg": "Details..."}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "STATS", "msg": "1 beat per second"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "WARNING", "msg": "not looking good"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "ERROR", "msg": "looking bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "CRITICAL", "msg": "sooo bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "FATAL", "msg": "shoot", "obj": { "field1" : "hi", "field2": 2 }, "arr" : ["hi", {"sub1": true}]}
{"ts": "2013-09-06T20:00:48.124817Z", "lvl": "TRACE", "msg": "trace test"}
{"ts": "2013-09-06T20:00:49.124817Z", "lvl": "INFO", "msg": "Starting up \u001B[0;32mservice\u001B[0m"}
{"ts": "2013-09-06T22:00:49.124817Z", "lvl": "INFO", "msg": "Shutting down service", "user": "steve@example.com"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG5", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG4", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG3", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG2", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG", "msg": "Details..."}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "STATS", "msg": "1 beat per second"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "WARNING", "msg": "not looking good"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "ERROR", "msg": "looking bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "CRITICAL", "msg": "sooo bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "FATAL", "msg": "shoot", "obj": { "field1" : "hi", "field2": 2 }, "arr" : ["hi", {"sub1": true}]}

12
test/expected/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.out
Unescape Escape View File

@ -6,13 +6,17 @@
[2013-09-06T22:00:49.124] ⋮ Shutting down servicebork bork bork
user: mailto:steve@example.com
[2013-09-06T22:00:59.124] ⋮ Details...bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...
bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...
bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...
bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...
bork bork bork
[2013-09-06T22:00:59.124] ⋮ Details...bork bork bork

1
test/expected/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out
Unescape Escape View File

@ -36,7 +36,6 @@
 at org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1720)
 at org.apache.catalina.loader.WebappClassLoader.loadClass(WebappClassLoader.java:1571)
 ... 33 common frames omitted
 @version: 1
 logger_name: org.apache.jasper.runtime.JspFactoryImpl
 thread_name: http-bio-0.0.0.0-8081-exec-198

26
test/logfile_json.json
Unescape Escape View File

@ -1,13 +1,13 @@
{"ts": "2013-09-06T20:00:48.124817Z", "lvl": "TRACE", "msg": "trace test"}
{"ts": "2013-09-06T20:00:49.124817Z", "lvl": "INFO", "msg": "Starting up \u001B[0;32mservice\u001B[0m"}
{"ts": "2013-09-06T22:00:49.124817Z", "lvl": "INFO", "msg": "Shutting down service", "user": "steve@example.com"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG5", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG4", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG3", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG2", "msg": "Details..."}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG", "msg": "Details..."}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "STATS", "msg": "1 beat per second"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "WARNING", "msg": "not looking good"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "ERROR", "msg": "looking bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "CRITICAL", "msg": "sooo bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "FATAL", "msg": "shoot", "obj": { "field1" : "hi", "field2": 2 }, "arr" : ["hi", {"sub1": true}]}
{"ts": "2013-09-06T20:00:48.124817Z", "lvl": "TRACE", "msg": "trace test"}
{"ts": "2013-09-06T20:00:49.124817Z", "lvl": "INFO", "msg": "Starting up \u001B[0;32mservice\u001B[0m"}
{"ts": "2013-09-06T22:00:49.124817Z", "lvl": "INFO", "msg": "Shutting down service", "user": "steve@example.com"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG5", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG4", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG3", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG2", "msg": "Details...\n"}
{"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG", "msg": "Details..."}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "STATS", "msg": "1 beat per second"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "WARNING", "msg": "not looking good"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "ERROR", "msg": "looking bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "CRITICAL", "msg": "sooo bad"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": "FATAL", "msg": "shoot", "obj": { "field1" : "hi", "field2": 2 }, "arr" : ["hi", {"sub1": true}]}

Write Preview
Loading…
Cancel
Save