diff --git a/configure b/configure index 722cef36..744f475f 100755 --- a/configure +++ b/configure @@ -3751,8 +3751,8 @@ fi -#CFLAGS=`echo $CFLAGS | sed 's/-O2//g'` -#CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'` +# CFLAGS=`echo $CFLAGS | sed 's/-O2//g'` +# CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'` # Check whether --enable-static was given. if test "${enable_static+set}" = set; then : diff --git a/configure.ac b/configure.ac index 11a2a0be..7f1b49bd 100644 --- a/configure.ac +++ b/configure.ac @@ -35,8 +35,8 @@ AC_SUBST(abssrcdir) AC_PROG_CXX -#CFLAGS=`echo $CFLAGS | sed 's/-O2//g'` -#CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'` +# CFLAGS=`echo $CFLAGS | sed 's/-O2//g'` +# CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'` AC_ARG_ENABLE([static], AS_HELP_STRING([--disable-static], diff --git a/src/Makefile.am b/src/Makefile.am index 171979d9..bb23b67c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -23,7 +23,8 @@ LDADD = \ $(READLINE_LIBS) \ $(CURSES_LIB) \ $(SQLITE3_LIBS) \ - -lpcrecpp + -lpcrecpp \ + -lcrypto noinst_HEADERS = \ auto_fd.hh \ @@ -32,6 +33,7 @@ noinst_HEADERS = \ bookmarks.hh \ bottom_status_source.hh \ byte_array.hh \ + column_namer.hh \ data_scanner.hh \ data_parser.hh \ db_sub_source.hh \ @@ -55,6 +57,7 @@ noinst_HEADERS = \ sequence_matcher.hh \ sequence_sink.hh \ statusview_curses.hh \ + strnatcmp.h \ strong_int.hh \ termios_guard.hh \ textfile_sub_source.hh \ @@ -69,6 +72,8 @@ noinst_HEADERS = \ libdiag_a_SOURCES = \ bookmarks.cc \ + collation-functions.cc \ + extension-functions.c \ grep_proc.cc \ hist_source.cc \ line_buffer.cc \ @@ -77,12 +82,14 @@ libdiag_a_SOURCES = \ log_format.cc \ logfile.cc \ logfile_sub_source.cc \ + network-extension-functions.cc \ data_scanner.cc \ data_parser.cc \ readline_curses.cc \ sequence_matcher.cc \ statusview_curses.cc \ piper_proc.cc \ + strnatcmp.c \ textview_curses.cc \ view_curses.cc \ vt52_curses.cc \ diff --git a/src/Makefile.in b/src/Makefile.in index 5c16ab05..ec1e85d9 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -77,16 +77,19 @@ am__v_AR_0 = @echo " AR " $@; am__v_AR_1 = libdiag_a_AR = $(AR) $(ARFLAGS) libdiag_a_LIBADD = -am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) grep_proc.$(OBJEXT) \ - hist_source.$(OBJEXT) line_buffer.$(OBJEXT) \ - listview_curses.$(OBJEXT) lnav_commands.$(OBJEXT) \ - log_format.$(OBJEXT) logfile.$(OBJEXT) \ - logfile_sub_source.$(OBJEXT) data_scanner.$(OBJEXT) \ +am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) \ + collation-functions.$(OBJEXT) extension-functions.$(OBJEXT) \ + grep_proc.$(OBJEXT) hist_source.$(OBJEXT) \ + line_buffer.$(OBJEXT) listview_curses.$(OBJEXT) \ + lnav_commands.$(OBJEXT) log_format.$(OBJEXT) logfile.$(OBJEXT) \ + logfile_sub_source.$(OBJEXT) \ + network-extension-functions.$(OBJEXT) data_scanner.$(OBJEXT) \ data_parser.$(OBJEXT) readline_curses.$(OBJEXT) \ sequence_matcher.$(OBJEXT) statusview_curses.$(OBJEXT) \ - piper_proc.$(OBJEXT) textview_curses.$(OBJEXT) \ - view_curses.$(OBJEXT) vt52_curses.$(OBJEXT) \ - log_vtab_impl.$(OBJEXT) xterm_mouse.$(OBJEXT) + piper_proc.$(OBJEXT) strnatcmp.$(OBJEXT) \ + textview_curses.$(OBJEXT) view_curses.$(OBJEXT) \ + vt52_curses.$(OBJEXT) log_vtab_impl.$(OBJEXT) \ + xterm_mouse.$(OBJEXT) libdiag_a_OBJECTS = $(am_libdiag_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) @@ -300,7 +303,8 @@ LDADD = \ $(READLINE_LIBS) \ $(CURSES_LIB) \ $(SQLITE3_LIBS) \ - -lpcrecpp + -lpcrecpp \ + -lcrypto noinst_HEADERS = \ auto_fd.hh \ @@ -309,6 +313,7 @@ noinst_HEADERS = \ bookmarks.hh \ bottom_status_source.hh \ byte_array.hh \ + column_namer.hh \ data_scanner.hh \ data_parser.hh \ db_sub_source.hh \ @@ -332,6 +337,7 @@ noinst_HEADERS = \ sequence_matcher.hh \ sequence_sink.hh \ statusview_curses.hh \ + strnatcmp.h \ strong_int.hh \ termios_guard.hh \ textfile_sub_source.hh \ @@ -346,6 +352,8 @@ noinst_HEADERS = \ libdiag_a_SOURCES = \ bookmarks.cc \ + collation-functions.cc \ + extension-functions.c \ grep_proc.cc \ hist_source.cc \ line_buffer.cc \ @@ -354,12 +362,14 @@ libdiag_a_SOURCES = \ log_format.cc \ logfile.cc \ logfile_sub_source.cc \ + network-extension-functions.cc \ data_scanner.cc \ data_parser.cc \ readline_curses.cc \ sequence_matcher.cc \ statusview_curses.cc \ piper_proc.cc \ + strnatcmp.c \ textview_curses.cc \ view_curses.cc \ vt52_curses.cc \ @@ -487,8 +497,10 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bin2c.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bookmarks.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/collation-functions.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_parser.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_scanner.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/extension-functions.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/grep_proc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hist_source.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/line_buffer.Po@am__quote@ @@ -499,10 +511,12 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log_vtab_impl.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile_sub_source.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/network-extension-functions.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/piper_proc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readline_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sequence_matcher.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statusview_curses.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/strnatcmp.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textview_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/view_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vt52_curses.Po@am__quote@ diff --git a/src/auto_mem.hh b/src/auto_mem.hh index c4e11c6b..10a28208 100644 --- a/src/auto_mem.hh +++ b/src/auto_mem.hh @@ -38,19 +38,28 @@ #include +typedef void (*free_func_t)(void *); + /** * Resource management class for memory allocated by a custom allocator. * * @param T The object type. * @param auto_free The function to call to free the managed object. */ -template +template class auto_mem { public: - auto_mem(T *ptr = NULL) : am_ptr(ptr) { }; + auto_mem(T *ptr = NULL) : am_ptr(ptr), am_free_func(default_free) { }; + + auto_mem(auto_mem &am) + : am_ptr(am.release()), am_free_func(am.am_free_func) + { + }; - auto_mem(auto_mem &am) : am_ptr(am.release()) { }; + template + auto_mem(F free_func) + : am_ptr(NULL), am_free_func((void (*)(void *))free_func) { }; ~auto_mem() { this->reset(); }; @@ -82,14 +91,15 @@ public: void reset(T *ptr = NULL) { if (this->am_ptr != ptr) { - auto_free(this->am_ptr); + this->am_free_func(this->am_ptr); this->am_ptr = ptr; } }; private: T *am_ptr; - + void (*am_free_func)(void *); + }; #endif diff --git a/src/bottom_status_source.hh b/src/bottom_status_source.hh index 402b5f71..4afd24d4 100644 --- a/src/bottom_status_source.hh +++ b/src/bottom_status_source.hh @@ -32,6 +32,11 @@ #include +#include "grep_proc.hh" +#include "textview_curses.hh" +#include "logfile_sub_source.hh" +#include "status_controllers.hh" + class bottom_status_source : public status_data_source, public grep_proc_control @@ -52,6 +57,7 @@ public: BSF_ERRORS, BSF_FILTERED, BSF_LOADING, + BSF_HELP, BSF__MAX } field_t; @@ -64,17 +70,20 @@ public: bss_error(80, view_colors::VCR_ALERT_STATUS), bss_hit_spinner(0), bss_load_percent(0) { - this->bss_fields[BSF_LINE_NUMBER].set_width(8); + this->bss_fields[BSF_LINE_NUMBER].set_width(10); this->bss_fields[BSF_PERCENT].set_width(4); this->bss_fields[BSF_HITS].set_width(16); this->bss_fields[BSF_HITS].set_cylon(true); this->bss_fields[BSF_WARNINGS].set_width(10); this->bss_fields[BSF_ERRORS].set_width(10); this->bss_fields[BSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS); - this->bss_fields[BSF_FILTERED].set_width(14); + this->bss_fields[BSF_FILTERED].set_width(20); this->bss_fields[BSF_LOADING].set_width(13); this->bss_fields[BSF_LOADING].set_cylon(true); this->bss_fields[BSF_LOADING].right_justify(true); + this->bss_fields[BSF_HELP].set_width(14); + this->bss_fields[BSF_HELP].set_value("?:View Help"); + this->bss_fields[BSF_HELP].right_justify(true); }; virtual ~bottom_status_source() { }; @@ -111,10 +120,12 @@ public: void update_line_number(listview_curses *lc) { status_field &sf = this->bss_fields[BSF_LINE_NUMBER]; - if (lc->get_inner_height() == 0) + if (lc->get_inner_height() == 0) { sf.set_value("L0"); - else - sf.set_value("L%d", (int)lc->get_top()); + } + else { + sf.set_value("L%'d", (int)lc->get_top()); + } }; void update_percent(listview_curses *lc) { @@ -153,7 +164,7 @@ public: bookmark_vector::iterator iter; iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height); - sfw.set_value("%9dW", distance(iter, bv.end())); + sfw.set_value("%'9dW", distance(iter, bv.end())); } else { sfw.clear(); @@ -164,7 +175,7 @@ public: bookmark_vector::iterator iter; iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height); - sfe.set_value("%9dE", distance(iter, bv.end())); + sfe.set_value("%'9dE", distance(iter, bv.end())); } else { sfe.clear(); @@ -189,7 +200,7 @@ public: } sf.set_role(new_role); this->bss_error.clear(); - sf.set_value("%9d hits", tc->get_match_count()); + sf.set_value("%'9d hits", tc->get_match_count()); }; void update_loading(off_t off, size_t total) { @@ -218,7 +229,7 @@ public: if (lss.get_filtered_count() == 0) sf.clear(); else - sf.set_value("%d Not Shown", lss.get_filtered_count()); + sf.set_value("%'9d Not Shown", lss.get_filtered_count()); }; private: diff --git a/src/byte_array.hh b/src/byte_array.hh index 811b22d3..c562cddf 100644 --- a/src/byte_array.hh +++ b/src/byte_array.hh @@ -43,7 +43,11 @@ struct byte_array { bool operator<(const byte_array &other) const { return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) < 0; }; - + + bool operator!=(const byte_array &other) const { + return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) != 0; + }; + unsigned char ba_data[BYTE_COUNT]; }; diff --git a/src/collation-functions.cc b/src/collation-functions.cc new file mode 100644 index 00000000..da2d83e6 --- /dev/null +++ b/src/collation-functions.cc @@ -0,0 +1,161 @@ +/** + * Copyright (c) 2013, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file logfile_sub_source.hh + */ + +#include +#include +#include + +#include +#include + +#include + +extern "C" { + #include "strnatcmp.h" +} + +#define MAX_ADDR_LEN 128 + +static int strncmp2(int a_len, const char *a_str, + int b_len, const char *b_str) +{ + int retval = strncmp(a_str, b_str, std::min(a_len, b_len)); + + if (retval == 0) { + if (a_len < b_len) { + retval = -1; + } + else { + retval = 1; + } + } + return retval; +} + +static int try_inet_pton(int p_len, const char *p, char *n) +{ + static int family[] = { AF_INET6, AF_INET, AF_MAX }; + + char buf[MAX_ADDR_LEN]; + int retval = AF_MAX; + + strncpy(buf, p, p_len); + buf[p_len] = '\0'; + for (int lpc = 0; family[lpc] != AF_MAX; lpc++) { + if (inet_pton(family[lpc], buf, n) == 1) { + retval = family[lpc]; + break; + } + } + + return retval; +} + +static int convert_v6_to_v4(int family, char *n) +{ + struct in6_addr *ia = (struct in6_addr *)n; + + if (family == AF_INET6 && + (IN6_IS_ADDR_V4COMPAT(ia) || + IN6_IS_ADDR_V4MAPPED(ia))) { + family = AF_INET; + memmove(n, n + 12, sizeof(struct in_addr)); + } + + return family; +} + +static +int ipaddress(void *ptr, + int a_len, const void *a_in, + int b_len, const void *b_in) +{ + char a_addr[sizeof(struct in6_addr)], b_addr[sizeof(struct in6_addr)]; + const char *a_str = (const char *)a_in, *b_str = (const char *)b_in; + int a_family, b_family, retval; + + if (a_len > MAX_ADDR_LEN || b_len > MAX_ADDR_LEN) { + return strncmp2(a_len, a_str, b_len, b_str); + } + + a_family = try_inet_pton(a_len, a_str, a_addr); + b_family = try_inet_pton(b_len, b_str, b_addr); + + if (a_family == AF_MAX && b_family != AF_MAX) { + retval = -1; + } + else if (a_family != AF_MAX && b_family == AF_MAX) { + retval = 1; + } + else { + a_family = convert_v6_to_v4(a_family, a_addr); + b_family = convert_v6_to_v4(b_family, b_addr); + if (a_family == b_family) { + retval = memcmp(a_addr, b_addr, + a_family == AF_INET ? + sizeof(struct in_addr) : + sizeof(struct in6_addr)); + } + else if (a_family == AF_INET) { + retval = -1; + } + else { + retval = 1; + } + } + + return retval; +} + +static +int sql_strnatcmp(void *ptr, + int a_len, const void *a_in, + int b_len, const void *b_in) +{ + return strnatcmp(a_len, (char *)a_in, b_len, (char *)b_in); +} + +static +int sql_strnatcasecmp(void *ptr, + int a_len, const void *a_in, + int b_len, const void *b_in) +{ + return strnatcasecmp(a_len, (char *)a_in, b_len, (char *)b_in); +} + +int register_collation_functions(sqlite3 *db) +{ + sqlite3_create_collation(db, "ipaddress", SQLITE_UTF8, NULL, ipaddress); + sqlite3_create_collation(db, "natural", SQLITE_UTF8, NULL, sql_strnatcmp); + sqlite3_create_collation(db, "naturalnocase", SQLITE_UTF8, NULL, sql_strnatcasecmp); + + return 0; +} diff --git a/src/column_namer.hh b/src/column_namer.hh new file mode 100644 index 00000000..d24cd8f1 --- /dev/null +++ b/src/column_namer.hh @@ -0,0 +1,86 @@ +/** + * Copyright (c) 2013, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file column_namer.hh + */ + +#include +#include +#include +#include + +class column_namer { + +public: + column_namer() { + this->cn_builtin_names.push_back("col"); + }; + + bool existing_name(const std::string &in_name) { + if (find(this->cn_builtin_names.begin(), + this->cn_builtin_names.end(), + in_name) != this->cn_builtin_names.end()) { + return true; + } + else if (find(this->cn_names.begin(), + this->cn_names.end(), + in_name) != this->cn_names.end()) { + return true; + } + + return false; + }; + + std::string add_column(const std::string &in_name) { + std::string base_name = in_name, retval; + size_t buf_size; + char *buffer; + int num = 0; + + buf_size = in_name.length() + 64; + buffer = (char *)alloca(buf_size); + if (in_name == "") { + base_name = "col"; + } + + retval = base_name; + while (this->existing_name(retval)) { + snprintf(buffer, buf_size, "%s_%d", base_name.c_str(), num); + retval = buffer; + num += 1; + } + + this->cn_names.push_back(retval); + + return retval; + }; + + std::vector cn_builtin_names; + std::vector cn_names; + +}; diff --git a/src/data_parser.cc b/src/data_parser.cc index 85c80325..101b255b 100644 --- a/src/data_parser.cc +++ b/src/data_parser.cc @@ -33,259 +33,88 @@ using namespace std; -static data_token_t PATTERN_KEY[] = { - DT_STRING, - DT_NUMBER, - DT_HEX_NUMBER, - // DT_QUALIFIED_NAME, -}; +data_format data_parser::FORMAT_SEMI(DT_COMMA, DT_SEMI); +data_format data_parser::FORMAT_COMMA(DT_INVALID, DT_COMMA); +data_format data_parser::FORMAT_PLAIN(DT_INVALID, DT_INVALID); -static data_token_t UPTO_SEPARATOR[] = { - DT_SEPARATOR, - DT_LINE, -}; - -static data_token_t UPTO_NT[] = { - DNT_PAIR, - DNT_ROW, - DT_SEPARATOR, -}; - -static data_token_t PATTERN_PAIR[] = { - DNT_ROW, - DT_SEPARATOR, - // DNT_KEY, -}; - -static data_token_t PATTERN_ROW[] = { - DT_ANY, - DT_COMMA, - DNT_ROW, -}; - -static data_token_t PATTERN_DATE_TIME[] = { - DT_TIME, - DT_NUMBER, - DT_STRING, -}; - -static data_token_t PATTERN_QUAL[] = { - DNT_KEY, - DT_SEPARATOR, - DNT_KEY, -}; - -bool data_parser::reducePattern(std::list &reduction, - const data_token_t *pattern_start, - const data_token_t *pattern_end, - bool repeating) -{ - size_t pattern_size = (pattern_end - pattern_start); - bool found, retval = false; - - reduction.clear(); - - do { - found = false; - if (pattern_size <= this->dp_stack.size() && - std::equal(pattern_start, pattern_end, - this->dp_stack.begin(), - element_cmp())) { - std::list::iterator match_end = this->dp_stack.begin(); - - advance(match_end, pattern_size); - reduction.splice(reduction.end(), - this->dp_stack, - this->dp_stack.begin(), - match_end); - - retval = found = true; - } - } while (found && repeating); - - reduction.reverse(); - - return retval; -} - -void data_parser::reduceQual(const struct element &lookahead) -{ - std::list reduction; - - if (this->reducePattern(reduction, - PATTERN_QUAL, - PATTERN_QUAL + - sizeof(PATTERN_QUAL) / sizeof(data_token_t))) { - // printf("qual hit\n"); - this->dp_qual.splice(this->dp_qual.end(), reduction); - } -} - -void data_parser::reduceRow(void) -{ - std::list reduction; - - if (this->reducePattern(reduction, - PATTERN_ROW, - PATTERN_ROW + - sizeof(PATTERN_ROW) / sizeof(data_token_t))) { - std::list::iterator match_end; - - if (reduction.back().e_sub_elements != NULL) - reduction.front().assign_elements(*reduction.back().e_sub_elements); - else - reduction.front().e_sub_elements->push_back(reduction.back()); - reduction.front().update_capture(); - match_end = reduction.begin(); - ++match_end; - this->dp_stack.splice(this->dp_stack.begin(), - reduction, - reduction.begin(), - match_end); - } -} - -void data_parser::reducePair(void) +data_format_state_t dfs_semi_next(data_format_state_t state, + data_token_t next_token) { - std::list reduction; - - if (this->reducePattern(reduction, - PATTERN_DATE_TIME, - PATTERN_DATE_TIME + - sizeof(PATTERN_DATE_TIME) / sizeof(data_token_t))) { - this->dp_stack.push_front(element(reduction, DNT_DATE_TIME)); - this->dp_stack.front().assign_elements(reduction); - } - - this->reduceRow(); - if (this->reduceUpTo(reduction, - UPTO_SEPARATOR, - UPTO_SEPARATOR + - sizeof(UPTO_SEPARATOR) / sizeof(data_token_t)) && - !reduction.empty()) { - if (reduction.front().e_token == DNT_ROW) { - reduction.reverse(); - this->dp_stack.splice(this->dp_stack.begin(), reduction); - } - else { - this->dp_stack.push_front(element(reduction, DNT_ROW)); - this->dp_stack.front().assign_elements(reduction); + data_format_state_t retval = state; + + switch (state) { + case DFS_INIT: + switch (next_token) { + case DT_COMMA: + case DT_SEMI: + retval = DFS_ERROR; + break; + default: retval = DFS_KEY; break; + } + break; + case DFS_KEY: + switch (next_token) { + case DT_SEPARATOR: retval = DFS_VALUE; break; + case DT_SEMI: retval = DFS_ERROR; break; + default: break; + } + break; + case DFS_VALUE: + switch (next_token) { + case DT_SEMI: retval = DFS_INIT; break; + default: break; + } + break; + case DFS_ERROR: retval = DFS_ERROR; break; } - } - if (this->reducePattern(reduction, - PATTERN_PAIR, - PATTERN_PAIR + - sizeof(PATTERN_PAIR) / sizeof(data_token_t))) { - if (this->dp_qual.empty()) { - this->dp_stack.splice(this->dp_stack.begin(), reduction); - } - else { - reduction.push_front(this->dp_qual.back()); - this->dp_qual.pop_back(); - this->dp_stack.push_front(element(reduction, DNT_PAIR)); - this->dp_stack.front().assign_elements(reduction); - } - } - // this->print(stdout); + return retval; } -#define DEB 0 - -void data_parser::reduce(const element &lookahead) +data_format_state_t dfs_comma_next(data_format_state_t state, + data_token_t next_token) { - std::list reduction; - bool push_lookahead = true; - - switch (lookahead.e_token) { - case DT_INVALID: - case DT_WHITE: - this->reducePair(); - push_lookahead = false; - break; - - case DT_GARBAGE: - push_lookahead = false; - break; - - case DT_LINE: - this->reduceRow(); - if (!this->reduceUpTo(reduction, - UPTO_NT, - UPTO_NT + - sizeof(UPTO_NT) / sizeof(data_token_t))) { - reduction.splice(reduction.begin(), this->dp_stack); - reduction.reverse(); - } - if (!reduction.empty()) { - if (this->dp_stack.front().e_token == DNT_ROW) { - this->dp_stack.front().assign_elements(reduction); - } - else if (this->dp_stack.front().e_token == DNT_PAIR) { - this->dp_stack.front().e_sub_elements->back().assign_elements(reduction); - } - else { - this->dp_stack.push_front(element(reduction, DNT_ROW)); - this->dp_stack.front().assign_elements(reduction); - } + data_format_state_t retval = state; + + switch (state) { + case DFS_INIT: + switch (next_token) { + case DT_COMMA: + case DT_SEMI: + retval = DFS_ERROR; + break; + default: + retval = DFS_KEY; + break; + } + break; + case DFS_KEY: + switch (next_token) { + case DT_SEPARATOR: + retval = DFS_VALUE; + break; + case DT_SEMI: + case DT_COMMA: + retval = DFS_ERROR; + break; + default: break; + } + break; + case DFS_VALUE: + switch (next_token) { + case DT_COMMA: + retval = DFS_INIT; + break; + case DT_SEPARATOR: + retval = DFS_VALUE; + break; + default: break; + } + break; + case DFS_ERROR: + retval = DFS_ERROR; + break; } - this->reducePair(); - push_lookahead = false; - break; - - case DT_COMMA: - this->reduceRow(); - - if (!this->dp_stack.empty() && - this->dp_stack.front().e_token != DNT_ROW) { - if (this->dp_stack.front().e_token == DT_SEPARATOR) { - push_lookahead = false; - } - else if (this->dp_stack.front().e_token == DNT_PAIR) { - std::list::iterator pair_iter = this->dp_stack.begin(); - - this->dp_qual.push_front(this->dp_stack.front().e_sub_elements->front()); - this->dp_stack.front().e_sub_elements->pop_front(); - this->dp_stack.front().e_sub_elements->reverse(); - this->dp_stack.splice(this->dp_stack.begin(), - *this->dp_stack.front().e_sub_elements); - this->dp_stack.erase(pair_iter); - } - else { - std::list::iterator next_elem = this->dp_stack.begin(); - - advance(next_elem, 1); - reduction.splice(reduction.end(), - this->dp_stack, - this->dp_stack.begin(), - next_elem); - this->dp_stack.push_front(element(reduction, DNT_ROW)); - this->dp_stack.front().assign_elements(reduction); - } - } - break; - - case DT_SEPARATOR: - if (this->reduceAnyOf(reduction, - PATTERN_KEY, - PATTERN_KEY + - sizeof(PATTERN_KEY) / sizeof(data_token_t))) { - this->reducePair(); - if (this->dp_stack.front().e_token == DT_SEPARATOR) - this->dp_stack.pop_front(); - this->dp_qual.push_back(element(reduction, DNT_KEY)); - // this->reduceQual(lookahead); - } - break; - - default: - break; - } - - if (push_lookahead) { - this->dp_stack.push_front(lookahead); - } - - // this->print(stdout); + return retval; } diff --git a/src/data_parser.hh b/src/data_parser.hh index 4f6bafd5..e033bc51 100644 --- a/src/data_parser.hh +++ b/src/data_parser.hh @@ -32,49 +32,152 @@ #include +#include + #include +#include +#include #include #include "pcrepp.hh" +#include "byte_array.hh" #include "data_scanner.hh" -template -ForwardIterator1 find_first_not_of(ForwardIterator1 first, ForwardIterator1 last, - ForwardIterator2 s_first, ForwardIterator2 s_last, - BinaryPredicate p) +/** + * Switch to the 'parser' view mode when the user hits ';' so they + * can easily see what columns are available. + * + * select * from logline; + * select itemfrom(csv_key, 0) from logline; + * select itemfrom(csv_key, -1) from logline; + * select itemfrom(dict_key, "key") from logline; + * select itemfrom(dict_key, "key[0]") from logline; + * select itemfrom(csv_key, 0:3) from logline; support splices ? + * + * Add a command to create a logline table with a given name so the user can + * do joins across the tables: + * + * create-logline-table sudo_logline + * select * from logline, sudo_logline where sudo_logline.COMMAND=logline.COMMAND; + * + * select timestmap / 60 as minute, sc_status, count(*) from access_log + * group by minute, sc_status + * order by minute, sc_status desc; + * (use group_concat() here?) + * + * The "itemfrom()" function parses the group and lets you specify an + * expression to query the contents. + * + * For 'report-on' command: + * 'report-on PWD' + * select PWD,count(*) as amount from logline group by PWD order by amount desc; + * 'report-on num_col num_col2' + * select avg(num_col),stddev(num_col),... from logline; + * Instead of a command, we should automatically create views with the + * relevant select statements. + * + * Add a tojson() aggregate function to sqlite: + * select foo,tojson(bar) group by foo; + * + * 1 ["a", "b", "c"] + * 2 ["d", "e", "f"] + * + * We should automatically detect sqlite files provided on the command line + * and attach the database. + * + * add a 'metadata' view that has all the metadata crud (sql tables/log) + * + * Add support for sqlite_log that writes to a temp file and is displayed in the + * metadata view. + * + * Add a function that bookmarks all lines in the log view based on line_numbers + * in the sql query result. + * + * select line_number from logline where A="b"; + * hit 'y/Y' to move forward and backwards through sql results + * hit 'R' key and all the lines are bookmarked + * + * add path manipulation functions like basename, dirname, splitext + * + * use the vt52_curses emulation to embed a editor for editing queries. For + * example, you could hit 'ctrl+;' and it would split the window in half with + * the bottom being used for nano. When the file was written, lnav should + * notice and do a 'prepare' on the sql to make sure it is correct. + * + * Maybe add other tables for accessing lnav state. For example, you could do + * a query to find log lines of interest and then insert their line numbers + * into the 'bookmarks' table to create new user bookmarks. + */ + + +template +void strip(Container &container, UnaryPredicate p) { - for (; first != last; ++first) { - bool found = false; - - for (ForwardIterator2 it = s_first; it != s_last; ++it) { - if (p(*first, *it)) { - found = true; - } - } - if (!found) - return first; - } - return last; + while (!container.empty() && p(container.front())) { + container.pop_front(); + } + while (!container.empty() && p(container.back())) { + container.pop_back(); + } } +enum data_format_state_t { + DFS_ERROR = -1, + DFS_INIT, + DFS_KEY, + DFS_VALUE, +}; + +struct data_format { + data_format(data_token_t appender = DT_INVALID, + data_token_t terminator = DT_INVALID) + : df_appender(appender), df_terminator(terminator) + { + }; + + const data_token_t df_appender; + const data_token_t df_terminator; +}; + +data_format_state_t dfs_semi_next(data_format_state_t state, + data_token_t next_token); +data_format_state_t dfs_comma_next(data_format_state_t state, + data_token_t next_token); + class data_parser { public: + static data_format FORMAT_SEMI; + static data_format FORMAT_COMMA; + static data_format FORMAT_PLAIN; + + typedef byte_array<20> schema_id_t; + + struct element; + typedef std::list element_list_t; + struct element { element() : e_token(DT_INVALID), e_sub_elements(NULL) { }; - element(std::list &subs, data_token_t token) + element(element_list_t &subs, + data_token_t token, + bool assign_subs_elements = true) : e_capture(subs.front().e_capture.c_begin, subs.back().e_capture.c_end), e_token(token), e_sub_elements(NULL) { + if (assign_subs_elements) { + this->assign_elements(subs); + } }; element(const element &other) { - assert(other.e_sub_elements == NULL); + // assert(other.e_sub_elements == NULL); this->e_capture = other.e_capture; this->e_token = other.e_token; this->e_sub_elements = NULL; + if (other.e_sub_elements != NULL) + this->assign_elements(*other.e_sub_elements); }; ~element() { @@ -84,10 +187,19 @@ public: } }; - void assign_elements(std::list &subs) { + element& operator=(const element &other) { + this->e_capture = other.e_capture; + this->e_token = other.e_token; + this->e_sub_elements = NULL; + if (other.e_sub_elements != NULL) + this->assign_elements(*other.e_sub_elements); + return *this; + }; + + void assign_elements(element_list_t &subs) { if (this->e_sub_elements == NULL) - this->e_sub_elements = new std::list(); - this->e_sub_elements->splice(this->e_sub_elements->end(), subs); + this->e_sub_elements = new element_list_t(); + this->e_sub_elements->swap(subs); this->update_capture(); }; @@ -100,9 +212,22 @@ public: } }; + data_token_t value_token(void) const { + data_token_t retval = DT_INVALID; + + if (this->e_token == DNT_VALUE && + this->e_sub_elements != NULL && + this->e_sub_elements->size() == 1) { + retval = this->e_sub_elements->front().e_token; + } + return retval; + }; + void print(FILE *out, pcre_input &pi, int offset = 0) { + int lpc; + if (this->e_sub_elements != NULL) { - for (std::list::iterator iter2 = + for (element_list_t::iterator iter2 = this->e_sub_elements->begin(); iter2 != this->e_sub_elements->end(); ++iter2) { @@ -114,7 +239,7 @@ public: data_scanner::token2name(this->e_token), this->e_capture.c_begin, this->e_capture.c_end); - for (int lpc = 0; lpc < this->e_capture.c_end; lpc++) { + for (lpc = 0; lpc < this->e_capture.c_end; lpc++) { if (lpc == this->e_capture.c_begin) fputc('^', out); else if (lpc == (this->e_capture.c_end - 1)) @@ -124,13 +249,18 @@ public: else fputc(' ', out); } - fputc('\n', out); + for (; lpc < (int)pi.pi_length; lpc++) { + fputc(' ', out); + } + + std::string sub = pi.get_substr(&this->e_capture); + fprintf(out, " %s\n", sub.c_str()); }; pcre_context::capture_t e_capture; data_token_t e_token; - std::list *e_sub_elements; + element_list_t *e_sub_elements; }; struct element_cmp { @@ -154,94 +284,343 @@ public: data_token_t ei_token; }; - data_parser(data_scanner *ds) : dp_scanner(ds) { }; + data_parser(data_scanner *ds) : dp_format(NULL), dp_scanner(ds) { }; - void parse(void) { - pcre_context_static<30> pc; - struct element elem; - - while (this->dp_scanner->tokenize(pc, elem.e_token)) { - elem.e_capture = *(pc.begin()); + void pairup(schema_id_t *schema, element_list_t &pairs_out, element_list_t &in_list) { + element_list_t el_stack, free_row, key_comps, value, prefix; + SHA_CTX context; + + for (element_list_t::iterator iter = in_list.begin(); + iter != in_list.end(); + ++iter) { + if (iter->e_token == DNT_GROUP) { + element_list_t group_pairs; - this->reduce(elem); + this->pairup(NULL, group_pairs, *iter->e_sub_elements); + if (!group_pairs.empty()) { + iter->assign_elements(group_pairs); + } + } + + if (iter->e_token == DT_SEPARATOR) { + element_list_t::iterator key_iter = key_comps.end(); + bool found = false; + + --key_iter; + for (; + key_iter != key_comps.begin() && !found; + --key_iter) { + if (key_iter->e_token == this->dp_format->df_appender) { + ++key_iter; + value.splice(value.end(), + key_comps, + key_comps.begin(), + key_iter); + key_comps.splice(key_comps.begin(), + key_comps, + key_comps.end()); + key_comps.resize(1); + found = true; + } + else if (key_iter->e_token == this->dp_format->df_terminator) { + std::vector key_copy; + + value.splice(value.end(), + key_comps, + key_comps.begin(), + key_iter); + ++key_iter; + key_comps.pop_front(); + strip(key_comps, element_if(DT_WHITE)); + found = true; + } + } + if (!found && !el_stack.empty() && !key_comps.empty()) { + element_list_t::iterator value_iter; + + value.splice(value.end(), + key_comps, + key_comps.begin(), + key_comps.end()); + value_iter = value.end(); + std::advance(value_iter, -1); + key_comps.splice(key_comps.begin(), + value, + value_iter); + key_comps.resize(1); + } + strip(value, element_if(DT_WHITE)); + value.remove_if(element_if(DT_COMMA)); + if (!value.empty()) { + el_stack.push_back(element(value, DNT_VALUE)); + } + strip(key_comps, element_if(DT_WHITE)); + if (!key_comps.empty()) { + el_stack.push_back(element(key_comps, DNT_KEY, false)); + } + key_comps.clear(); + value.clear(); + } + else { + key_comps.push_back(*iter); + } } - }; - void reduce(const element &elem); + if (el_stack.empty()) { + free_row.splice(free_row.begin(), + key_comps, key_comps.begin(), key_comps.end()); + } + else { + value.splice(value.begin(), + key_comps, + key_comps.begin(), + key_comps.end()); + strip(value, element_if(DT_WHITE)); + value.remove_if(element_if(DT_COMMA)); + if (!value.empty()) { + el_stack.push_back(element(value, DNT_VALUE)); + } + } + + SHA_Init(&context); + while (!el_stack.empty()) { + element_list_t::iterator kv_iter = el_stack.begin(); + if (kv_iter->e_token == DNT_VALUE) { + free_row.push_back(el_stack.front()); + } + if (kv_iter->e_token != DNT_KEY) { + el_stack.pop_front(); + continue; + } + + ++kv_iter; + if (kv_iter == el_stack.end()) { + el_stack.pop_front(); + continue; + } - bool reducePattern(std::list &reduction, - const data_token_t *pattern_start, - const data_token_t *pattern_end, - bool repeating = false); + if (kv_iter->e_token != DNT_VALUE) { + el_stack.pop_front(); + continue; + } - bool reduceAnyOf(std::list &reduction, - const data_token_t *possibilities_start, - const data_token_t *possibilities_end) { - std::list::iterator iter; - bool retval = false; + std::string key_val = this->get_element_string(el_stack.front()); + element_list_t pair_subs; - reduction.clear(); + if (schema != NULL) { + SHA_Update(&context, key_val.c_str(), key_val.length()); + } - iter = find_first_not_of(this->dp_stack.begin(), - this->dp_stack.end(), - possibilities_start, possibilities_end, - element_cmp()); - if (iter != this->dp_stack.begin()) { - reduction.splice(reduction.end(), - this->dp_stack, - this->dp_stack.begin(), - iter); + ++kv_iter; + pair_subs.splice(pair_subs.begin(), + el_stack, + el_stack.begin(), + kv_iter); + pairs_out.push_back(element(pair_subs, DNT_PAIR)); + } - retval = true; + if (pairs_out.size() == 1) { + element &pair = pairs_out.front(); + element &value = pair.e_sub_elements->back(); + + if (value.e_token == DNT_VALUE && + value.e_sub_elements != NULL && + value.e_sub_elements->size() > 1) { + prefix.splice(prefix.begin(), + *pair.e_sub_elements, + pair.e_sub_elements->begin()); + free_row.clear(); + free_row.splice(free_row.begin(), + *value.e_sub_elements, + value.e_sub_elements->begin(), + value.e_sub_elements->end()); + pairs_out.clear(); + SHA_Init(&context); + } } - reduction.reverse(); - - return retval; + if (pairs_out.empty() && !free_row.empty()) { + while (!free_row.empty()) { + switch (free_row.front().e_token) { + case DNT_GROUP: + case DT_NUMBER: + case DT_SYMBOL: + case DT_HEX_NUMBER: + case DT_OCTAL_NUMBER: + case DT_VERSION_NUMBER: + case DT_QUOTED_STRING: + case DT_IPV4_ADDRESS: + case DT_IPV6_ADDRESS: + case DT_MAC_ADDRESS: + case DT_UUID: + case DT_URL: + case DT_PATH: + case DT_TIME: + case DT_PERCENTAGE: { + element_list_t pair_subs; + struct element blank; + + blank.e_capture.c_begin = blank.e_capture.c_end = free_row.front().e_capture.c_begin; + blank.e_token = DNT_KEY; + pair_subs.push_back(blank); + pair_subs.push_back(free_row.front()); + pairs_out.push_back(element(pair_subs, DNT_PAIR)); + } + break; + default: { + std::string key_val = this->get_element_string(free_row.front()); + + SHA_Update(&context, key_val.c_str(), key_val.length()); + } + break; + } + + free_row.pop_front(); + } + } + + if (!prefix.empty()) { + element_list_t pair_subs; + struct element blank; + + blank.e_capture.c_begin = blank.e_capture.c_end = prefix.front().e_capture.c_begin; + blank.e_token = DNT_KEY; + pair_subs.push_back(blank); + pair_subs.push_back(prefix.front()); + pairs_out.push_front(element(pair_subs, DNT_PAIR)); + } + + if (schema != NULL) + SHA_Final(this->dp_schema_id.ba_data, &context); }; + void discover_format(void) { + pcre_context_static<30> pc; + int hist[DT_TERMINAL_MAX]; + struct element elem; - bool reduceUpTo(std::list &reduction, - const data_token_t *possibilities_start, - const data_token_t *possibilities_end) { - std::list::iterator iter; - bool retval = false; + this->dp_group_token.push_back(DT_INVALID); + this->dp_group_stack.resize(1); - reduction.clear(); + data_format_state_t semi_state = DFS_INIT; + data_format_state_t comma_state = DFS_INIT; - iter = std::find_first_of(this->dp_stack.begin(), this->dp_stack.end(), - possibilities_start, possibilities_end, - element_cmp()); - if (iter != this->dp_stack.end()) { - reduction.splice(reduction.end(), - this->dp_stack, - this->dp_stack.begin(), - iter); + memset(hist, 0, sizeof(hist)); + while (this->dp_scanner->tokenize(pc, elem.e_token)) { + pcre_context::iterator pc_iter; + + pc_iter = std::find_if(pc.begin(), pc.end(), capture_if_not(-1)); + assert(pc_iter != pc.end()); + + elem.e_capture = *pc_iter; + + assert(elem.e_capture.c_begin != -1); + assert(elem.e_capture.c_end != -1); + + semi_state = dfs_semi_next(semi_state, elem.e_token); + comma_state = dfs_comma_next(comma_state, elem.e_token); + hist[elem.e_token] += 1; + switch (elem.e_token) { + case DT_LPAREN: + case DT_LANGLE: + case DT_LCURLY: + case DT_LSQUARE: + this->dp_group_token.push_back(elem.e_token); + this->dp_group_stack.push_back(element_list_t()); + break; + + case DT_RPAREN: + case DT_RANGLE: + case DT_RCURLY: + case DT_RSQUARE: + if (this->dp_group_token.back() == (elem.e_token - 1)) { + this->dp_group_token.pop_back(); + + std::list::reverse_iterator riter = this->dp_group_stack.rbegin(); + ++riter; + if (!this->dp_group_stack.back().empty()) { + (*riter).push_back(element(this->dp_group_stack.back(), DNT_GROUP)); + } + this->dp_group_stack.pop_back(); + } + else { + this->dp_group_stack.back().push_back(elem); + } + break; + default: + this->dp_group_stack.back().push_back(elem); + break; + } + } + + while (this->dp_group_stack.size() > 1) { + this->dp_group_token.pop_back(); - retval = true; + std::list::reverse_iterator riter = this->dp_group_stack.rbegin(); + ++riter; + if (!this->dp_group_stack.back().empty()) { + (*riter).push_back(element(this->dp_group_stack.back(), DNT_GROUP)); + } + this->dp_group_stack.pop_back(); } - reduction.reverse(); + if (semi_state != DFS_ERROR && hist[DT_SEMI]) { + this->dp_format = &FORMAT_SEMI; + } + else if (comma_state != DFS_ERROR) { + this->dp_format = &FORMAT_COMMA; + } + else { + this->dp_format = &FORMAT_PLAIN; + } + + }; + + void parse(void) { - return retval; + this->discover_format(); + + this->pairup(&this->dp_schema_id, + this->dp_pairs, + this->dp_group_stack.front()); + + for (element_list_t::iterator iter = this->dp_pairs.begin(); + iter != this->dp_pairs.end(); + ++iter) { + if (iter->e_token == DNT_PAIR) { + element_list_t &pair_subs = *iter->e_sub_elements; + std::string key_val = this->get_element_string(pair_subs.front()); + + } + } + + }; - void reduceQual(const struct element &lookahead); - void reduceRow(void); - void reducePair(void); + std::string get_element_string(element &elem) { + pcre_input &pi = this->dp_scanner->get_input(); + + return pi.get_substr(&elem.e_capture); + }; - void print(FILE *out) { + void print(FILE *out, element_list_t &el) { fprintf(out, " %s\n", this->dp_scanner->get_input().get_string()); - for (std::list::iterator iter = this->dp_stack.begin(); - iter != this->dp_stack.end(); + for (element_list_t::iterator iter = el.begin(); + iter != el.end(); ++iter) { iter->print(out, this->dp_scanner->get_input()); } }; - std::list dp_qual; - std::list dp_stack; + std::vector dp_group_token; + std::list dp_group_stack; + + element_list_t dp_errors; + + element_list_t dp_pairs; + schema_id_t dp_schema_id; + data_format *dp_format; private: data_scanner *dp_scanner; diff --git a/src/data_scanner.cc b/src/data_scanner.cc index e8b2c639..a5ced657 100644 --- a/src/data_scanner.cc +++ b/src/data_scanner.cc @@ -29,6 +29,8 @@ #include "config.h" +#include + #include "pcrepp.hh" #include "data_scanner.hh" @@ -38,33 +40,50 @@ static struct { const char *name; pcrepp pcre; } MATCHERS[DT_TERMINAL_MAX] = { - { "url", pcrepp("([\\w]+://[^\\s'\"\\[\\](){}]+)"), }, - { "path", pcrepp("(?)"), }, { "ipv4", pcrepp("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})"), }, + { "uuid", pcrepp("([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})"), }, { "vers", pcrepp("([0-9]+(?:\\.[0-9]+){2,}\\b)"), }, { "oct", pcrepp("(-?0[0-7]+\\b)"), }, { "pcnt", pcrepp("(-?[0-9]+(\\.[0-9]+)?[ ]*%\\b)"), }, { "num", pcrepp("(-?[0-9]+(\\.[0-9]+)?([eE][-+][0-9]+)?\\b)"), }, { "hex", pcrepp("(-?(?:0x|[0-9])[0-9a-fA-F]+\\b)"), }, - - { "word", pcrepp("([^\"';\\s:=,/(){}\\[\\]]+)"), }, + + { "word", pcrepp("([a-zA-Z][a-z']+(?=[\\s\\(\\)!\\*:;'\\\"\\?,]|\\.\\s|$))"), }, + { "sym", pcrepp("([^\";\\s:=,/(){}\\[\\]]+)"), }, { "line", pcrepp("(\r?\n|\r|;)"), }, - { "whit", pcrepp("([ \r\t]+)"), }, + { "wspc", pcrepp("([ \r\t]+)"), }, { "dot", pcrepp("(\\.)"), }, - + { "gbg", pcrepp("(.)"), }, }; -const char *DNT_NAMES[] = { +const char *DNT_NAMES[DNT_MAX - DNT_KEY] = { "key", "pair", "val", @@ -74,6 +93,7 @@ const char *DNT_NAMES[] = { "var", "rang", "date", + "grp", }; const char *data_scanner::token2name(data_token_t token) @@ -88,6 +108,24 @@ const char *data_scanner::token2name(data_token_t token) return DNT_NAMES[token - DNT_KEY]; } +static +bool find_string_end(const char *str, size_t &start, size_t length, char term) +{ + for (; start < length; start++) { + if (str[start] == term) { + start += 1; + return true; + } + if (str[start] == '\\') { + if (start + 1 >= length) { + return false; + } + start += 1; + } + } + return false; +} + bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out) { int lpc; @@ -102,14 +140,74 @@ bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out) this->ds_pcre_input.pi_next_offset += 1; token_out = DT_LINE; - return true; + return false; } for (lpc = 0; lpc < DT_TERMINAL_MAX; lpc++) { - if (MATCHERS[lpc].pcre.match(pc, this->ds_pcre_input, PCRE_ANCHORED)) { - token_out = data_token_t(lpc); - break; - } + switch (lpc) { + case DT_QUOTED_STRING: { + pcre_input &pi = this->ds_pcre_input; + const char *str = pi.get_string(); + size_t str_start, str_end; + bool found = false; + + pi.pi_offset = pi.pi_next_offset; + str_end = str_start = pi.pi_offset + 1; + switch (str[pi.pi_offset]) { + case 'u': + case 'r': + if (pi.pi_offset + 1 < pi.pi_length && + (str[pi.pi_offset + 1] == '\'' || + str[pi.pi_offset + 1] == '\"')) { + str_start += 1; + str_end += 1; + found = find_string_end(str, + str_end, + pi.pi_length, + str[pi.pi_offset]); + } + break; + case '\'': + case '\"': + found = find_string_end(str, + str_end, + pi.pi_length, + str[pi.pi_offset]); + break; + } + if (found) { + token_out = data_token_t(DT_QUOTED_STRING); + pi.pi_next_offset = str_end; + pc.all()[0].c_begin = pi.pi_offset; + pc.all()[0].c_end = str_end; + pc.all()[1].c_begin = str_start; + pc.all()[1].c_end = str_end - 1; + pc.set_count(2); + return true; + } + } + break; + default: + if (MATCHERS[lpc].pcre.match(pc, this->ds_pcre_input, PCRE_ANCHORED)) { + switch (lpc) { + case DT_IPV6_ADDRESS: { + std::string addr = this->ds_pcre_input.get_substr(pc.all()); + char buf[sizeof(struct in6_addr)]; + + if (inet_pton(AF_INET6, addr.c_str(), buf) == 1) { + token_out = data_token_t(lpc); + return true; + } + this->ds_pcre_input.pi_next_offset = this->ds_pcre_input.pi_offset; + break; + } + default: + token_out = data_token_t(lpc); + return true; + } + } + break; + } } assert((0 <= token_out && token_out < DT_TERMINAL_MAX)); diff --git a/src/data_scanner.hh b/src/data_scanner.hh index 7cddd6dc..485bec55 100644 --- a/src/data_scanner.hh +++ b/src/data_scanner.hh @@ -36,18 +36,33 @@ enum data_token_t { DT_INVALID = -1, - - DT_URL = 0, + + DT_QUOTED_STRING = 0, + DT_URL, DT_PATH, - DT_TIME, DT_MAC_ADDRESS, - DT_QUOTED_STRING, + DT_TIME, + DT_IPV6_ADDRESS, // DT_QUALIFIED_NAME, DT_SEPARATOR, DT_COMMA, + DT_SEMI, + + DT_LCURLY, + DT_RCURLY, - DT_IP_ADDRESS, + DT_LSQUARE, + DT_RSQUARE, + + DT_LPAREN, + DT_RPAREN, + + DT_LANGLE, + DT_RANGLE, + + DT_IPV4_ADDRESS, + DT_UUID, DT_VERSION_NUMBER, DT_OCTAL_NUMBER, @@ -55,7 +70,8 @@ enum data_token_t { DT_NUMBER, DT_HEX_NUMBER, - DT_STRING, + DT_WORD, + DT_SYMBOL, DT_LINE, DT_WHITE, DT_DOT, @@ -73,6 +89,9 @@ enum data_token_t { DNT_VARIABLE_KEY, DNT_ROWRANGE, DNT_DATE_TIME, + DNT_GROUP, + + DNT_MAX, DT_ANY = 100, }; @@ -84,6 +103,9 @@ public: data_scanner(const std::string &line) : ds_line(line), ds_pcre_input(ds_line.c_str()) { + if (!line.empty() && line[line.length() - 1] == '.') { + this->ds_pcre_input.pi_length -= 1; + } }; bool tokenize(pcre_context &pc, data_token_t &token_out); diff --git a/src/db_sub_source.hh b/src/db_sub_source.hh index d74a5137..61322a60 100644 --- a/src/db_sub_source.hh +++ b/src/db_sub_source.hh @@ -33,7 +33,9 @@ #include #include +#include "listview_curses.hh" #include "hist_source.hh" +#include "log_vtab_impl.hh" class db_label_source : public hist_source::label_source { public: @@ -43,16 +45,6 @@ public: void hist_label_for_group(int group, std::string &label_out) { label_out.clear(); - for (int lpc = 0; lpc < (int)this->dls_headers.size(); lpc++) { - int before, total_fill = - this->dls_column_sizes[lpc] - this->dls_headers[lpc].length(); - - before = total_fill / 2; - total_fill -= before; - label_out.append(before, ' '); - label_out.append(this->dls_headers[lpc]); - label_out.append(total_fill, ' '); - } }; void hist_label_for_bucket(int bucket_start_value, @@ -67,11 +59,42 @@ public: if (bucket_start_value >= (int)this->dls_rows.size()) return; for (int lpc = 0; lpc < (int)this->dls_rows[bucket_start_value].size(); lpc++) { - label_out.append(this->dls_column_sizes[lpc] - this->dls_rows[bucket_start_value][lpc].length(), ' '); + int padding = (this->dls_column_sizes[lpc] - + this->dls_rows[bucket_start_value][lpc].length() - + 1); + + if (this->dls_column_types[lpc] != SQLITE3_TEXT) { + label_out.append(padding, ' '); + } label_out.append(this->dls_rows[bucket_start_value][lpc]); + if (this->dls_column_types[lpc] == SQLITE3_TEXT) { + label_out.append(padding, ' '); + } + label_out.append(1, ' '); } }; + void hist_attrs_for_bucket(int bucket_start_value, + const hist_source::bucket_t &bucket, + string_attrs_t &sa) { + struct line_range lr = { 0, 0 }; + struct line_range lr2 = { 0, -1 }; + + if (bucket_start_value >= (int)this->dls_rows.size()) + return; + for (size_t lpc = 0; lpc < this->dls_column_sizes.size() - 1; lpc++) { + if (bucket_start_value % 2 == 0) { + sa[lr2].insert(make_string_attr("style", A_BOLD)); + } + lr.lr_start += this->dls_column_sizes[lpc] - 1; + lr.lr_end = lr.lr_start + 1; + sa[lr].insert(make_string_attr("graphic", ACS_VLINE)); + lr.lr_start += 1; + } + } + + // TODO: add support for left and right justification... numbers should + // be right justified and strings should be left. void push_column(const char *colstr) { int index = this->dls_rows.back().size(); @@ -83,7 +106,7 @@ public: std::max(this->dls_column_sizes[index], strlen(colstr) + 1); }; - void push_header(const std::string &colstr) { + void push_header(const std::string &colstr, int type, bool graphable) { int index = this->dls_headers.size(); this->dls_headers.push_back(colstr); @@ -92,17 +115,74 @@ public: } this->dls_column_sizes[index] = std::max(this->dls_column_sizes[index], colstr.length() + 1); + this->dls_column_types.push_back(type); + this->dls_headers_to_graph.push_back(graphable); } void clear(void) { this->dls_headers.clear(); + this->dls_headers_to_graph.clear(); + this->dls_column_types.clear(); this->dls_rows.clear(); this->dls_column_sizes.clear(); } std::vector< std::string > dls_headers; + std::vector< bool > dls_headers_to_graph; + std::vector dls_column_types; std::vector< std::vector< std::string > > dls_rows; std::vector< size_t > dls_column_sizes; }; +class db_overlay_source : public list_overlay_source { +public: + db_overlay_source() : dos_labels(NULL) { }; + + size_t list_overlay_count(const listview_curses &lv) { + return 1; + }; + + bool list_value_for_overlay(const listview_curses &lv, + vis_line_t y, + attr_line_t &value_out) { + view_colors &vc = view_colors::singleton(); + if (y != 0) { + return false; + } + + std::string &line = value_out.get_string(); + db_label_source *dls = this->dos_labels; + + for (size_t lpc = 0; + lpc < this->dos_labels->dls_column_sizes.size(); + lpc++) { + int before, total_fill = + dls->dls_column_sizes[lpc] - dls->dls_headers[lpc].length(); + + struct line_range header_range = { line.length(), line.length() + dls->dls_column_sizes[lpc] }; + + int attrs = vc.attrs_for_role(this->dos_hist_source->get_role_for_type(bucket_type_t(lpc))) | A_UNDERLINE; + if (!this->dos_labels->dls_headers_to_graph[lpc]) { + attrs = A_UNDERLINE; + } + value_out.get_attrs()[header_range].insert(make_string_attr("style", attrs)); + + before = total_fill / 2; + total_fill -= before; + line.append(before, ' '); + line.append(dls->dls_headers[lpc]); + line.append(total_fill, ' '); + } + + struct line_range lr = { 0, -1 }; + + value_out.get_attrs()[lr].insert(make_string_attr("style", A_BOLD | A_UNDERLINE)); + + return true; + }; + + db_label_source *dos_labels; + hist_source *dos_hist_source; +}; + #endif diff --git a/src/extension-functions.c b/src/extension-functions.c new file mode 100644 index 00000000..411ecd05 --- /dev/null +++ b/src/extension-functions.c @@ -0,0 +1,1945 @@ +/* +This library will provide common mathematical and string functions in +SQL queries using the operating system libraries or provided +definitions. It includes the following functions: + +Math: acos, asin, atan, atn2, atan2, acosh, asinh, atanh, difference, +degrees, radians, cos, sin, tan, cot, cosh, sinh, tanh, coth, exp, +log, log10, power, sign, sqrt, square, ceil, floor, pi. + +String: replicate, charindex, leftstr, rightstr, ltrim, rtrim, trim, +replace, reverse, proper, padl, padr, padc, strfilter. + +Aggregate: stdev, variance, mode, median, lower_quartile, +upper_quartile. + +The string functions ltrim, rtrim, trim, replace are included in +recent versions of SQLite and so by default do not build. + +Compilation instructions: + Compile this C source file into a dynamic library as follows: + * Linux: + gcc -fPIC -lm -shared extension-functions.c -o libsqlitefunctions.so + * Mac OS X: + gcc -fno-common -dynamiclib extension-functions.c -o libsqlitefunctions.dylib + (You may need to add flags + -I /opt/local/include/ -L/opt/local/lib -lsqlite3 + if your sqlite3 is installed from Mac ports, or + -I /sw/include/ -L/sw/lib -lsqlite3 + if installed with Fink.) + * Windows: + 1. Install MinGW (http://www.mingw.org/) and you will get the gcc + (gnu compiler collection) + 2. add the path to your path variable (isn't done during the + installation!) + 3. compile: + gcc -shared -I "path" -o libsqlitefunctions.so extension-functions.c + (path = path of sqlite3ext.h; i.e. C:\programs\sqlite) + +Usage instructions for applications calling the sqlite3 API functions: + In your application, call sqlite3_enable_load_extension(db,1) to + allow loading external libraries. Then load the library libsqlitefunctions + using sqlite3_load_extension; the third argument should be 0. + See http://www.sqlite.org/cvstrac/wiki?p=LoadableExtensions. + Select statements may now use these functions, as in + SELECT cos(radians(inclination)) FROM satsum WHERE satnum = 25544; + +Usage instructions for the sqlite3 program: + If the program is built so that loading extensions is permitted, + the following will work: + sqlite> SELECT load_extension('./libsqlitefunctions.so'); + sqlite> select cos(radians(45)); + 0.707106781186548 + Note: Loading extensions is by default prohibited as a + security measure; see "Security Considerations" in + http://www.sqlite.org/cvstrac/wiki?p=LoadableExtensions. + If the sqlite3 program and library are built this + way, you cannot use these functions from the program, you + must write your own program using the sqlite3 API, and call + sqlite3_enable_load_extension as described above, or else + rebuilt the sqlite3 program to allow loadable extensions. + +Alterations: +The instructions are for Linux, Mac OS X, and Windows; users of other +OSes may need to modify this procedure. In particular, if your math +library lacks one or more of the needed trig or log functions, comment +out the appropriate HAVE_ #define at the top of file. If you do not +wish to make a loadable module, comment out the define for +COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE. If you are using a +version of SQLite without the trim functions and replace, comment out +the HAVE_TRIM #define. + +Liam Healy + +History: +2010-01-06 Correct check for argc in squareFunc, and add Windows +compilation instructions. +2009-06-24 Correct check for argc in properFunc. +2008-09-14 Add check that memory was actually allocated after +sqlite3_malloc or sqlite3StrDup, call sqlite3_result_error_nomem if +not. Thanks to Robert Simpson. +2008-06-13 Change to instructions to indicate use of the math library +and that program might work. +2007-10-01 Minor clarification to instructions. +2007-09-29 Compilation as loadable module is optional with +COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE. +2007-09-28 Use sqlite3_extension_init and macros +SQLITE_EXTENSION_INIT1, SQLITE_EXTENSION_INIT2, so that it works with +sqlite3_load_extension. Thanks to Eric Higashino and Joe Wilson. +New instructions for Mac compilation. +2007-09-17 With help from Joe Wilson and Nuno Luca, made use of +external interfaces so that compilation is no longer dependent on +SQLite source code. Merged source, header, and README into a single +file. Added casts so that Mac will compile without warnings (unsigned +and signed char). +2007-09-05 Included some definitions from sqlite 3.3.13 so that this +will continue to work in newer versions of sqlite. Completed +description of functions available. +2007-03-27 Revised description. +2007-03-23 Small cleanup and a bug fix on the code. This was mainly +letting errno flag errors encountered in the math library and checking +the result, rather than pre-checking. This fixes a bug in power that +would cause an error if any non-positive number was raised to any +power. +2007-02-07 posted by Mikey C to sqlite mailing list. +Original code 2006 June 05 by relicoder. + +*/ + +//#include "config.h" + +// #define COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE 1 +#define HAVE_ACOSH 1 +#define HAVE_ASINH 1 +#define HAVE_ATANH 1 +#define HAVE_SINH 1 +#define HAVE_COSH 1 +#define HAVE_TANH 1 +#define HAVE_LOG10 1 +#define HAVE_ISBLANK 1 +#define SQLITE_SOUNDEX 1 +#define HAVE_TRIM 1 /* LMH 2007-03-25 if sqlite has trim functions */ + +#ifdef COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#else +#include "sqlite3.h" +#endif + +#include +/* relicoder */ +#include +#include +#include +#include /* LMH 2007-03-25 */ + +#include +#include + +#ifndef _MAP_H_ +#define _MAP_H_ + +#include + +/* +** Simple binary tree implementation to use in median, mode and quartile calculations +** Tree is not necessarily balanced. That would require something like red&black trees of AVL +*/ + +typedef int(*cmp_func)(const void *, const void *); +typedef void(*map_iterator)(void*, int64_t, void*); + +typedef struct node{ + struct node *l; + struct node *r; + void* data; + int64_t count; +} node; + +typedef struct map{ + node *base; + cmp_func cmp; + short free; +} map; + +/* +** creates a map given a comparison function +*/ +map map_make(cmp_func cmp); + +/* +** inserts the element e into map m +*/ +void map_insert(map *m, void *e); + +/* +** executes function iter over all elements in the map, in key increasing order +*/ +void map_iterate(map *m, map_iterator iter, void* p); + +/* +** frees all memory used by a map +*/ +void map_destroy(map *m); + +/* +** compares 2 integers +** to use with map_make +*/ +int int_cmp(const void *a, const void *b); + +/* +** compares 2 doubles +** to use with map_make +*/ +int double_cmp(const void *a, const void *b); + +#endif /* _MAP_H_ */ + +typedef uint8_t u8; +typedef uint16_t u16; +typedef int64_t i64; + +static char *sqlite3StrDup( const char *z ) { + char *res = sqlite3_malloc( strlen(z)+1 ); + return strcpy( res, z ); +} + +/* +** These are copied verbatim from fun.c so as to not have the names exported +*/ + +/* LMH from sqlite3 3.3.13 */ +/* +** This table maps from the first byte of a UTF-8 character to the number +** of trailing bytes expected. A value '4' indicates that the table key +** is not a legal first byte for a UTF-8 character. +*/ +static const u8 xtra_utf8_bytes[256] = { +/* 0xxxxxxx */ +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + +/* 10wwwwww */ +4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, +4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, +4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, +4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + +/* 110yyyyy */ +1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + +/* 1110zzzz */ +2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + +/* 11110yyy */ +3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, +}; + + +/* +** This table maps from the number of trailing bytes in a UTF-8 character +** to an integer constant that is effectively calculated for each character +** read by a naive implementation of a UTF-8 character reader. The code +** in the READ_UTF8 macro explains things best. +*/ +static const int xtra_utf8_bits[] = { + 0, + 12416, /* (0xC0 << 6) + (0x80) */ + 925824, /* (0xE0 << 12) + (0x80 << 6) + (0x80) */ + 63447168 /* (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ +}; + +/* +** If a UTF-8 character contains N bytes extra bytes (N bytes follow +** the initial byte so that the total character length is N+1) then +** masking the character with utf8_mask[N] must produce a non-zero +** result. Otherwise, we have an (illegal) overlong encoding. +*/ +static const int utf_mask[] = { + 0x00000000, + 0xffffff80, + 0xfffff800, + 0xffff0000, +}; + +/* LMH salvaged from sqlite3 3.3.13 source code src/utf.c */ +#define READ_UTF8(zIn, c) { \ + int xtra; \ + c = *(zIn)++; \ + xtra = xtra_utf8_bytes[c]; \ + switch( xtra ){ \ + case 4: c = (int)0xFFFD; break; \ + case 3: c = (c<<6) + *(zIn)++; \ + case 2: c = (c<<6) + *(zIn)++; \ + case 1: c = (c<<6) + *(zIn)++; \ + c -= xtra_utf8_bits[xtra]; \ + if( (utf_mask[xtra]&c)==0 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } \ +} + +static int sqlite3ReadUtf8(const unsigned char *z){ + int c; + READ_UTF8(z, c); + return c; +} + +#define SKIP_UTF8(zIn) { \ + zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1); \ +} + +/* +** pZ is a UTF-8 encoded unicode string. If nByte is less than zero, +** return the number of unicode characters in pZ up to (but not including) +** the first 0x00 byte. If nByte is not less than zero, return the +** number of unicode characters in the first nByte of pZ (or up to +** the first 0x00, whichever comes first). +*/ +static int sqlite3Utf8CharLen(const char *z, int nByte){ + int r = 0; + const char *zTerm; + if( nByte>=0 ){ + zTerm = &z[nByte]; + }else{ + zTerm = (const char *)(-1); + } + assert( z<=zTerm ); + while( *z!=0 && z 0) ? 1: ( iVal < 0 ) ? -1: 0; + sqlite3_result_int64(context, iVal); + break; + } + case SQLITE_NULL: { + sqlite3_result_null(context); + break; + } + default: { + /* 2nd change below. Line for abs was: if( rVal<0 ) rVal = rVal * -1.0; */ + + rVal = sqlite3_value_double(argv[0]); + rVal = ( rVal > 0) ? 1: ( rVal < 0 ) ? -1: 0; + sqlite3_result_double(context, rVal); + break; + } + } +} + + +/* +** smallest integer value not less than argument +*/ +static void ceilFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + double rVal=0.0; + assert( argc==1 ); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_value_int64(argv[0]); + sqlite3_result_int64(context, iVal); + break; + } + case SQLITE_NULL: { + sqlite3_result_null(context); + break; + } + default: { + rVal = sqlite3_value_double(argv[0]); + sqlite3_result_int64(context, (i64) ceil(rVal)); + break; + } + } +} + +/* +** largest integer value not greater than argument +*/ +static void floorFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + double rVal=0.0; + assert( argc==1 ); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_value_int64(argv[0]); + sqlite3_result_int64(context, iVal); + break; + } + case SQLITE_NULL: { + sqlite3_result_null(context); + break; + } + default: { + rVal = sqlite3_value_double(argv[0]); + sqlite3_result_int64(context, (i64) floor(rVal)); + break; + } + } +} + +/* +** Given a string (s) in the first argument and an integer (n) in the second returns the +** string that constains s contatenated n times +*/ +static void replicateFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + unsigned char *z; /* input string */ + unsigned char *zo; /* result string */ + i64 iCount; /* times to repeat */ + i64 nLen; /* length of the input string (no multibyte considerations) */ + i64 nTLen; /* length of the result string (no multibyte considerations) */ + i64 i=0; + + if( argc!=2 || SQLITE_NULL==sqlite3_value_type(argv[0]) ) + return; + + iCount = sqlite3_value_int64(argv[1]); + + if( iCount<0 ){ + sqlite3_result_error(context, "domain error", -1); + }else{ + + nLen = sqlite3_value_bytes(argv[0]); + nTLen = nLen*iCount; + z=sqlite3_malloc(nTLen+1); + zo=sqlite3_malloc(nLen+1); + if (!z || !zo){ + sqlite3_result_error_nomem(context); + if (z) sqlite3_free(z); + if (zo) sqlite3_free(zo); + return; + } + strcpy((char*)zo, (char*)sqlite3_value_text(argv[0])); + + for(i=0; i=n it's a NOP +** padl(NULL) = NULL +*/ +static void padlFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + i64 ilen; /* length to pad to */ + i64 zl; /* length of the input string (UTF-8 chars) */ + int i = 0; + const char *zi; /* input string */ + char *zo; /* output string */ + char *zt; + + assert( argc==2 ); + + if( sqlite3_value_type(argv[0]) == SQLITE_NULL ){ + sqlite3_result_null(context); + }else{ + zi = (char *)sqlite3_value_text(argv[0]); + ilen = sqlite3_value_int64(argv[1]); + /* check domain */ + if(ilen<0){ + sqlite3_result_error(context, "domain error", -1); + return; + } + zl = sqlite3Utf8CharLen(zi, -1); + if( zl>=ilen ){ + /* string is longer than the requested pad length, return the same string (dup it) */ + zo = sqlite3StrDup(zi); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + }else{ + zo = sqlite3_malloc(strlen(zi)+ilen-zl+1); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + zt = zo; + for(i=1; i+zl<=ilen; ++i){ + *(zt++)=' '; + } + /* no need to take UTF-8 into consideration here */ + strcpy(zt,zi); + } + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + sqlite3_free(zo); + } +} + +/* +** given an input string (s) and an integer (n) appends spaces at the end of s +** until it has a length of n characters. +** When s has a length >=n it's a NOP +** padl(NULL) = NULL +*/ +static void padrFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + i64 ilen; /* length to pad to */ + i64 zl; /* length of the input string (UTF-8 chars) */ + i64 zll; /* length of the input string (bytes) */ + int i = 0; + const char *zi; /* input string */ + char *zo; /* output string */ + char *zt; + + assert( argc==2 ); + + if( sqlite3_value_type(argv[0]) == SQLITE_NULL ){ + sqlite3_result_null(context); + }else{ + zi = (char *)sqlite3_value_text(argv[0]); + ilen = sqlite3_value_int64(argv[1]); + /* check domain */ + if(ilen<0){ + sqlite3_result_error(context, "domain error", -1); + return; + } + zl = sqlite3Utf8CharLen(zi, -1); + if( zl>=ilen ){ + /* string is longer than the requested pad length, return the same string (dup it) */ + zo = sqlite3StrDup(zi); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + }else{ + zll = strlen(zi); + zo = sqlite3_malloc(zll+ilen-zl+1); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + zt = strcpy(zo,zi)+zll; + for(i=1; i+zl<=ilen; ++i){ + *(zt++) = ' '; + } + *zt = '\0'; + } + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + sqlite3_free(zo); + } +} + +/* +** given an input string (s) and an integer (n) appends spaces at the end of s +** and adds spaces at the begining of s until it has a length of n characters. +** Tries to add has many characters at the left as at the right. +** When s has a length >=n it's a NOP +** padl(NULL) = NULL +*/ +static void padcFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + i64 ilen; /* length to pad to */ + i64 zl; /* length of the input string (UTF-8 chars) */ + i64 zll; /* length of the input string (bytes) */ + int i = 0; + const char *zi; /* input string */ + char *zo; /* output string */ + char *zt; + + assert( argc==2 ); + + if( sqlite3_value_type(argv[0]) == SQLITE_NULL ){ + sqlite3_result_null(context); + }else{ + zi = (char *)sqlite3_value_text(argv[0]); + ilen = sqlite3_value_int64(argv[1]); + /* check domain */ + if(ilen<0){ + sqlite3_result_error(context, "domain error", -1); + return; + } + zl = sqlite3Utf8CharLen(zi, -1); + if( zl>=ilen ){ + /* string is longer than the requested pad length, return the same string (dup it) */ + zo = sqlite3StrDup(zi); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + }else{ + zll = strlen(zi); + zo = sqlite3_malloc(zll+ilen-zl+1); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + zt = zo; + for(i=1; 2*i+zl<=ilen; ++i){ + *(zt++) = ' '; + } + strcpy(zt, zi); + zt+=zll; + for(; i+zl<=ilen; ++i){ + *(zt++) = ' '; + } + *zt = '\0'; + } + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + sqlite3_free(zo); + } +} + +/* +** given 2 string (s1,s2) returns the string s1 with the characters NOT in s2 removed +** assumes strings are UTF-8 encoded +*/ +static void strfilterFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *zi1; /* first parameter string (searched string) */ + const char *zi2; /* second parameter string (vcontains valid characters) */ + const char *z1; + const char *z21; + const char *z22; + char *zo; /* output string */ + char *zot; + int c1 = 0; + int c2 = 0; + + assert( argc==2 ); + + if( sqlite3_value_type(argv[0]) == SQLITE_NULL || sqlite3_value_type(argv[1]) == SQLITE_NULL ){ + sqlite3_result_null(context); + }else{ + zi1 = (char *)sqlite3_value_text(argv[0]); + zi2 = (char *)sqlite3_value_text(argv[1]); + /* + ** maybe I could allocate less, but that would imply 2 passes, rather waste + ** (possibly) some memory + */ + zo = sqlite3_malloc(strlen(zi1)+1); + if (!zo){ + sqlite3_result_error_nomem(context); + return; + } + zot = zo; + z1 = zi1; + while( (c1=sqliteCharVal((unsigned char *)z1))!=0 ){ + z21=zi2; + while( (c2=sqliteCharVal((unsigned char *)z21))!=0 && c2!=c1 ){ + sqliteNextChar(z21); + } + if( c2!=0){ + z22=z21; + sqliteNextChar(z22); + strncpy(zot, z21, z22-z21); + zot+=z22-z21; + } + sqliteNextChar(z1); + } + *zot = '\0'; + + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + sqlite3_free(zo); + } +} + +/* +** Given a string z1, retutns the (0 based) index of it's first occurence +** in z2 after the first s characters. +** Returns -1 when there isn't a match. +** updates p to point to the character where the match occured. +** This is an auxiliary function. +*/ +static int _substr(const char* z1, const char* z2, int s, const char** p){ + int c = 0; + int rVal=-1; + const char* zt1; + const char* zt2; + int c1,c2; + + if( '\0'==*z1 ){ + return -1; + } + + while( (sqliteCharVal((unsigned char *)z2) != 0) && (c++)=0 ? rVal+s : rVal; +} + +/* +** given 2 input strings (s1,s2) and an integer (n) searches from the nth character +** for the string s1. Returns the position where the match occured. +** Characters are counted from 1. +** 0 is returned when no match occurs. +*/ + +static void charindexFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const u8 *z1; /* s1 string */ + u8 *z2; /* s2 string */ + int s=0; + int rVal=0; + + assert( argc==3 ||argc==2); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) || SQLITE_NULL==sqlite3_value_type(argv[1])){ + sqlite3_result_null(context); + return; + } + + z1 = sqlite3_value_text(argv[0]); + if( z1==0 ) return; + z2 = (u8*) sqlite3_value_text(argv[1]); + if(argc==3){ + s = sqlite3_value_int(argv[2])-1; + if(s<0){ + s=0; + } + }else{ + s = 0; + } + + rVal = _substr((char *)z1,(char *)z2,s,NULL); + sqlite3_result_int(context, rVal+1); +} + +/* +** given a string (s) and an integer (n) returns the n leftmost (UTF-8) characters +** if the string has a length<=n or is NULL this function is NOP +*/ +static void leftFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + int c=0; + int cc=0; + int l=0; + const unsigned char *z; /* input string */ + const unsigned char *zt; + unsigned char *rz; /* output string */ + + assert( argc==2); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) || SQLITE_NULL==sqlite3_value_type(argv[1])){ + sqlite3_result_null(context); + return; + } + + z = sqlite3_value_text(argv[0]); + l = sqlite3_value_int(argv[1]); + zt = z; + + while( sqliteCharVal(zt) && c++ 0 ){ + sqliteNextChar(zt); + } + + rz = sqlite3_malloc(ze-zt+1); + if (!rz){ + sqlite3_result_error_nomem(context); + return; + } + strcpy((char*) rz, (char*) (zt)); + sqlite3_result_text(context, (char*)rz, -1, SQLITE_TRANSIENT); + sqlite3_free(rz); +} + +#ifndef HAVE_TRIM +/* +** removes the whitespaces at the begining of a string. +*/ +const char* ltrim(const char* s){ + while( *s==' ' ) + ++s; + return s; +} + +/* +** removes the whitespaces at the end of a string. +** !mutates the input string! +*/ +void rtrim(char* s){ + char* ss = s+strlen(s)-1; + while( ss>=s && *ss==' ' ) + --ss; + *(ss+1)='\0'; +} + +/* +** Removes the whitespace at the begining of a string +*/ +static void ltrimFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *z; + + assert( argc==1); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) ){ + sqlite3_result_null(context); + return; + } + z = sqlite3_value_text(argv[0]); + sqlite3_result_text(context, ltrim(z), -1, SQLITE_TRANSIENT); +} + +/* +** Removes the whitespace at the end of a string +*/ +static void rtrimFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *z; + char *rz; + /* try not to change data in argv */ + + assert( argc==1); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) ){ + sqlite3_result_null(context); + return; + } + z = sqlite3_value_text(argv[0]); + rz = sqlite3StrDup(z); + rtrim(rz); + sqlite3_result_text(context, rz, -1, SQLITE_TRANSIENT); + sqlite3_free(rz); +} + +/* +** Removes the whitespace at the begining and end of a string +*/ +static void trimFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *z; + char *rz; + /* try not to change data in argv */ + + assert( argc==1); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) ){ + sqlite3_result_null(context); + return; + } + z = sqlite3_value_text(argv[0]); + rz = sqlite3StrDup(z); + rtrim(rz); + sqlite3_result_text(context, ltrim(rz), -1, SQLITE_TRANSIENT); + sqlite3_free(rz); +} +#endif + +/* +** given a pointer to a string s1, the length of that string (l1), a new string (s2) +** and it's length (l2) appends s2 to s1. +** All lengths in bytes. +** This is just an auxiliary function +*/ +// static void _append(char **s1, int l1, const char *s2, int l2){ +// *s1 = realloc(*s1, (l1+l2+1)*sizeof(char)); +// strncpy((*s1)+l1, s2, l2); +// *(*(s1)+l1+l2) = '\0'; +// } + +#ifndef HAVE_TRIM + +/* +** given strings s, s1 and s2 replaces occurrences of s1 in s by s2 +*/ +static void replaceFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *z1; /* string s (first parameter) */ + const char *z2; /* string s1 (second parameter) string to look for */ + const char *z3; /* string s2 (third parameter) string to replace occurrences of s1 with */ + int lz1; + int lz2; + int lz3; + int lzo=0; + char *zo=0; + int ret=0; + const char *zt1; + const char *zt2; + + assert( 3==argc ); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) ){ + sqlite3_result_null(context); + return; + } + + z1 = sqlite3_value_text(argv[0]); + z2 = sqlite3_value_text(argv[1]); + z3 = sqlite3_value_text(argv[2]); + /* handle possible null values */ + if( 0==z2 ){ + z2=""; + } + if( 0==z3 ){ + z3=""; + } + + lz1 = strlen(z1); + lz2 = strlen(z2); + lz3 = strlen(z3); + +#if 0 + /* special case when z2 is empty (or null) nothing will be changed */ + if( 0==lz2 ){ + sqlite3_result_text(context, z1, -1, SQLITE_TRANSIENT); + return; + } +#endif + + zt1=z1; + zt2=z1; + + while(1){ + ret=_substr(z2,zt1 , 0, &zt2); + + if( ret<0 ) + break; + + _append(&zo, lzo, zt1, zt2-zt1); + lzo+=zt2-zt1; + _append(&zo, lzo, z3, lz3); + lzo+=lz3; + + zt1=zt2+lz2; + } + _append(&zo, lzo, zt1, lz1-(zt1-z1)); + sqlite3_result_text(context, zo, -1, SQLITE_TRANSIENT); + sqlite3_free(zo); +} +#endif + +/* +** given a string returns the same string but with the characters in reverse order +*/ +static void reverseFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *z; + const char *zt; + char *rz; + char *rzt; + int l = 0; + int i = 0; + + assert( 1==argc ); + + if( SQLITE_NULL==sqlite3_value_type(argv[0]) ){ + sqlite3_result_null(context); + return; + } + z = (char *)sqlite3_value_text(argv[0]); + l = strlen(z); + rz = sqlite3_malloc(l+1); + if (!rz){ + sqlite3_result_error_nomem(context); + return; + } + rzt = rz+l; + *(rzt--) = '\0'; + + zt=z; + while( sqliteCharVal((unsigned char *)zt)!=0 ){ + z=zt; + sqliteNextChar(zt); + for(i=1; zt-i>=z; ++i){ + *(rzt--)=*(zt-i); + } + } + + sqlite3_result_text(context, rz, -1, SQLITE_TRANSIENT); + sqlite3_free(rz); +} + +/* +** An instance of the following structure holds the context of a +** stdev() or variance() aggregate computation. +** implementaion of http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Algorithm_II +** less prone to rounding errors +*/ +typedef struct StdevCtx StdevCtx; +struct StdevCtx { + double rM; + double rS; + i64 cnt; /* number of elements */ +}; + +/* +** An instance of the following structure holds the context of a +** mode() or median() aggregate computation. +** Depends on structures defined in map.c (see map & map) +** These aggregate functions only work for integers and floats although +** they could be made to work for strings. This is usually considered meaningless. +** Only usuall order (for median), no use of collation functions (would this even make sense?) +*/ +typedef struct ModeCtx ModeCtx; +struct ModeCtx { + i64 riM; /* integer value found so far */ + double rdM; /* double value found so far */ + i64 cnt; /* number of elements so far */ + double pcnt; /* number of elements smaller than a percentile */ + i64 mcnt; /* maximum number of occurrences (for mode) */ + i64 mn; /* number of occurrences (for mode and percentiles) */ + i64 is_double; /* whether the computation is being done for doubles (>0) or integers (=0) */ + map* m; /* map structure used for the computation */ + int done; /* whether the answer has been found */ +}; + +/* +** called for each value received during a calculation of stdev or variance +*/ +static void varianceStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + StdevCtx *p; + + double delta; + double x; + + assert( argc==1 ); + p = sqlite3_aggregate_context(context, sizeof(*p)); + /* only consider non-null values */ + if( SQLITE_NULL != sqlite3_value_numeric_type(argv[0]) ){ + p->cnt++; + x = sqlite3_value_double(argv[0]); + delta = (x-p->rM); + p->rM += delta/p->cnt; + p->rS += delta*(x-p->rM); + } +} + +/* +** called for each value received during a calculation of mode of median +*/ +static void modeStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + ModeCtx *p; + i64 xi=0; + double xd=0.0; + i64 *iptr; + double *dptr; + int type; + + assert( argc==1 ); + type = sqlite3_value_numeric_type(argv[0]); + + if( type == SQLITE_NULL) + return; + + p = sqlite3_aggregate_context(context, sizeof(*p)); + + if( 0==(p->m) ){ + p->m = calloc(1, sizeof(map)); + if( type==SQLITE_INTEGER ){ + /* map will be used for integers */ + *(p->m) = map_make(int_cmp); + p->is_double = 0; + }else{ + p->is_double = 1; + /* map will be used for doubles */ + *(p->m) = map_make(double_cmp); + } + } + + ++(p->cnt); + + if( 0==p->is_double ){ + xi = sqlite3_value_int64(argv[0]); + iptr = (i64*)calloc(1,sizeof(i64)); + *iptr = xi; + map_insert(p->m, iptr); + }else{ + xd = sqlite3_value_double(argv[0]); + dptr = (double*)calloc(1,sizeof(double)); + *dptr = xd; + map_insert(p->m, dptr); + } +} + +/* +** Auxiliary function that iterates all elements in a map and finds the mode +** (most frequent value) +*/ +static void modeIterate(void* e, i64 c, void* pp){ + i64 ei; + double ed; + ModeCtx *p = (ModeCtx*)pp; + + if( 0==p->is_double ){ + ei = *(int*)(e); + + if( p->mcnt==c ){ + ++p->mn; + }else if( p->mcntriM = ei; + p->mcnt = c; + p->mn=1; + } + }else{ + ed = *(double*)(e); + + if( p->mcnt==c ){ + ++p->mn; + }else if(p->mcntrdM = ed; + p->mcnt = c; + p->mn=1; + } + } +} + +/* +** Auxiliary function that iterates all elements in a map and finds the median +** (the value such that the number of elements smaller is equal the the number of +** elements larger) +*/ +static void medianIterate(void* e, i64 c, void* pp){ + i64 ei; + double ed; + double iL; + double iR; + int il; + int ir; + ModeCtx *p = (ModeCtx*)pp; + + if(p->done>0) + return; + + iL = p->pcnt; + iR = p->cnt - p->pcnt; + il = p->mcnt + c; + ir = p->cnt - p->mcnt; + + if( il >= iL ){ + if( ir >= iR ){ + ++p->mn; + if( 0==p->is_double ){ + ei = *(int*)(e); + p->riM += ei; + }else{ + ed = *(double*)(e); + p->rdM += ed; + } + }else{ + p->done=1; + } + } + p->mcnt+=c; +} + +/* +** Returns the mode value +*/ +static void modeFinalize(sqlite3_context *context){ + ModeCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->m ){ + map_iterate(p->m, modeIterate, p); + map_destroy(p->m); + free(p->m); + + if( 1==p->mn ){ + if( 0==p->is_double ) + sqlite3_result_int64(context, p->riM); + else + sqlite3_result_double(context, p->rdM); + } + } +} + +/* +** auxiliary function for percentiles +*/ +static void _medianFinalize(sqlite3_context *context){ + ModeCtx *p; + p = (ModeCtx*) sqlite3_aggregate_context(context, 0); + if( p && p->m ){ + p->done=0; + map_iterate(p->m, medianIterate, p); + map_destroy(p->m); + free(p->m); + + if( 0==p->is_double ) + if( 1==p->mn ) + sqlite3_result_int64(context, p->riM); + else + sqlite3_result_double(context, p->riM*1.0/p->mn); + else + sqlite3_result_double(context, p->rdM/p->mn); + } +} + +/* +** Returns the median value +*/ +static void medianFinalize(sqlite3_context *context){ + ModeCtx *p; + p = (ModeCtx*) sqlite3_aggregate_context(context, 0); + if( p!=0 ){ + p->pcnt = (p->cnt)/2.0; + _medianFinalize(context); + } +} + +/* +** Returns the lower_quartile value +*/ +static void lower_quartileFinalize(sqlite3_context *context){ + ModeCtx *p; + p = (ModeCtx*) sqlite3_aggregate_context(context, 0); + if( p!=0 ){ + p->pcnt = (p->cnt)/4.0; + _medianFinalize(context); + } +} + +/* +** Returns the upper_quartile value +*/ +static void upper_quartileFinalize(sqlite3_context *context){ + ModeCtx *p; + p = (ModeCtx*) sqlite3_aggregate_context(context, 0); + if( p!=0 ){ + p->pcnt = (p->cnt)*3/4.0; + _medianFinalize(context); + } +} + +/* +** Returns the stdev value +*/ +static void stdevFinalize(sqlite3_context *context){ + StdevCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>1 ){ + sqlite3_result_double(context, sqrt(p->rS/(p->cnt-1))); + }else{ + sqlite3_result_double(context, 0.0); + } +} + +/* +** Returns the variance value +*/ +static void varianceFinalize(sqlite3_context *context){ + StdevCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>1 ){ + sqlite3_result_double(context, p->rS/(p->cnt-1)); + }else{ + sqlite3_result_double(context, 0.0); + } +} + +#ifdef SQLITE_SOUNDEX + +/* relicoder factored code */ +/* +** Calculates the soundex value of a string +*/ + +static void soundex(const u8 *zIn, char *zResult){ + int i, j; + static const unsigned char iCode[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + }; + + for(i=0; zIn[i] && !isalpha(zIn[i]); i++){} + if( zIn[i] ){ + zResult[0] = toupper(zIn[i]); + for(j=1; j<4 && zIn[i]; i++){ + int code = iCode[zIn[i]&0x7f]; + if( code>0 ){ + zResult[j++] = code + '0'; + } + } + while( j<4 ){ + zResult[j++] = '0'; + } + zResult[j] = 0; + }else{ + strcpy(zResult, "?000"); + } +} + +/* +** computes the number of different characters between the soundex value fo 2 strings +*/ +static void differenceFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + char zResult1[8]; + char zResult2[8]; + char *zR1 = zResult1; + char *zR2 = zResult2; + int rVal = 0; + int i = 0; + const u8 *zIn1; + const u8 *zIn2; + + assert( argc==2 ); + + if( sqlite3_value_type(argv[0])==SQLITE_NULL || sqlite3_value_type(argv[1])==SQLITE_NULL ){ + sqlite3_result_null(context); + return; + } + + zIn1 = (u8*)sqlite3_value_text(argv[0]); + zIn2 = (u8*)sqlite3_value_text(argv[1]); + + soundex(zIn1, zR1); + soundex(zIn2, zR2); + + for(i=0; i<4; ++i){ + if( sqliteCharVal((unsigned char *)zR1)==sqliteCharVal((unsigned char *)zR2) ) + ++rVal; + sqliteNextChar(zR1); + sqliteNextChar(zR2); + } + sqlite3_result_int(context, rVal); +} +#endif + +/* +** This function registered all of the above C functions as SQL +** functions. This should be the only routine in this file with +** external linkage. +*/ +int RegisterExtensionFunctions(sqlite3 *db){ + static const struct FuncDef { + char *zName; + signed char nArg; + u8 argType; /* 0: none. 1: db 2: (-1) */ + u8 eTextRep; /* 1: UTF-16. 0: UTF-8 */ + u8 needCollSeq; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFuncs[] = { + /* math.h */ + { "acos", 1, 0, SQLITE_UTF8, 0, acosFunc }, + { "asin", 1, 0, SQLITE_UTF8, 0, asinFunc }, + { "atan", 1, 0, SQLITE_UTF8, 0, atanFunc }, + { "atn2", 2, 0, SQLITE_UTF8, 0, atn2Func }, + /* XXX alias */ + { "atan2", 2, 0, SQLITE_UTF8, 0, atn2Func }, + { "acosh", 1, 0, SQLITE_UTF8, 0, acoshFunc }, + { "asinh", 1, 0, SQLITE_UTF8, 0, asinhFunc }, + { "atanh", 1, 0, SQLITE_UTF8, 0, atanhFunc }, + + { "difference", 2, 0, SQLITE_UTF8, 0, differenceFunc}, + { "degrees", 1, 0, SQLITE_UTF8, 0, rad2degFunc }, + { "radians", 1, 0, SQLITE_UTF8, 0, deg2radFunc }, + + { "cos", 1, 0, SQLITE_UTF8, 0, cosFunc }, + { "sin", 1, 0, SQLITE_UTF8, 0, sinFunc }, + { "tan", 1, 0, SQLITE_UTF8, 0, tanFunc }, + { "cot", 1, 0, SQLITE_UTF8, 0, cotFunc }, + { "cosh", 1, 0, SQLITE_UTF8, 0, coshFunc }, + { "sinh", 1, 0, SQLITE_UTF8, 0, sinhFunc }, + { "tanh", 1, 0, SQLITE_UTF8, 0, tanhFunc }, + { "coth", 1, 0, SQLITE_UTF8, 0, cothFunc }, + + { "exp", 1, 0, SQLITE_UTF8, 0, expFunc }, + { "log", 1, 0, SQLITE_UTF8, 0, logFunc }, + { "log10", 1, 0, SQLITE_UTF8, 0, log10Func }, + { "power", 2, 0, SQLITE_UTF8, 0, powerFunc }, + { "sign", 1, 0, SQLITE_UTF8, 0, signFunc }, + { "sqrt", 1, 0, SQLITE_UTF8, 0, sqrtFunc }, + { "square", 1, 0, SQLITE_UTF8, 0, squareFunc }, + + { "ceil", 1, 0, SQLITE_UTF8, 0, ceilFunc }, + { "floor", 1, 0, SQLITE_UTF8, 0, floorFunc }, + + { "pi", 0, 0, SQLITE_UTF8, 1, piFunc }, + + + /* string */ + { "replicate", 2, 0, SQLITE_UTF8, 0, replicateFunc }, + { "charindex", 2, 0, SQLITE_UTF8, 0, charindexFunc }, + { "charindex", 3, 0, SQLITE_UTF8, 0, charindexFunc }, + { "leftstr", 2, 0, SQLITE_UTF8, 0, leftFunc }, + { "rightstr", 2, 0, SQLITE_UTF8, 0, rightFunc }, +#ifndef HAVE_TRIM + { "ltrim", 1, 0, SQLITE_UTF8, 0, ltrimFunc }, + { "rtrim", 1, 0, SQLITE_UTF8, 0, rtrimFunc }, + { "trim", 1, 0, SQLITE_UTF8, 0, trimFunc }, + { "replace", 3, 0, SQLITE_UTF8, 0, replaceFunc }, +#endif + { "reverse", 1, 0, SQLITE_UTF8, 0, reverseFunc }, + { "proper", 1, 0, SQLITE_UTF8, 0, properFunc }, + { "padl", 2, 0, SQLITE_UTF8, 0, padlFunc }, + { "padr", 2, 0, SQLITE_UTF8, 0, padrFunc }, + { "padc", 2, 0, SQLITE_UTF8, 0, padcFunc }, + { "strfilter", 2, 0, SQLITE_UTF8, 0, strfilterFunc }, + + }; + /* Aggregate functions */ + static const struct FuncDefAgg { + char *zName; + signed char nArg; + u8 argType; + u8 needCollSeq; + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinalize)(sqlite3_context*); + } aAggs[] = { + { "stdev", 1, 0, 0, varianceStep, stdevFinalize }, + { "variance", 1, 0, 0, varianceStep, varianceFinalize }, + { "mode", 1, 0, 0, modeStep, modeFinalize }, + { "median", 1, 0, 0, modeStep, medianFinalize }, + { "lower_quartile", 1, 0, 0, modeStep, lower_quartileFinalize }, + { "upper_quartile", 1, 0, 0, modeStep, upper_quartileFinalize }, + }; + int i; + + for(i=0; ineedCollSeq = 1; + } + } +#endif + } + + for(i=0; ineedCollSeq = 1; + } + } +#endif + } + return 0; +} + +#ifdef COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE +int sqlite3_extension_init( + sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + RegisterExtensionFunctions(db); + return 0; +} +#endif /* COMPILE_SQLITE_EXTENSIONS_AS_LOADABLE_MODULE */ + +map map_make(cmp_func cmp){ + map r; + r.cmp=cmp; + r.base = 0; + + return r; +} + +void* xcalloc(size_t nmemb, size_t size, char* s){ + void* ret = calloc(nmemb, size); + return ret; +} + +static void xfree(void* p){ + free(p); +} + +void node_insert(node** n, cmp_func cmp, void *e){ + int c; + node* nn; + if(*n==0){ + nn = (node*)xcalloc(1,sizeof(node), "for node"); + nn->data = e; + nn->count = 1; + *n=nn; + }else{ + c=cmp((*n)->data,e); + if(0==c){ + ++((*n)->count); + xfree(e); + }else if(c>0){ + /* put it right here */ + node_insert(&((*n)->l), cmp, e); + }else{ + node_insert(&((*n)->r), cmp, e); + } + } +} + +void map_insert(map *m, void *e){ + node_insert(&(m->base), m->cmp, e); +} + +void node_iterate(node *n, map_iterator iter, void* p){ + if(n){ + if(n->l) + node_iterate(n->l, iter, p); + iter(n->data, n->count, p); + if(n->r) + node_iterate(n->r, iter, p); + } +} + +void map_iterate(map *m, map_iterator iter, void* p){ + node_iterate(m->base, iter, p); +} + +void node_destroy(node *n){ + if(0!=n){ + xfree(n->data); + if(n->l) + node_destroy(n->l); + if(n->r) + node_destroy(n->r); + + xfree(n); + } +} + +void map_destroy(map *m){ + node_destroy(m->base); +} + +int int_cmp(const void *a, const void *b){ + int64_t aa = *(int64_t *)(a); + int64_t bb = *(int64_t *)(b); + /* printf("cmp %d <=> %d\n",aa,bb); */ + if(aa==bb) + return 0; + else if(aa %d\n",aa,bb); */ + if(aa==bb) + return 0; + else if(aa %lld\n", ee,c); +} + diff --git a/src/help.txt b/src/help.txt index d1ca908e..14129fc3 100644 --- a/src/help.txt +++ b/src/help.txt @@ -57,9 +57,8 @@ On color displays, the lines will be highlighted as follows: * Errors will be colored in red; * warnings will be yellow; * boundaries between days will be underlined; and - * various color highlights will be applied to: SQL keywords, XML - tags, file and line numbers in Java backtraces, and - quoted strings. + * various color highlights will be applied to: IP addresses, SQL keywords, + XML tags, file and line numbers in Java backtraces, and quoted strings. To give you an idea of where you are in the file spatially, the right side of the display has a proportionally sized 'scrollbar' that @@ -221,6 +220,9 @@ through the file. that can be used in queries. See the SQL section below for more information. + y/Y Move forward/backward through the log view based on the + "line_number" column in the SQL result view. + v Switch to/from the SQL result view. V Switch between the log and SQL result views while @@ -232,6 +234,11 @@ through the file. to the log view and move to the line number that was selected in the "line_number" column. + p Enable or disable the display of the fields that the + log message parser knows about or has discovered. + This overlay is temporarily enabled when the semicolon + key (;) is pressed so that it is easier to write queries. + MOUSE SUPPORT (experimental) ---------------------------- @@ -244,6 +251,8 @@ environment variable to "mouse". COMMANDS -------- + help Switch to this help text view. + unix-time Convert a unix-timestamp in seconds to a human-readable form or vice-versa. @@ -297,7 +306,7 @@ COMMANDS filter-related commands can be added to the session file. -SQL QUERIES +SQL QUERIES (experimental) ----------- Lnav has support for performing SQL queries on log files using the @@ -312,24 +321,46 @@ being accessed in any loaded Apache log files, you can execute: The query result view shows the results as text and graphs any number values found in the result, much like the histogram view. -The basic set of log file tables are: +The builtin set of log tables are listed below. Note that only the +log messages that match a particular format can be queried by a +particular table. You can find the file format and table name for +the top log message by looking in the upper right hand corner of the +log file view. - syslog_log, generic_log - Contains any syslog lines and any lines picked up - by the generic log file parser. +The log table names are as follows: + + access_log Apache common access log format + syslog_log Syslog format + glog_log Google glog format + strace_log Strace log format + generic_log 'Generic' log format. This table contains messages + from files that have a very simple format with a + leading timestamp followed by the message. + +The columns available for the top log line in the view will +automatically be displayed after pressing the semicolon (;) key. +All log tables contain at least the following columns: line_number The line number in the file, starting at zero. - path The full path to the file. log_time The time of the log entry. + idle_msecs The amount of time, in milliseconds, between the + current log message and the previous one. level The log level (e.g. info, error, etc...). + path The full path to the file. raw_line The raw line of text. The following tables include the basic columns as listed above and include a few more columns since the log file format is more structured. - access_log Contains Apache log file lines. The column names - are the same as those in the Microsoft LogParser. + syslog_log + + log_hostname The hostname the message was received from. + log_procname The name of the process that sent the message. + log_pid The process ID of the process that sent the message. + + access_log (The column names are the same as those in the + Microsoft LogParser tool.) c_ip The client IP address. cs_username The client user name. @@ -342,8 +373,9 @@ structured. cs_referrer The URL of the referring page. cs_user_agent The user agent string. - strace_log Contains strace log file lines. Currently, you - need to run strace with the "-tt -T" options. + strace_log (Currently, you need to run strace with the + "-tt -T" options so there are timestamps for + each function call.) funcname The name of the syscall. result The result code. @@ -360,3 +392,51 @@ example of a top ten query into the "/tmp/topten.db" file, you can do: ;create table topten as select cs_uri_stem, count(*) as total from access_log group by cs_uri_stem order by total desc limit 10; + + +DYNAMIC LOG LINE TABLE (experimental) +---------------------- + +(NOTE: This feature is still very new and not completely reliable yet, + use with care.) + +For log formats that lack message structure, lnav can parse the log +message and attempt to extract any data fields that it finds. This +feature is available through the "logline" log table. This table is +dynamically created and defined based on the message at the top of +the log view. For example, given the following log message from "sudo", +lnav will create the "logline" table with columns for "TTY", "PWD", +"USER", and "COMMAND": + + May 24 06:48:38 Tim-Stacks-iMac.local sudo[76387]: stack : TTY=ttys003 ; + PWD=/Users/stack/github/lbuild ; USER=root ; + COMMAND=/bin/echo Hello, World! + +Queries executed against this table will then only return results for +other log messages that have the same format. So, if you were to +execute the following query while viewing the above line, you might +get the following results: + + ;select USER,COMMAND from logline; + + USER | COMMAND + ---- | ------------------------- + root | /bin/echo Hello, World! + mal | /bin/echo Goodbye, World! + + +The log parser works by examining each message for key/value pairs +separated by an equal sign (=) or a colon (:). For example, in the +previous example of a "sudo" message, the parser sees the "USER=root" +string as a pair where the key is "USER" and the value is "root". +If no pairs can be found, then anything that looks like a value is +extracted and assigned a numbered column. For example, the following +line is from "dhcpd": + + Sep 16 22:35:57 drill dhcpd: DHCPDISCOVER from 00:16:ce:54:4e:f3 via hme3 + +In this case, the lnav parser recognizes that "DHCPDISCOVER", the MAC +address and the "hme3" device name are values and not normal words. So, +it builds a table with three columns for each of these values. The +regular words in the message, like "from" and "via", are then used to +find other messages with a similar format. diff --git a/src/hist_source.cc b/src/hist_source.cc index 608306fa..c4199175 100644 --- a/src/hist_source.cc +++ b/src/hist_source.cc @@ -88,6 +88,11 @@ void hist_source::text_attrs_for_line(textview_curses &tc, int row, string_attrs_t &value_out) { + int grow = row / (this->buckets_per_group() + 1); + int brow = row % (this->buckets_per_group() + 1); + bucket_group_t bg = this->hs_group_keys[grow]; + int bucket_index = brow - 1; + assert(this->hs_analyzed); if (this->hs_token_bucket != NULL) { @@ -120,9 +125,14 @@ void hist_source::text_attrs_for_line(textview_curses &tc, lr.lr_end = lr.lr_start + amount; value_out[lr].insert(make_string_attr("style", attrs)); - + lr.lr_start = lr.lr_end; } + + this->hs_label_source->hist_attrs_for_bucket((bg * this->hs_group_size) + + (bucket_index * this->hs_bucket_size), + *this->hs_token_bucket, + value_out); } } diff --git a/src/hist_source.hh b/src/hist_source.hh index 0a1833ec..04ac59c8 100644 --- a/src/hist_source.hh +++ b/src/hist_source.hh @@ -79,6 +79,10 @@ public: virtual void hist_label_for_bucket(int bucket_start_value, const bucket_t &bucket, std::string &label_out) { }; + + virtual void hist_attrs_for_bucket(int bucket_start_value, + const bucket_t &bucket, + string_attrs_t &sa) { }; }; hist_source(); @@ -186,6 +190,21 @@ public: void add_value(unsigned int value, bucket_type_t bt, bucket_count_t amount = 1.0); + + void add_empty_value(unsigned int value) { + bucket_group_t bg; + + this->hs_analyzed = false; + + bg = bucket_group_t(value / this->hs_group_size); + + bucket_array_t &ba = this->hs_groups[bg]; + + if (ba.empty()) { + ba.resize(this->buckets_per_group()); + } + }; + void analyze(void); protected: diff --git a/src/listview_curses.cc b/src/listview_curses.cc index afab4849..d01c0411 100644 --- a/src/listview_curses.cc +++ b/src/listview_curses.cc @@ -125,11 +125,18 @@ bool listview_curses::handle_key(int ch) void listview_curses::do_update(void) { if (this->lv_window != NULL && this->lv_needs_update) { - vis_line_t y(this->lv_y), height, bottom, lines; + vis_line_t y(this->lv_y), height, bottom, lines, row; + attr_line_t overlay_line; + vis_line_t overlay_height(0); struct line_range lr; unsigned long width; size_t row_count; + if (this->lv_overlay_source != NULL) { + overlay_height = vis_line_t( + this->lv_overlay_source->list_overlay_count(*this)); + } + this->get_dimensions(height, width); lr.lr_start = this->lv_left; lr.lr_end = this->lv_left + width; @@ -139,20 +146,31 @@ void listview_curses::do_update(void) this->lv_top = max(vis_line_t(0), vis_line_t(row_count) - height); } - lines = y + min(height, vis_line_t(row_count) - this->lv_top); + row = this->lv_top; + lines = min(height - overlay_height, + vis_line_t(row_count) - this->lv_top); bottom = y + height; - for (; y < lines; ++y) { - vis_line_t row = this->lv_top + y - vis_line_t(this->lv_y); - attr_line_t al; - - this->lv_source->listview_value_for_row(*this, row, al); - this->mvwattrline(this->lv_window, y, 0, al, lr); - } - - /* Clear out any remaining lines on the display. */ for (; y < bottom; ++y) { - wmove(this->lv_window, y, 0); - wclrtoeol(this->lv_window); + if (this->lv_overlay_source != NULL && + this->lv_overlay_source->list_value_for_overlay( + *this, + y - vis_line_t(this->lv_y), + overlay_line)) { + this->mvwattrline(this->lv_window, y, 0, overlay_line, lr); + overlay_line.clear(); + } + else if (lines > 0) { + attr_line_t al; + + this->lv_source->listview_value_for_row(*this, row, al); + this->mvwattrline(this->lv_window, y, 0, al, lr); + --lines; + ++row; + } + else { + wmove(this->lv_window, y, 0); + wclrtoeol(this->lv_window); + } } if (this->lv_show_scrollbar) { diff --git a/src/listview_curses.hh b/src/listview_curses.hh index 52e6d6f8..ad0b5918 100644 --- a/src/listview_curses.hh +++ b/src/listview_curses.hh @@ -35,6 +35,7 @@ #include #include +#include #include #include "strong_int.hh" @@ -66,6 +67,31 @@ public: attr_line_t &value_out) = 0; }; +struct listview_overlay { + listview_overlay(int y, const attr_line_t &al) : lo_y(y), lo_line(al) { }; + + int get_absolute_y(int height) { + if (this->lo_y >= 0) + return this->lo_y; + + return height + this->lo_y; + }; + + int lo_y; + attr_line_t lo_line; +}; + +class list_overlay_source { +public: + virtual ~list_overlay_source() { }; + + virtual size_t list_overlay_count(const listview_curses &lv) = 0; + + virtual bool list_value_for_overlay(const listview_curses &lv, + vis_line_t y, + attr_line_t &value_out) = 0; +}; + /** * View that displays a list of lines that can optionally contain highlighting. */ @@ -87,7 +113,19 @@ public: }; /** @return The data source delegate. */ - list_data_source *get_data_source() { return this->lv_source; }; + list_data_source *get_data_source() const { return this->lv_source; }; + + /** @param src The data source delegate. */ + void set_overlay_source(list_overlay_source *src) + { + this->lv_overlay_source = src; + this->reload_data(); + }; + + /** @return The overlay source delegate. */ + list_overlay_source *get_overlay_source() { + return this->lv_overlay_source; + }; /** * @param va The action to invoke when the view is scrolled. @@ -265,6 +303,7 @@ public: else { height_out = this->lv_height; } + }; /** This method should be called when the data source has changed. */ @@ -283,6 +322,7 @@ public: protected: list_data_source *lv_source; /*< The data source delegate. */ + list_overlay_source *lv_overlay_source; action lv_scroll; /*< The scroll action. */ WINDOW *lv_window; /*< The window that contains this view. */ unsigned int lv_y; /*< The y offset of this view. */ diff --git a/src/lnav.cc b/src/lnav.cc index caef3f04..359904f9 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -94,6 +95,7 @@ #include "data_parser.hh" #include "xterm_mouse.hh" #include "lnav_commands.hh" +#include "column_namer.hh" using namespace std; @@ -117,6 +119,7 @@ static struct hist_level HIST_ZOOM_VALUES[] = { static const int HIST_ZOOM_LEVELS = sizeof(HIST_ZOOM_VALUES) / sizeof(struct hist_level); static bookmark_type_t BM_EXAMPLE; +static bookmark_type_t BM_QUERY; /** * Check if an experimental feature should be enabled by @@ -169,6 +172,385 @@ void sqlite_close_wrapper(void *mem) sqlite3_close((sqlite3*)mem); } +class field_overlay_source : public list_overlay_source { +public: + field_overlay_source() : + fos_active(false), + fos_active_prev(false), + fos_scanner(NULL), + fos_parser(NULL), + fos_namer(NULL) { }; + + ~field_overlay_source() { + if (this->fos_scanner) { + delete this->fos_scanner; + } + if (this->fos_parser) { + delete this->fos_parser; + } + if (this->fos_namer) { + delete this->fos_namer; + } + }; + + size_t list_overlay_count(const listview_curses &lv) { + logfile_sub_source &lss = lnav_data.ld_log_source; + + if (!this->fos_active) + return 0; + + if (lss.text_line_count() == 0) + return 0; + + content_line_t cl = lss.at(lv.get_top()); + logfile *lf = lss.find(cl); + std::string line = lf->read_line(lf->begin() + cl); + struct line_range body; + string_attrs_t sa; + + this->fos_line_values.clear(); + this->fos_key_size = 0; + + lf->get_format()->annotate(line, sa, this->fos_line_values); + + for (std::vector::iterator iter = this->fos_line_values.begin(); + iter != this->fos_line_values.end(); + ++iter) { + this->fos_key_size = max(this->fos_key_size, + (int)iter->lv_name.length()); + } + + body = find_string_attr_range(sa, "body"); + if (body.lr_end != -1) { + line = line.substr(body.lr_start); + } + + if (this->fos_parser) { + delete this->fos_parser; + delete this->fos_scanner; + delete this->fos_namer; + } + + this->fos_scanner = new data_scanner(line); + this->fos_parser = new data_parser(this->fos_scanner); + this->fos_parser->parse(); + this->fos_namer = new column_namer(); + + for (data_parser::element_list_t::iterator iter = this->fos_parser->dp_pairs.begin(); + iter != this->fos_parser->dp_pairs.end(); + ++iter) { + std::string colname = this->fos_parser->get_element_string(iter->e_sub_elements->front()); + + colname = this->fos_namer->add_column(colname); + this->fos_key_size = max(this->fos_key_size, + (int)colname.length()); + } + + return (1 + + this->fos_line_values.size() + + 1 + + this->fos_parser->dp_pairs.size()); + }; + + bool list_value_for_overlay(const listview_curses &lv, + vis_line_t y, + attr_line_t &value_out) { + if (!this->fos_active || this->fos_parser == NULL) + return false; + + size_t total_count = (1 + + this->fos_line_values.size() + + 1 + + this->fos_parser->dp_pairs.size()); + int row = (int)y - 1; + bool last_line = (row == (int)(total_count - 1)); + + if (row < 0 || row >= (int)total_count) + return false; + + std::string &str = value_out.get_string(); + + if (row == 0) { + if (this->fos_line_values.empty()) + str = " No known message fields"; + else + str = " Known message fields:"; + return true; + } + row -= 1; + if (row == this->fos_line_values.size()) { + if (this->fos_parser->dp_pairs.empty()) + str = " No discovered message fields"; + else + str = " Discovered message fields:"; + return true; + } + + if (row < (int)this->fos_line_values.size()) { + str = " " + this->fos_line_values[row].lv_name; + + int padding = this->fos_key_size - str.length() + 3; + + str.append(padding, ' '); + str += " = " + this->fos_line_values[row].to_string(); + } + else { + data_parser::element_list_t::iterator iter; + + row -= this->fos_line_values.size() + 1; + + iter = this->fos_parser->dp_pairs.begin(); + std::advance(iter, row); + + str = " " + this->fos_namer->cn_names[row]; + int padding = this->fos_key_size - str.length() + 3; + + str.append(padding, ' '); + str += " = " + this->fos_parser->get_element_string(iter->e_sub_elements->back()); + } + + string_attrs_t &sa = value_out.get_attrs(); + struct line_range lr = { 1, 2 }; + sa[lr].insert(make_string_attr("graphic", + last_line ? ACS_LLCORNER : ACS_LTEE)); + + lr.lr_start = 3 + this->fos_key_size + 3; + lr.lr_end = -1; + sa[lr].insert(make_string_attr("style", A_BOLD)); + + return true; + }; + + bool fos_active; + bool fos_active_prev; + data_scanner *fos_scanner; + data_parser *fos_parser; + column_namer *fos_namer; + std::vector fos_line_values; + int fos_key_size; +}; + +typedef std::map > db_table_map_t; + +class db_metadata +{ +public: + db_metadata() { }; + + db_table_map_t dm_db_list; +}; + +static int handle_collation_list(void *ptr, + int ncols, + char **colvalues, + char **colnames) +{ + lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[1]); + + return 0; +} + +static int handle_db_list(void *ptr, + int ncols, + char **colvalues, + char **colnames) +{ + db_metadata *dbm = (db_metadata *)ptr; + + dbm->dm_db_list[colvalues[1]] = std::vector(); + + lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[1]); + + return 0; +} + +static int handle_table_list(void *ptr, + int ncols, + char **colvalues, + char **colnames) +{ + std::map >::iterator iter = *(std::map >::iterator *)ptr; + + lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[0]); + iter->second.push_back(colvalues[0]); + + return 0; +} + +static int handle_table_info(void *ptr, + int ncols, + char **colvalues, + char **colnames) +{ + // string &table_name = *((string *)ptr); + + lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[1]); + if (strcmp(colvalues[5], "1") == 0) { + lnav_data.ld_db_key_names.push_back(colvalues[1]); + } + return 0; +} + +static int handle_foreign_key_list(void *ptr, + int ncols, + char **colvalues, + char **colnames) +{ + lnav_data.ld_db_key_names.push_back(colvalues[3]); + lnav_data.ld_db_key_names.push_back(colvalues[4]); + return 0; +} + +static void walk_sqlite_metadata(sqlite3 *db) +{ + db_metadata dbm; + + lnav_data.ld_db_key_names.clear(); + lnav_data.ld_rl_view->clear_possibilities(LNM_SQL, "*"); + + { + const char *sql_commands[] = { + "ADD", + "ALL", + "ALTER", + "ANALYZE", + "AND", + "ASC", + "ATTACH", + "BEGIN", + "BETWEEN", + "CASE", + "CAST", + "COLLATE", + "COLUMN", + "COMMIT", + "CONFLICT", + "CREATE", + "CROSS", + "DATABASE", + "DELETE", + "DESC", + "DETACH", + "DISTINCT", + "DROP", + "ELSE", + "END", + "ESCAPE", + "EXCEPT", + "EXISTS", + "EXPLAIN", + "FROM", + "GLOB", + "GROUP", + "HAVING", + "IN", + "INDEX", + "INDEXED", + "INNER", + "INSERT", + "INTERSECT", + "ISNULL", + "JOIN", + "LEFT", + "LIKE", + "LIMIT", + "MATCH", + "NATURAL", + "NOT", + "NOTNULL", + "NULL", + "OFFSET", + "OR", + "ORDER", + "OUTER", + "PRAGMA", + "REGEXP", + "REINDEX", + "RENAME", + "REPLACE", + "ROLLBACK", + "SELECT", + "TABLE", + "THEN", + "TRANSACTION", + "TRIGGER", + "UNION", + "UNIQUE", + "UPDATE", + "USING", + "VACUUM", + "VIEW", + "WHERE", + "WHEN", + + "logline", + + NULL + }; + + for (int lpc = 0; sql_commands[lpc]; lpc++) { + lnav_data.ld_rl_view-> + add_possibility(LNM_SQL, "*", sql_commands[lpc]); + } + } + + sqlite3_exec(db, + "pragma collation_list", + handle_collation_list, + NULL, + NULL); + + sqlite3_exec(db, + "pragma database_list", + handle_db_list, + &dbm, + NULL); + + for (db_table_map_t::iterator iter = dbm.dm_db_list.begin(); + iter != dbm.dm_db_list.end(); + ++iter) { + auto_mem query; + + query = sqlite3_mprintf("SELECT name FROM %Q.sqlite_master " + "WHERE type='table'", + iter->first.c_str()); + + sqlite3_exec(db, query, handle_table_list, &iter, NULL); + + for (std::vector::iterator table_iter = iter->second.begin(); + table_iter != iter->second.end(); + ++table_iter) { + auto_mem table_query; + string &table_name = *table_iter; + + table_query = sqlite3_mprintf( + "pragma %Q.table_info(%Q)", + iter->first.c_str(), + table_name.c_str()); + + sqlite3_exec(db, + table_query, + handle_table_info, + &table_name, + NULL); + + table_query = sqlite3_mprintf( + "pragma %Q.foreign_key_list(%Q)", + iter->first.c_str(), + table_name.c_str()); + + sqlite3_exec(db, + table_query, + handle_foreign_key_list, + &table_name, + NULL); + } + } + + stable_sort(lnav_data.ld_db_key_names.begin(), + lnav_data.ld_db_key_names.end()); +} + /** * Observer for loading progress that updates the bottom status bar. */ @@ -225,6 +607,135 @@ private: content_line_t lo_last_line; }; +class log_data_table : public log_vtab_impl { +public: + + log_data_table(content_line_t template_line) + : log_vtab_impl("logline"), + ldt_template_line(template_line) { + }; + + void get_columns(vector &cols) { + content_line_t cl_copy = this->ldt_template_line; + logfile *lf = lnav_data.ld_log_source.find(cl_copy); + std::string val = lf->read_line(lf->begin() + cl_copy); + struct line_range body; + string_attrs_t sa; + std::vector line_values; + + lf->get_format()->annotate(val, sa, line_values); + body = find_string_attr_range(sa, "body"); + if (body.lr_end != -1) { + val = val.substr(body.lr_start); + } + data_scanner ds(val); + data_parser dp(&ds); + column_namer cn; + + dp.parse(); + + for (data_parser::element_list_t::iterator pair_iter = dp.dp_pairs.begin(); + pair_iter != dp.dp_pairs.end(); + ++pair_iter) { + std::string key_str = dp.get_element_string(pair_iter->e_sub_elements->front()); + std::string colname = cn.add_column(key_str); + int sql_type = SQLITE3_TEXT; + const char *collator = NULL; + char *name; + + /* XXX LEAK */ + name = strdup(colname.c_str()); + fprintf(stderr, "name: %s\n", name); + switch (pair_iter->e_sub_elements->back().e_token) { + case DT_IPV4_ADDRESS: + case DT_IPV6_ADDRESS: + collator = "ipaddress"; + break; + case DT_NUMBER: + sql_type = SQLITE_FLOAT; + break; + default: + collator = "naturalnocase"; + break; + } + cols.push_back(vtab_column(name, sql_type, collator)); + } + this->ldt_schema_id = dp.dp_schema_id; + // cols.push_back(vtab_column("value", "text")); + }; + + bool next(log_cursor &lc, logfile_sub_source &lss) { + lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1); + lc.lc_sub_index = 0; + + if (lc.lc_curr_line == (int)lss.text_line_count()) + return true; + + std::string line; + content_line_t cl; + string_attrs_t sa; + struct line_range body; + std::vector line_values; + + cl = lss.at(lc.lc_curr_line); + logfile *lf = lss.find(cl); + logfile::iterator lf_iter = lf->begin() + cl; + + if (lf_iter->get_level() & logline::LEVEL_CONTINUED) { + return false; + } + + lf->read_line(lf_iter, line); + lf->get_format()->annotate(line, sa, line_values); + body = find_string_attr_range(sa, "body"); + if (body.lr_end == -1) { + return false; + } + + line = line.substr(body.lr_start); + if (line.empty()) + return false; + + data_scanner ds(line); + data_parser dp(&ds); + dp.parse(); + if (dp.dp_schema_id != this->ldt_schema_id) + return false; + + return true; + }; + + void extract(logfile *lf, + const std::string &line, + std::vector &values) { + std::vector std_values; + struct line_range body; + string_attrs_t sa; + + lf->get_format()->annotate(line, sa, std_values); + body = find_string_attr_range(sa, "body"); + string sub = line.substr(body.lr_start); + + data_scanner ds(sub); + data_parser dp(&ds); + + dp.parse(); + + for (data_parser::element_list_t::iterator pair_iter = dp.dp_pairs.begin(); + pair_iter != dp.dp_pairs.end(); + ++pair_iter) { + std::string tmp = dp.get_element_string(pair_iter->e_sub_elements->back()); + + values.push_back(logline_value("", tmp)); + } + }; + +private: + int ldt_column; + const content_line_t ldt_template_line; + data_parser::schema_id_t ldt_schema_id; +}; + static void rebuild_hist(size_t old_count, bool force) { textview_curses &hist_view = lnav_data.ld_views[LNV_HISTOGRAM]; @@ -583,7 +1094,7 @@ static void change_text_file(void) * * @param expected_tc The text view that should be on top. */ -static void ensure_view(textview_curses *expected_tc) +void ensure_view(textview_curses *expected_tc) { textview_curses *tc = lnav_data.ld_view_stack.top(); @@ -758,6 +1269,14 @@ static void handle_paging_key(int ch) tc->set_top(bm[&textview_curses::BM_SEARCH].prev(tc->get_top())); break; + case 'y': + tc->set_top(bm[&BM_QUERY].next(tc->get_top())); + break; + + case 'Y': + tc->set_top(bm[&BM_QUERY].prev(tc->get_top())); + break; + case '>': { std::pair range; @@ -1056,10 +1575,44 @@ static void handle_paging_key(int ch) break; case ';': - lnav_data.ld_mode = LNM_SQL; - lnav_data.ld_rl_view->focus(LNM_SQL, ";"); + if (tc == &lnav_data.ld_views[LNV_LOG] || + tc == &lnav_data.ld_views[LNV_DB]) { + textview_curses &log_view = lnav_data.ld_views[LNV_LOG]; + + lnav_data.ld_mode = LNM_SQL; + walk_sqlite_metadata(lnav_data.ld_db.in()); + lnav_data.ld_rl_view->focus(LNM_SQL, ";"); + + if (log_view.get_inner_height()) { + vis_line_t vl = log_view.get_top(); + content_line_t cl = lnav_data.ld_log_source.at(vl); + + lnav_data.ld_vtab_manager->unregister_vtab("logline"); + lnav_data.ld_vtab_manager->register_vtab(new log_data_table(cl)); + } + + lnav_data.ld_bottom_source.update_loading(0, 0); + lnav_data.ld_status[LNS_BOTTOM].do_update(); + + field_overlay_source *fos; + + fos = (field_overlay_source *)log_view.get_overlay_source(); + fos->fos_active_prev = fos->fos_active; + if (!fos->fos_active) { + fos->fos_active = true; + tc->reload_data(); + } + } break; + case 'p': + field_overlay_source *fos; + + fos = (field_overlay_source *)lnav_data.ld_views[LNV_LOG].get_overlay_source(); + fos->fos_active = !fos->fos_active; + tc->reload_data(); + break; + case 't': toggle_view(&lnav_data.ld_views[LNV_TEXT]); break; @@ -1261,13 +1814,12 @@ static void execute_file(string path) } } -static int sql_callback(void *arg, - int ncols, - char **values, - char **colnames) +static int sql_callback(sqlite3_stmt *stmt) { + logfile_sub_source &lss = lnav_data.ld_log_source; db_label_source &dls = lnav_data.ld_db_rows; hist_source &hs = lnav_data.ld_db_source; + int ncols = sqlite3_column_count(stmt); int row_number; int lpc, retval = 0; @@ -1275,19 +1827,45 @@ static int sql_callback(void *arg, dls.dls_rows.resize(row_number + 1); if (dls.dls_headers.empty()) { for (lpc = 0; lpc < ncols; lpc++) { - dls.push_header(colnames[lpc]); - hs.set_role_for_type(bucket_type_t(lpc), - view_colors::singleton(). - next_highlight()); - } + int type = sqlite3_column_type(stmt, lpc); + string colname = sqlite3_column_name(stmt, lpc); + bool graphable; + + graphable = ((type == SQLITE_INTEGER || type == SQLITE_FLOAT) && + colname != "line_number" && + !binary_search(lnav_data.ld_db_key_names.begin(), + lnav_data.ld_db_key_names.end(), + colname)); + + dls.push_header(colname, type, graphable); + if (graphable) { + hs.set_role_for_type(bucket_type_t(lpc), + view_colors::singleton(). + next_plain_highlight()); + } + } } for (lpc = 0; lpc < ncols; lpc++) { + const char *value = (const char *)sqlite3_column_text(stmt, lpc); double num_value = 0.0; - dls.push_column(values[lpc]); - if (strcmp(colnames[lpc], "line_number") != 0) - sscanf(values[lpc], "%lf", &num_value); - hs.add_value(row_number, bucket_type_t(lpc), num_value); + if (value == NULL) + value = ""; + dls.push_column(value); + if (dls.dls_headers[lpc] == "line_number") { + int line_number = -1; + + if (sscanf(value, "%d", &line_number) == 1) { + lss.text_mark(&BM_QUERY, line_number, true); + } + } + if (dls.dls_headers_to_graph[lpc]) { + sscanf(value, "%lf", &num_value); + hs.add_value(row_number, bucket_type_t(lpc), num_value); + } + else { + hs.add_empty_value(row_number); + } } return retval; @@ -1317,15 +1895,14 @@ static void rl_search(void *dummy, readline_curses *rc) grep_error("sql error: incomplete statement"); } else { - sqlite3_stmt *stmt; - const char *tail; + auto_mem stmt(sqlite3_finalize); int retcode; retcode = sqlite3_prepare_v2(lnav_data.ld_db, rc->get_value().c_str(), -1, - &stmt, - &tail); + stmt.out(), + NULL); if (retcode != SQLITE_OK) { const char *errmsg = sqlite3_errmsg(lnav_data.ld_db); @@ -1432,29 +2009,74 @@ static void rl_callback(void *dummy, readline_curses *rc) { db_label_source &dls = lnav_data.ld_db_rows; hist_source &hs = lnav_data.ld_db_source; - auto_mem errmsg; + auto_mem stmt(sqlite3_finalize); + int retcode; lnav_data.ld_bottom_source.grep_error(""); hs.clear(); dls.clear(); - if (sqlite3_exec(lnav_data.ld_db, - rc->get_value().c_str(), - sql_callback, - NULL, - errmsg.out()) != SQLITE_OK) { - rc->set_value(errmsg.in()); + retcode = sqlite3_prepare_v2(lnav_data.ld_db, + rc->get_value().c_str(), + -1, + stmt.out(), + NULL); + if (retcode != SQLITE_OK) { + const char *errmsg = sqlite3_errmsg(lnav_data.ld_db); + + rc->set_value(errmsg); } - else { - rc->set_value(""); + else if (stmt == NULL) { + rc->set_value(""); + } + else { + bool done = false; + + lnav_data.ld_log_source.text_clear_marks(&BM_QUERY); + while (!done) { + retcode = sqlite3_step(stmt.in()); + + switch (retcode) { + case SQLITE_OK: + case SQLITE_DONE: + done = true; + rc->set_value(""); + break; + case SQLITE_ROW: + sql_callback(stmt.in()); + break; + default: + { + const char *errmsg; - hs.analyze(); - lnav_data.ld_views[LNV_DB].reload_data(); - lnav_data.ld_views[LNV_DB].set_left(0); + fprintf(stderr, "code %d\n", retcode); + errmsg = sqlite3_errmsg(lnav_data.ld_db); + rc->set_value(errmsg); + done = true; + } + break; + } + } + + if (retcode == SQLITE_DONE) { + hs.analyze(); + lnav_data.ld_views[LNV_LOG].reload_data(); + lnav_data.ld_views[LNV_DB].reload_data(); + lnav_data.ld_views[LNV_DB].set_left(0); - if (dls.dls_rows.size() > 0) { - ensure_view(&lnav_data.ld_views[LNV_DB]); + if (dls.dls_rows.size() > 0) { + ensure_view(&lnav_data.ld_views[LNV_DB]); + } } + + lnav_data.ld_bottom_source.update_loading(0, 0); + lnav_data.ld_status[LNS_BOTTOM].do_update(); } + + field_overlay_source *fos; + + fos = (field_overlay_source *)lnav_data.ld_views[LNV_LOG].get_overlay_source(); + fos->fos_active = fos->fos_active_prev; + lnav_data.ld_views[LNV_LOG].reload_data(); } lnav_data.ld_mode = LNM_PAGING; @@ -1552,7 +2174,74 @@ static void update_times(void *, listview_curses *lv) lnav_data.ld_bottom_time = hs.value_for_row(lv->get_bottom()); } } - + +enum file_format_t { + FF_UNKNOWN, + FF_SQLITE_DB, +}; + +file_format_t detect_file_format(const string &filename) +{ + file_format_t retval = FF_UNKNOWN; + int fd; + + if ((fd = open(filename.c_str(), O_RDONLY)) != -1) { + char buffer[32]; + int rc; + + if ((rc = read(fd, buffer, sizeof(buffer))) > 0) { + if (rc > 16 && + strncmp(buffer, "SQLite format 3", 16) == 0) { + retval = FF_SQLITE_DB; + } + } + } + + + return retval; +} + +void attach_sqlite_db(const string &filename) +{ + static pcrecpp::RE db_name_converter("[^\\w]"); + + auto_mem stmt(sqlite3_finalize); + + if (sqlite3_prepare_v2(lnav_data.ld_db.in(), + "ATTACH DATABASE ? as ?", + -1, + stmt.out(), + NULL) != SQLITE_OK) { + return; + } + + if (sqlite3_bind_text(stmt.in(), 1, + filename.c_str(), filename.length(), + SQLITE_TRANSIENT) != SQLITE_OK) { + return; + } + + size_t base_start = filename.find_last_of("/\\"); + string db_name; + + if (base_start == string::npos) { + db_name = filename; + } + else { + db_name = filename.substr(base_start + 1); + } + + db_name_converter.GlobalReplace("_", &db_name); + + if (sqlite3_bind_text(stmt.in(), 2, + db_name.c_str(), db_name.length(), + SQLITE_TRANSIENT) != SQLITE_OK) { + return; + } + + sqlite3_step(stmt.in()); +} + /** * Functor used to compare files based on their device and inode number. */ @@ -1618,11 +2307,25 @@ static void watch_logfile(string filename, int fd, bool required) same_file(st)); if (file_iter == lnav_data.ld_files.end()) { - /* It's a new file, load it in. */ - logfile *lf = new logfile(filename, fd); - - lnav_data.ld_files.push_back(lf); - lnav_data.ld_text_source.tss_files.push_back(lf); + if (find(lnav_data.ld_other_files.begin(), + lnav_data.ld_other_files.end(), + filename) == lnav_data.ld_other_files.end()) { + file_format_t ff = detect_file_format(filename); + + switch (ff) { + case FF_SQLITE_DB: + lnav_data.ld_other_files.push_back(filename); + attach_sqlite_db(filename); + break; + default: + /* It's a new file, load it in. */ + logfile *lf = new logfile(filename, fd); + + lnav_data.ld_files.push_back(lf); + lnav_data.ld_text_source.tss_files.push_back(lf); + break; + } + } } else { /* The file is already loaded, but has been found under a different @@ -1752,7 +2455,7 @@ public: case xterm_mouse::XT_BUTTON1: if (this->lb_selection_start == vis_line_t(-1) && tc->get_inner_height() && - ((this->lb_scrollbar_y != -1) || (x >= (width - 2)))) { + ((this->lb_scrollbar_y != -1) || (x >= (int)(width - 2)))) { double top_pct, bot_pct, pct; int scroll_top, scroll_bottom, shift_amount = 0, new_top = 0; @@ -1883,7 +2586,7 @@ static void looper(void) readline_context search_context("search"); readline_context index_context("capture"); - readline_context sql_context("sql"); + readline_context sql_context("sql", NULL, false); textview_curses *tc; readline_curses rlc; int lpc; @@ -1904,112 +2607,6 @@ static void looper(void) lnav_data.ld_rl_view-> add_possibility(LNM_COMMAND, "graph", "([:= \\t]\\d+(?:\\.\\d+)?)"); - { - const char *sql_commands[] = { - "add", - "all", - "alter", - "analyze", - "asc", - "attach", - "begin", - "collate", - "column", - "commit", - "conflict", - "create", - "cross", - "database", - "delete", - "desc", - "detach", - "distinct", - "drop", - "end", - "except", - "explain", - "from", - "group", - "having", - "idle_msecs", - "index", - "indexed", - "inner", - "insert", - "intersect", - "join", - "left", - "limit", - "natural", - "offset", - "order", - "outer", - "pragma", - "reindex", - "rename", - "replace", - "rollback", - "select", - "table", - "transaction", - "trigger", - "union", - "unique", - "update", - "using", - "vacuum", - "view", - "where", - "when", - - // XXX do the following dynamically by reading sqlite_master - - "access_log", - "syslog_log", - "generic_log", - "glog_log", - "strace_log", - - "line_number", - "path", - "log_time", - "level", - "raw_line", - - "c_ip", - "cs_username", - "cs_method", - "cs_uri_stem", - "cs_uri_query", - "cs_version", - "sc_status", - "sc_bytes", - "cs_referer", - "cs_user_agent", - - "funcname", - "result", - "duration", - "arg0", - "arg1", - "arg2", - "arg3", - "arg4", - "arg5", - "arg6", - "arg7", - "arg8", - "arg9", - - NULL - }; - - for (int lpc = 0; sql_commands[lpc]; lpc++) { - lnav_data.ld_rl_view-> - add_possibility(LNM_SQL, "*", sql_commands[lpc]); - } - } - (void)signal(SIGINT, sigint); (void)signal(SIGTERM, sigint); (void)signal(SIGWINCH, sigwinch); @@ -2108,7 +2705,7 @@ static void looper(void) hist_source &hs = lnav_data.ld_db_source; hs.set_bucket_size(1); - hs.set_group_size(100); + hs.set_group_size(10); hs.set_label_source(&lnav_data.ld_db_rows); } @@ -2117,6 +2714,9 @@ static void looper(void) lnav_data.ld_max_fd = max(STDIN_FILENO, rlc.update_fd_set(lnav_data.ld_read_fds)); + lnav_data.ld_status[0].window_change(); + lnav_data.ld_status[1].window_change(); + execute_file(dotlnav_path("session")); while (lnav_data.ld_looping) { @@ -2241,6 +2841,8 @@ static void looper(void) resizeterm(size.ws_row, size.ws_col); } rlc.window_change(); + lnav_data.ld_status[0].window_change(); + lnav_data.ld_status[1].window_change(); lnav_data.ld_view_stack.top()->set_needs_update(); lnav_data.ld_winched = false; } @@ -2255,233 +2857,83 @@ class access_log_table : public log_vtab_impl { public: access_log_table() - : log_vtab_impl("access_log"), - alt_regex("([\\w\\.-]+) [\\w\\.-]+ ([\\w\\.-]+) " - "\\[[^\\]]+\\] \"(\\w+) ([^ \\?]+)(\\?[^ ]+)? " - "([\\w/\\.]+)\" (\\d+) " - "(\\d+|-)(?: \"([^\"]+)\" \"([^\"]+)\")?.*") { + : log_vtab_impl("access_log") { }; void get_columns(vector &cols) { - cols.push_back(vtab_column("c_ip", "text")); - cols.push_back(vtab_column("cs_username", "text")); - cols.push_back(vtab_column("cs_method", "text")); - cols.push_back(vtab_column("cs_uri_stem", "text")); - cols.push_back(vtab_column("cs_uri_query", "text")); - cols.push_back(vtab_column("cs_version", "text")); - cols.push_back(vtab_column("sc_status", "text")); - cols.push_back(vtab_column("sc_bytes", "int")); - cols.push_back(vtab_column("cs_referer", "text")); - cols.push_back(vtab_column("cs_user_agent", "text")); + cols.push_back(vtab_column("c_ip", SQLITE3_TEXT, "ipaddress")); + cols.push_back(vtab_column("cs_username", SQLITE3_TEXT)); + cols.push_back(vtab_column("cs_method", SQLITE3_TEXT)); + cols.push_back(vtab_column("cs_uri_stem", SQLITE3_TEXT)); + cols.push_back(vtab_column("cs_uri_query", SQLITE3_TEXT)); + cols.push_back(vtab_column("cs_version", SQLITE3_TEXT)); + cols.push_back(vtab_column("sc_status", SQLITE_INTEGER)); + cols.push_back(vtab_column("sc_bytes", SQLITE_INTEGER)); + cols.push_back(vtab_column("cs_referer", SQLITE3_TEXT)); + cols.push_back(vtab_column("cs_user_agent", SQLITE3_TEXT)); }; - void extract(const std::string &line, - int column, - sqlite3_context *ctx) { - string c_ip, cs_username, cs_method, cs_uri_stem, cs_uri_query; - string cs_version, sc_status, cs_referer, cs_user_agent; - string sc_bytes; - - if (!this->alt_regex.FullMatch(line, - &c_ip, - &cs_username, - &cs_method, - &cs_uri_stem, - &cs_uri_query, - &cs_version, - &sc_status, - &sc_bytes, - &cs_referer, - &cs_user_agent)) { - fprintf(stderr, "bad match! %d %s\n", column, line.c_str()); - } - switch (column) { - case 0: - sqlite3_result_text(ctx, - c_ip.c_str(), - c_ip.length(), - SQLITE_TRANSIENT); - break; - case 1: - sqlite3_result_text(ctx, - cs_username.c_str(), - cs_username.length(), - SQLITE_TRANSIENT); - break; - case 2: - sqlite3_result_text(ctx, - cs_method.c_str(), - cs_method.length(), - SQLITE_TRANSIENT); - break; - case 3: - sqlite3_result_text(ctx, - cs_uri_stem.c_str(), - cs_uri_stem.length(), - SQLITE_TRANSIENT); - break; - case 4: - sqlite3_result_text(ctx, - cs_uri_query.c_str(), - cs_uri_query.length(), - SQLITE_TRANSIENT); - break; - case 5: - sqlite3_result_text(ctx, - cs_version.c_str(), - cs_version.length(), - SQLITE_TRANSIENT); - break; - case 6: - sqlite3_result_text(ctx, - sc_status.c_str(), - sc_status.length(), - SQLITE_TRANSIENT); - break; - case 7: - { - int sc_bytes_int = 0; +}; - sscanf(sc_bytes.c_str(), "%d", &sc_bytes_int); - sqlite3_result_int64(ctx, sc_bytes_int); - } - break; - case 8: - sqlite3_result_text(ctx, - cs_referer.c_str(), - cs_referer.length(), - SQLITE_TRANSIENT); - break; - case 9: - sqlite3_result_text(ctx, - cs_user_agent.c_str(), - cs_user_agent.length(), - SQLITE_TRANSIENT); - break; - } +class syslog_log_table : public log_vtab_impl { +public: + + syslog_log_table() + : log_vtab_impl("syslog_log") { + }; + + void get_columns(vector &cols) { + cols.push_back(vtab_column("log_hostname", SQLITE3_TEXT)); + cols.push_back(vtab_column("log_pid", SQLITE3_TEXT)); }; -private: - pcrecpp::RE alt_regex; }; class glog_log_table : public log_vtab_impl { public: glog_log_table() - : log_vtab_impl("glog_log"), - slt_regex( - "\\s*(?:[IWECF])([0-9]*) ([0-9:.]*)" // level, date - "\\s*([0-9]*)" // thread - "\\s*(.*):(\\d*)\\]" // filename:number - "\\s*(.*)" - ) { + : log_vtab_impl("glog_log") { }; void get_columns(vector &cols) { - cols.push_back(vtab_column("timestamp", "text")); - cols.push_back(vtab_column("thread", "text")); - cols.push_back(vtab_column("src_file", "text")); - cols.push_back(vtab_column("src_line", "int")); - cols.push_back(vtab_column("message", "text")); - }; - - void extract(const std::string &line, - int column, - sqlite3_context *ctx) { - string date, time, thread, file, src_line, message = "0"; - - if (!this->slt_regex.FullMatch(line, - &date, - &time, - &thread, - &file, - &src_line, - &message - )) { - fprintf(stderr, "bad match! %s\n", line.c_str()); - } - struct tm log_time; - time_t now = ::time(NULL); - stringstream timestamp; - char buf[128]; - switch (column) { - case 0: - localtime_r(&now, &log_time); // need year data - strptime(date.data(), "%m%d", &log_time); - strftime(buf, sizeof(buf), "%Y-%m-%d", &log_time); - // C++11 can do this much more nicely: - //timestamp << std::put_time(&log_time, "%Y-%m-%d "); - timestamp - << buf - << " " - << time; - sqlite3_result_text(ctx, - timestamp.str().c_str(), - timestamp.str().length(), - SQLITE_TRANSIENT); - break; - case 1: - sqlite3_result_text(ctx, - thread.c_str(), - thread.length(), - SQLITE_TRANSIENT); - break; - case 2: - sqlite3_result_text(ctx, - file.c_str(), - file.length(), - SQLITE_TRANSIENT); - break; - case 3: - sqlite3_result_text(ctx, - src_line.c_str(), - src_line.length(), - SQLITE_TRANSIENT); - break; - case 4: - sqlite3_result_text(ctx, - message.c_str(), - message.length(), - SQLITE_TRANSIENT); - break; - default: - fprintf(stderr, "bad match! %s\n", line.c_str()); - break; - } + cols.push_back(vtab_column("micros", SQLITE_INTEGER)); + cols.push_back(vtab_column("thread", SQLITE3_TEXT)); + cols.push_back(vtab_column("src_file", SQLITE3_TEXT)); + cols.push_back(vtab_column("src_line", SQLITE_INTEGER)); + cols.push_back(vtab_column("message", SQLITE3_TEXT)); }; - -private: - pcrecpp::RE slt_regex; }; class strace_log_table : public log_vtab_impl { public: strace_log_table() - : log_vtab_impl("strace_log"), - slt_regex("[0-9:.]* ([a-zA-Z_][a-zA-Z_0-9]*)\\(" - "(.*)\\)" - "\\s+= ([-xa-fA-F\\d\\?]+).*(?:<(\\d+\\.\\d+)>)?") { + : log_vtab_impl("strace_log") { }; void get_columns(vector &cols) { - cols.push_back(vtab_column("funcname", "text")); - cols.push_back(vtab_column("result", "text")); - cols.push_back(vtab_column("duration", "text")); - cols.push_back(vtab_column("arg0", "text")); - cols.push_back(vtab_column("arg1", "text")); - cols.push_back(vtab_column("arg2", "text")); - cols.push_back(vtab_column("arg3", "text")); - cols.push_back(vtab_column("arg4", "text")); - cols.push_back(vtab_column("arg5", "text")); - cols.push_back(vtab_column("arg6", "text")); - cols.push_back(vtab_column("arg7", "text")); - cols.push_back(vtab_column("arg8", "text")); - cols.push_back(vtab_column("arg9", "text")); + cols.push_back(vtab_column("funcname", SQLITE_TEXT)); + cols.push_back(vtab_column("args", SQLITE_TEXT)); + cols.push_back(vtab_column("result", SQLITE_TEXT)); + cols.push_back(vtab_column("duration", SQLITE_TEXT)); +#if 0 + cols.push_back(vtab_column("arg0", SQLITE_TEXT)); + cols.push_back(vtab_column("arg1", SQLITE_TEXT)); + cols.push_back(vtab_column("arg2", SQLITE_TEXT)); + cols.push_back(vtab_column("arg3", SQLITE_TEXT)); + cols.push_back(vtab_column("arg4", SQLITE_TEXT)); + cols.push_back(vtab_column("arg5", SQLITE_TEXT)); + cols.push_back(vtab_column("arg6", SQLITE_TEXT)); + cols.push_back(vtab_column("arg7", SQLITE_TEXT)); + cols.push_back(vtab_column("arg8", SQLITE_TEXT)); + cols.push_back(vtab_column("arg9", SQLITE_TEXT)); +#endif }; - void extract(const std::string &line, +#if 0 + void extract(logfile *lf, + const std::string &line, int column, sqlite3_context *ctx) { string function, args, result, duration = "0"; @@ -2581,138 +3033,7 @@ public: private: pcrecpp::RE slt_regex; -}; - -class log_data_table : public log_vtab_impl { -public: - - log_data_table() - : log_vtab_impl("log_data") { - this->ldt_pair_iter = this->ldt_pairs.end(); - }; - - void get_columns(vector &cols) { - cols.push_back(vtab_column("qualifier", "text")); - cols.push_back(vtab_column("key", "text")); - cols.push_back(vtab_column("subindex", "int")); - cols.push_back(vtab_column("value", "text")); - }; - - bool next(log_cursor &lc, logfile_sub_source &lss) { - fprintf(stderr, "next %d\n", (int)lc.lc_curr_line); - if (this->ldt_pair_iter == this->ldt_pairs.end()) { - fprintf(stderr, "try %d\n", (int)lc.lc_curr_line); - log_vtab_impl::next(lc, lss); - this->ldt_pairs.clear(); - - fprintf(stderr, "esc %d\n", (int)lc.lc_curr_line); - if (lc.lc_curr_line < (int)lss.text_line_count()) { - content_line_t cl(lss.at(lc.lc_curr_line)); - logfile *lf = lss.find(cl); - logfile::iterator line_iter; - string line; - - line_iter = lf->begin() + cl; - lf->read_line(line_iter, line); - - data_scanner ds(line); - data_parser dp(&ds); - - dp.parse(); - - fprintf(stderr, "got %zd\n", dp.dp_stack.size()); - while (!dp.dp_stack.empty()) { - fprintf(stderr, "got %d\n", dp.dp_stack.front().e_token); - if (dp.dp_stack.front().e_token == DNT_PAIR) { - this->ldt_pairs.splice(this->ldt_pairs.end(), - dp.dp_stack, - dp.dp_stack.begin()); - } - else { - dp.dp_stack.pop_front(); - } - } - - if (!this->ldt_pairs.empty()) { - this->ldt_pair_iter = this->ldt_pairs.begin(); - this->ldt_column = 0; - this->ldt_row_iter = - this->ldt_pair_iter->e_sub_elements->back().e_sub_elements->begin(); - return true; - } - } - else { - fprintf(stderr, "EOF %d %zd\n", - (int)lc.lc_curr_line, - lss.text_line_count()); - return true; - } - - return false; - } - else { - fprintf(stderr, "else %d\n", (int)lc.lc_curr_line); - ++(this->ldt_row_iter); - this->ldt_column += 1; - if (this->ldt_row_iter == this->ldt_pair_iter->e_sub_elements->back().e_sub_elements->end()) { - ++(this->ldt_pair_iter); - if (this->ldt_pair_iter != this->ldt_pairs.end()) { - this->ldt_row_iter = this->ldt_pair_iter->e_sub_elements->back().e_sub_elements->begin(); - this->ldt_column = 0; - - lc.lc_sub_index += 1; - return true; - } - return false; - } - if (this->ldt_pair_iter == this->ldt_pairs.end()) { - return false; - } - - lc.lc_sub_index += 1; - - return true; - } - }; - - void extract(const std::string &line, - int column, - sqlite3_context *ctx) { - pcre_context::capture_t cap; - - fprintf(stderr, "col %d -- %s\n", column, line.c_str()); - switch (column) { - case 0: - sqlite3_result_text(ctx, - "", - 0, - SQLITE_TRANSIENT); - break; - case 1: - cap = this->ldt_pair_iter->e_sub_elements->front().e_capture; - sqlite3_result_text(ctx, - &(line.c_str()[cap.c_begin]), - cap.length(), - SQLITE_TRANSIENT); - break; - case 2: - sqlite3_result_int64( ctx, this->ldt_column ); - break; - case 3: - cap = this->ldt_row_iter->e_capture; - sqlite3_result_text(ctx, - &(line.c_str()[cap.c_begin]), - cap.length(), - SQLITE_TRANSIENT); - break; - } - }; - -private: - std::list ldt_pairs; - std::list::iterator ldt_pair_iter; - std::list::iterator ldt_row_iter; - int ldt_column; +#endif }; void ensure_dotlnav(void) @@ -2749,6 +3070,34 @@ static void setup_highlights(textview_curses::highlight_map_t &hm) highlighter(xpcre_compile("^\\@@ .*"), false, view_colors::VCR_DIFF_SECTION); hm["(ip"] = textview_curses:: highlighter(xpcre_compile("\\d+\\.\\d+\\.\\d+\\.\\d+")); + hm["(cdef"] = textview_curses:: + highlighter(xpcre_compile("^#\\s*(?:if|ifndef|ifdef|else|define|undef|endif)\\b")); +} + +extern "C" { + int RegisterExtensionFunctions(sqlite3 *db); +} + +int register_network_extension_functions(sqlite3 *db); + +int sql_progress(const struct log_cursor &lc) +{ + size_t total = lnav_data.ld_log_source.text_line_count(); + off_t off = lc.lc_curr_line; + + if (lnav_data.ld_window == NULL) + return 0; + + if (!lnav_data.ld_looping) + return 1; + + lnav_data.ld_bottom_source.update_loading(off, total); + lnav_data.ld_top_source.update_time(); + lnav_data.ld_status[LNS_TOP].do_update(); + lnav_data.ld_status[LNS_BOTTOM].do_update(); + refresh(); + + return 0; } int main(int argc, char *argv[]) @@ -2757,6 +3106,8 @@ int main(int argc, char *argv[]) auto_ptr stdin_reader; const char *stdin_out = NULL; + setlocale(LC_NUMERIC, ""); + /* If we statically linked against an ncurses library that had a non- * standard path to the terminfo database, we need to set this variable * so that it will try the default path. @@ -2770,17 +3121,26 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } + { + int register_collation_functions(sqlite3 *db); + + RegisterExtensionFunctions(lnav_data.ld_db.in()); + register_network_extension_functions(lnav_data.ld_db.in()); + register_collation_functions(lnav_data.ld_db.in()); + } + lnav_data.ld_program_name = argv[0]; lnav_data.ld_vtab_manager = - new log_vtab_manager(lnav_data.ld_db, lnav_data.ld_log_source); + new log_vtab_manager(lnav_data.ld_db, + lnav_data.ld_log_source, + sql_progress); - lnav_data.ld_vtab_manager->register_vtab(new log_vtab_impl("syslog_log")); + lnav_data.ld_vtab_manager->register_vtab(new syslog_log_table()); lnav_data.ld_vtab_manager->register_vtab(new log_vtab_impl("generic_log")); lnav_data.ld_vtab_manager->register_vtab(new access_log_table()); lnav_data.ld_vtab_manager->register_vtab(new glog_log_table()); lnav_data.ld_vtab_manager->register_vtab(new strace_log_table()); - lnav_data.ld_vtab_manager->register_vtab(new log_data_table()); DEFAULT_FILES.insert(make_pair(LNF_SYSLOG, string("var/log/messages"))); DEFAULT_FILES.insert(make_pair(LNF_SYSLOG, string("var/log/system.log"))); @@ -2801,6 +3161,12 @@ int main(int argc, char *argv[]) set_sub_source(&lnav_data.ld_graph_source); lnav_data.ld_views[LNV_DB]. set_sub_source(&lnav_data.ld_db_source); + lnav_data.ld_db_overlay.dos_labels = &lnav_data.ld_db_rows; + lnav_data.ld_views[LNV_DB]. + set_overlay_source(&lnav_data.ld_db_overlay); + lnav_data.ld_views[LNV_LOG]. + set_overlay_source(new field_overlay_source()); + lnav_data.ld_db_overlay.dos_hist_source = &lnav_data.ld_db_source; { setup_highlights(lnav_data.ld_views[LNV_LOG].get_highlights()); @@ -2841,6 +3207,15 @@ int main(int argc, char *argv[]) stdin_out = optarg; break; + /* + * Add an option that will write stdin to a file with timestamps. + * Maybe add two options, one to write to a file and one to add + * timestamps. If the file already exists, we should load in the + * existing contents and append any new content to the file. + * Timestamps should be added only at the beginning of a line and + * only after a delay period. + */ + case 'V': printf("%s\n", PACKAGE_STRING); exit(0); diff --git a/src/lnav.hh b/src/lnav.hh index 3d41d78e..d858e59d 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -120,6 +120,7 @@ struct _lnav_data { std::set< std::pair > ld_file_names; std::list ld_files; + std::list ld_other_files; sig_atomic_t ld_looping; sig_atomic_t ld_winched; unsigned long ld_flags; @@ -152,6 +153,8 @@ struct _lnav_data { hist_source ld_db_source; db_label_source ld_db_rows; + db_overlay_source ld_db_overlay; + std::vector ld_db_key_names; int ld_max_fd; fd_set ld_read_fds; @@ -168,6 +171,7 @@ void rebuild_indexes(bool force); std::string dotlnav_path(const char *sub); +void ensure_view(textview_curses *expected_tc); bool toggle_view(textview_curses *toggle_tc); #endif diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index a90b6ce2..f0d76c83 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -159,13 +159,15 @@ static string com_save_to(string cmdline, vector &args) return "error: expecting file name"; } - snprintf(command, sizeof(command), "echo -n %s", args[1].c_str()); + snprintf(command, sizeof(command), "/bin/echo -n %s", args[1].c_str()); if ((pfile = popen(command, "r")) == NULL) { return "error: unable to compute file name"; } - if (fgets(command, sizeof(command), pfile) == 0) + if (fgets(command, sizeof(command), pfile) == 0) { perror("fgets"); + return "error: unable to compute file name"; + } fclose(pfile); pfile = NULL; @@ -280,6 +282,20 @@ static string com_graph(string cmdline, vector &args) return retval; } +static string com_help(string cmdline, vector &args) +{ + string retval = ""; + + if (args.size() == 0) { + + } + else { + ensure_view(&lnav_data.ld_views[LNV_HELP]); + } + + return retval; +} + class pcre_filter : public logfile_filter { public: @@ -507,6 +523,7 @@ void init_lnav_commands(readline_context::command_map_t &cmd_map) cmd_map["current-time"] = com_current_time; cmd_map["goto"] = com_goto; cmd_map["graph"] = com_graph; + cmd_map["help"] = com_help; cmd_map["highlight"] = com_highlight; cmd_map["filter-in"] = com_filter; cmd_map["filter-out"] = com_filter; diff --git a/src/log_format.hh b/src/log_format.hh index 96348936..6ca73a72 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -41,6 +41,8 @@ #include #include +#include "view_curses.hh" + /** * Metadata for a single line in a log file. */ @@ -139,6 +141,65 @@ private: uint8_t ll_module; }; +class logline_value { + +public: + enum kind_t { + VALUE_TEXT, + VALUE_INTEGER, + VALUE_FLOAT, + }; + + logline_value(std::string name, int64_t i) + : lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i) { }; + logline_value(std::string name, double i) + : lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i) { }; + logline_value(std::string name, std::string s) + : lv_name(name), lv_kind(VALUE_TEXT), lv_string(s) { }; + logline_value(std::string name, kind_t kind, std::string s) + : lv_name(name), lv_kind(kind) { + switch (kind) { + case VALUE_TEXT: + this->lv_string = s; + break; + case VALUE_INTEGER: + sscanf(s.c_str(), "%qd", &this->lv_number.i); + break; + case VALUE_FLOAT: + sscanf(s.c_str(), "%lf", &this->lv_number.d); + break; + } + }; + + const std::string to_string() { + char buffer[128]; + switch (this->lv_kind) { + case VALUE_TEXT: + return this->lv_string; + case VALUE_INTEGER: + snprintf(buffer, sizeof(buffer), "%qd", this->lv_number.i); + break; + case VALUE_FLOAT: + snprintf(buffer, sizeof(buffer), "%lf", this->lv_number.d); + break; + } + + return std::string(buffer); + }; + + std::string lv_name; + kind_t lv_kind; + union value_u { + int64_t i; + double d; + + value_u() : i(0) { }; + value_u(int64_t i) : i(i) { }; + value_u(double d) : d(d) { }; + } lv_number; + std::string lv_string; +}; + /** * Base class for implementations of log format parsers. */ @@ -199,6 +260,11 @@ public: * @param line The log line to edit. */ virtual void scrub(std::string &line) { }; + + virtual void annotate(const std::string &line, + string_attrs_t &sa, + std::vector &values) const + { }; virtual std::auto_ptr specialized(void) = 0; diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 556970ad..642b8fed 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -76,6 +76,17 @@ static string scrub_rdns(const string &str) } class access_log_format : public log_format { + + static pcrepp &value_pattern(void) { + static pcrepp VALUE_PATTERN( + "^([\\w\\.-]+) [\\w\\.-]+ ([\\w\\.-]+) " + "\\[([^\\]]+)\\] \"(?:\\-|(\\w+) ([^ \\?]+)(\\?[^ ]+)? " + "([\\w/\\.]+))\" (\\d+) " + "(\\d+|-)(?: \"([^\"]+)\" \"([^\"]+)\")?.*"); + + return VALUE_PATTERN; + }; + string get_name() { return "access_log"; }; bool scan(vector < logline > &dst, @@ -126,12 +137,75 @@ class access_log_format : public log_format { return retval; }; + + void annotate(const std::string &line, + string_attrs_t &sa, + std::vector &values) const { + pcre_context_static<30> pc; + pcre_input pi(line); + + if (value_pattern().match(pc, pi)) { + static struct { + const char *name; + logline_value::kind_t kind; + } columns[] = { + { "c_ip", logline_value::VALUE_TEXT }, + { "cs_username", logline_value::VALUE_TEXT }, + { "", }, + { "cs_method", logline_value::VALUE_TEXT }, + { "cs_uri_stem", logline_value::VALUE_TEXT }, + { "cs_uri_query", logline_value::VALUE_TEXT }, + { "cs_version", logline_value::VALUE_TEXT }, + { "sc_status", logline_value::VALUE_INTEGER }, + { "sc_bytes", logline_value::VALUE_INTEGER }, + { "cs_referer", logline_value::VALUE_TEXT }, + { "cs_user_agent", logline_value::VALUE_TEXT }, + + { NULL }, + }; + + pcre_context::iterator iter; + struct line_range lr; + + iter = pc.begin() + 3; + lr.lr_start = iter->c_begin; + lr.lr_end = iter->c_end; + sa[lr].insert(make_string_attr("timestamp", 0)); + + lr.lr_start = 0; + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("prefix", 0)); + + lr.lr_start = line.length(); + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("body", 0)); + + for (int lpc = 0; columns[lpc].name; lpc++) { + if (columns[lpc].name[0] == '\0') + continue; + values.push_back(logline_value(columns[lpc].name, + columns[lpc].kind, + pi.get_substr(pc.begin() + lpc))); + } + } + else { + fprintf(stderr, "bad match! %s\n", line.c_str()); + } + }; }; log_format::register_root_format access_log_instance; class syslog_log_format : public log_format { + static const int TIMESTAMP_LENGTH = 16; + + static pcrepp &repeated_pattern(void) { + static pcrepp REPEATED_PATTERN("last message repeated \\d+ times"); + + return REPEATED_PATTERN; + } + static pcrepp &scrub_pattern(void) { static pcrepp SCRUB_PATTERN("(\\w+\\s[\\s\\d]\\d \\d+:\\d+:\\d+) [\\.\\-\\w]+( .*)"); @@ -169,6 +243,90 @@ class syslog_log_format : public log_format { } }; + void annotate(const string &line, + string_attrs_t &sa, + std::vector &values) const { + bool found_hostname = false, found_procname = false, found_prefix = false; + struct line_range lr, hostname_range = { 0, 0 }; + struct line_range procname_range = { 0, 0 }; + int log_pid = -1; + + lr.lr_start = 0; + lr.lr_end = TIMESTAMP_LENGTH; + sa[lr].insert(make_string_attr("timestamp", 0)); + + for (size_t lpc = TIMESTAMP_LENGTH; lpc < line.size(); lpc++) { + if (line[lpc] == ' ' && !found_hostname) { + hostname_range.lr_start = TIMESTAMP_LENGTH; + hostname_range.lr_end = lpc; + sa[hostname_range].insert(make_string_attr("log_hostname", 0)); + + found_hostname = true; + } + + if (line[lpc] == ' ' && found_hostname && !found_procname) { + bool done = false; + + procname_range.lr_start = procname_range.lr_end = lpc + 1; + while (!done) { + switch (line[lpc]) { + case '\0': + done = true; + break; + case ':': + case '[': + procname_range.lr_end = lpc; + done = true; + break; + default: + lpc += 1; + break; + } + } + + sa[procname_range].insert(make_string_attr("log_procname", 0)); + + found_procname = true; + } + + if (line[lpc] == '[' && log_pid == -1) { + const char *line_c_str = line.c_str(); + + sscanf(&line_c_str[lpc + 1], "%d", &log_pid); + } + + if (line[lpc] == ':') { + lr.lr_start = 0; + lr.lr_end = lpc + 1; + sa[lr].insert(make_string_attr("prefix", 0)); + + lr.lr_start = lpc + 1; + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("body", 0)); + + found_prefix = true; + break; + } + } + + if (!found_prefix) { + lr.lr_start = 0; + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("prefix", 0)); + + lr.lr_start = line.length(); + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("body", 0)); + + hostname_range.lr_start = 0; + hostname_range.lr_end = 0; + } + + values.push_back(logline_value("log_hostname", line.substr(hostname_range.lr_start, hostname_range.length()))); + values.push_back(logline_value("log_procname", line.substr(procname_range.lr_start, procname_range.length()))); + values.push_back(logline_value("log_pid", (int64_t)log_pid)); + }; + bool scan(vector < logline > &dst, off_t offset, char *prefix, @@ -192,7 +350,10 @@ class syslog_log_format : public log_format { logline::level_t ll = logline::LEVEL_UNKNOWN; time_t log_gmt; - if (error_pattern().match(context, pi)) { + if (repeated_pattern().match(context, pi)) { + ll = logline::LEVEL_CONTINUED; + } + else if (error_pattern().match(context, pi)) { ll = logline::LEVEL_ERROR; } else if (warning_pattern().match(context, pi)) { @@ -281,6 +442,24 @@ class generic_log_format : public log_format { return SCRUB_PATTERN; } + static const char**get_log_formats() { + static const char *log_fmt[] = { + "%63[0-9: ,.-]%63[^:]%n", + "%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]%n", + "%63[a-zA-Z0-9:.,-] %63[^\n]%n", + "%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]%n", + "%63[a-zA-Z0-9: .,-] %63[^\n]%n", + "[%63[0-9: .-] %*s %63[^\n]%n", + "[%63[a-zA-Z0-9: -+/]] %63[^\n]%n", + "[%63[a-zA-Z0-9: -+/]] [%63[a-zA-Z]]%n", + "[%63[a-zA-Z0-9: .-+/] %*s %63[^\n]%n", + "[%63[a-zA-Z0-9: -+/]] (%*d) %63[^\n]%n", + NULL + }; + + return log_fmt; + }; + string get_name() { return "generic_log"; }; void scrub(string &line) { @@ -303,19 +482,6 @@ class generic_log_format : public log_format { off_t offset, char *prefix, int len) { - static const char *log_fmt[] = { - "%63[0-9: ,.-]%63[^:]", - "%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]", - "%63[a-zA-Z0-9:.,-] %63[^\n]", - "%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]", - "%63[a-zA-Z0-9: .,-] %63[^\n]", - "[%63[0-9: .-] %*s %63[^\n]", - "[%63[a-zA-Z0-9: -+/]] %63[^\n]", - "[%63[a-zA-Z0-9: -+/]] [%63[a-zA-Z]]", - "[%63[a-zA-Z0-9: .-+/] %*s %63[^\n]", - "[%63[a-zA-Z0-9: -+/]] (%*d) %63[^\n]", - NULL - }; bool retval = false; struct tm log_time; @@ -323,9 +489,10 @@ class generic_log_format : public log_format { time_t line_time; char level[64]; char *last_pos; + int prefix_len; if ((last_pos = this->log_scanf(prefix, - log_fmt, + get_log_formats(), 2, NULL, timestr, @@ -333,7 +500,8 @@ class generic_log_format : public log_format { line_time, timestr, - level)) != NULL) { + level, + &prefix_len)) != NULL) { uint16_t millis = 0; /* Try to pull out the milliseconds value. */ @@ -352,6 +520,30 @@ class generic_log_format : public log_format { return retval; }; + void annotate(const string &line, + string_attrs_t &sa, + std::vector &values) const { + const char *fmt = get_log_formats()[this->lf_fmt_lock]; + char timestr[64 + 32] = ""; + char level[64] = ""; + struct line_range lr; + int prefix_len; + + sscanf(line.c_str(), fmt, timestr, level, &prefix_len); + + lr.lr_start = 0 ? fmt[0] == '%' : 1; + lr.lr_end = lr.lr_start + strlen(timestr); + sa[lr].insert(make_string_attr("timestamp", 0)); + + lr.lr_start = 0; + lr.lr_end = prefix_len; + sa[lr].insert(make_string_attr("prefix", 0)); + + lr.lr_start = prefix_len; + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("body", 0)); + }; + auto_ptr specialized() { auto_ptr retval((log_format *) new generic_log_format(*this)); @@ -363,6 +555,16 @@ class generic_log_format : public log_format { log_format::register_root_format generic_log_instance; class glog_log_format : public log_format { + static pcrepp &value_pattern(void) { + static pcrepp VALUE_PATTERN( + "\\s*(?:[IWECF])\\d+ \\d+:\\d+:\\d+\\.(\\d+)" // level, date + "\\s*([0-9]*)" // thread + "\\s*(.*):(\\d*)\\]" // filename:number + "\\s*(.*)"); + + return VALUE_PATTERN; + }; + string get_name() { return "glog_log"; }; bool scan(vector < logline > &dst, @@ -419,11 +621,68 @@ class glog_log_format : public log_format { return retval; }; + void annotate(const std::string &line, + string_attrs_t &sa, + std::vector &values) const { + pcre_context_static<30> pc; + pcre_input pi(line); + + if (value_pattern().match(pc, pi)) { + static struct { + const char *name; + logline_value::kind_t kind; + } columns[] = { + { "micros", logline_value::VALUE_INTEGER }, + { "thread", logline_value::VALUE_TEXT }, + { "src_file", logline_value::VALUE_TEXT }, + { "src_line", logline_value::VALUE_INTEGER }, + { "message", logline_value::VALUE_TEXT }, + + { NULL } + }; + + pcre_context::iterator iter; + struct line_range lr; + + lr.lr_start = 1; + lr.lr_end = 21; + sa[lr].insert(make_string_attr("timestamp", 0)); + + iter = pc.begin() + 4; + lr.lr_start = 0; + lr.lr_end = iter->c_begin; + sa[lr].insert(make_string_attr("prefix", 0)); + + lr.lr_start = iter->c_begin; + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("body", 0)); + + for (int lpc = 0; columns[lpc].name; lpc++) { + if (columns[lpc].name[0] == '\0') + continue; + values.push_back(logline_value(columns[lpc].name, + columns[lpc].kind, + pi.get_substr(pc.begin() + lpc))); + } + } + else { + fprintf(stderr, "bad match! %s\n", line.c_str()); + } + }; }; log_format::register_root_format glog_instance; class strace_log_format : public log_format { + static pcrepp &value_pattern(void) { + static pcrepp VALUE_PATTERN( + "[0-9:.]* ([a-zA-Z_][a-zA-Z_0-9]*)\\(" + "(.*)\\)" + "\\s+= ([-xa-fA-F\\d\\?]+).*(?:<(\\d+\\.\\d+)>)?"); + + return VALUE_PATTERN; + }; + string get_name() { return "strace_log"; }; bool scan(vector < logline > &dst, @@ -486,6 +745,54 @@ class strace_log_format : public log_format { return retval; }; + + void annotate(const std::string &line, + string_attrs_t &sa, + std::vector &values) const { + pcre_context_static<30> pc; + pcre_input pi(line); + + if (value_pattern().match(pc, pi)) { + static struct { + const char *name; + logline_value::kind_t kind; + } columns[] = { + { "funcname", logline_value::VALUE_TEXT }, + { "args", logline_value::VALUE_TEXT }, + { "result", logline_value::VALUE_TEXT }, + { "duration", logline_value::VALUE_TEXT }, + + { NULL }, + }; + + pcre_context::iterator iter; + struct line_range lr; + + iter = pc.begin() + 3; + lr.lr_start = iter->c_begin; + lr.lr_end = iter->c_end; + sa[lr].insert(make_string_attr("timestamp", 0)); + + lr.lr_start = 0; + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("prefix", 0)); + + lr.lr_start = line.length(); + lr.lr_end = line.length(); + sa[lr].insert(make_string_attr("body", 0)); + + for (int lpc = 0; columns[lpc].name; lpc++) { + if (columns[lpc].name[0] == '\0') + continue; + values.push_back(logline_value(columns[lpc].name, + columns[lpc].kind, + pi.get_substr(pc.begin() + lpc))); + } + } + else { + fprintf(stderr, "bad match! %s\n", line.c_str()); + } + }; }; log_format::register_root_format strace_log_instance; diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index 8754fcf5..69053ab0 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -35,25 +35,52 @@ using namespace std; +static struct log_cursor log_cursor_latest; + +static sql_progress_callback_t vtab_progress_callback; + +static const char *type_to_string(int type) +{ + switch (type) { + case SQLITE_FLOAT: + return "float"; + case SQLITE_INTEGER: + return "integer"; + case SQLITE_TEXT: + return "text"; + } + + assert("Invalid sqlite type"); + + return NULL; +} + static string declare_table_statement(log_vtab_impl *vi) { std::vector cols; std::vector::const_iterator iter; std::ostringstream oss; - + oss << "CREATE TABLE unused (\n" - << " line_number int,\n" - << " path text,\n" + << " line_number text,\n" << " log_time datetime,\n" << " idle_msecs int,\n" - << " level text,\n" - << " raw_line text"; + << " level text,\n"; vi->get_columns(cols); + vi->vi_column_count = cols.size(); for (iter = cols.begin(); iter != cols.end(); iter++) { - oss << ",\n"; - oss << " " << iter->vc_name << " " << iter->vc_type; + auto_mem coldecl; + + coldecl = sqlite3_mprintf(" %Q %s collate %Q,\n", + iter->vc_name, + type_to_string(iter->vc_type), + iter->vc_collator == NULL ? + "BINARY" : iter->vc_collator); + oss << coldecl; } - oss << "\n);"; + oss << " path text collate naturalnocase,\n" + << " raw_line text\n" + << ");"; return oss.str(); } @@ -68,6 +95,7 @@ struct vtab { struct vtab_cursor { sqlite3_vtab_cursor base; struct log_cursor log_cursor; + std::vector line_values; }; static int vt_destructor(sqlite3_vtab *p_svt); @@ -95,6 +123,9 @@ static int vt_create( sqlite3 *db, /* Declare the vtable's structure */ p_vt->vi = vm->lookup_impl(argv[3]); + if (p_vt->vi == NULL) { + return SQLITE_ERROR; + } p_vt->lss = vm->get_source(); rc = sqlite3_declare_vtab(db, declare_table_statement(p_vt->vi).c_str()); @@ -138,8 +169,7 @@ static int vt_open(sqlite3_vtab *p_svt, sqlite3_vtab_cursor **pp_cursor) vtab* p_vt = (vtab*)p_svt; p_vt->base.zErrMsg = NULL; - vtab_cursor *p_cur = - (vtab_cursor*)sqlite3_malloc(sizeof(vtab_cursor)); + vtab_cursor *p_cur = (vtab_cursor*)new vtab_cursor(); *pp_cursor = (sqlite3_vtab_cursor*)p_cur; @@ -156,7 +186,7 @@ static int vt_close(sqlite3_vtab_cursor *cur) vtab_cursor *p_cur = (vtab_cursor*)cur; /* Free cursor struct. */ - sqlite3_free(p_cur); + delete p_cur; return SQLITE_OK; } @@ -175,7 +205,13 @@ static int vt_next(sqlite3_vtab_cursor *cur) vtab *vt = (vtab *)cur->pVtab; bool done = false; + vc->line_values.clear(); do { + log_cursor_latest = vc->log_cursor; + if (vtab_progress_callback(log_cursor_latest)) { + done = true; + break; + } done = vt->vi->next(vc->log_cursor, *vt->lss); } while (!done); @@ -202,16 +238,6 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) sqlite3_result_int64( ctx, vc->log_cursor.lc_curr_line ); } break; - case VT_COL_PATH: - { - const string &fn = lf->get_filename(); - - sqlite3_result_text( ctx, - fn.c_str(), - fn.length(), - SQLITE_STATIC ); - } - break; case VT_COL_LOG_TIME: { time_t line_time; @@ -252,25 +278,57 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) SQLITE_STATIC); } break; - case VT_COL_RAW_LINE: - { - string line; - - lf->read_line(ll, line); - sqlite3_result_text(ctx, - line.c_str(), - line.length(), - SQLITE_TRANSIENT); - } - break; default: - { - logfile::iterator line_iter; - string line, value; - - line_iter = lf->begin() + cl; - lf->read_line(line_iter, line); - vt->vi->extract(line, col - VT_COL_MAX, ctx); + if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) { + int post_col_number = col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1; + + if (post_col_number == 0) + { + const string &fn = lf->get_filename(); + + sqlite3_result_text( ctx, + fn.c_str(), + fn.length(), + SQLITE_STATIC ); + } + else { + string line; + + lf->read_line(ll, line); + sqlite3_result_text(ctx, + line.c_str(), + line.length(), + SQLITE_TRANSIENT); + } + } + else { + if (vc->line_values.empty()) { + logfile::iterator line_iter; + string line, value; + + line_iter = lf->begin() + cl; + lf->read_line(line_iter, line); + vt->vi->extract(lf, line, vc->line_values); + } + + { + logline_value &lv = vc->line_values[col - VT_COL_MAX]; + + switch (lv.lv_kind) { + case logline_value::VALUE_TEXT: + sqlite3_result_text(ctx, + lv.lv_string.c_str(), + lv.lv_string.length(), + SQLITE_TRANSIENT); + break; + case logline_value::VALUE_INTEGER: + sqlite3_result_int64(ctx, lv.lv_number.i); + break; + case logline_value::VALUE_FLOAT: + sqlite3_result_double(ctx, lv.lv_number.d); + break; + } + } } break; } @@ -322,30 +380,64 @@ static sqlite3_module vtab_module = { NULL, /* xFindFunction - function overloading */ }; -log_vtab_manager::log_vtab_manager(sqlite3 *memdb, logfile_sub_source &lss) +static int progress_callback(void *ptr) +{ + int retval = 0; + + if (vtab_progress_callback != NULL) { + retval = vtab_progress_callback(log_cursor_latest); + } + + return retval; +} + +log_vtab_manager::log_vtab_manager(sqlite3 *memdb, + logfile_sub_source &lss, + sql_progress_callback_t pc) : vm_db(memdb), vm_source(lss) { sqlite3_create_module(this->vm_db, "log_vtab_impl", &vtab_module, this); + vtab_progress_callback = pc; + sqlite3_progress_handler(memdb, 10, progress_callback, NULL); } void log_vtab_manager::register_vtab(log_vtab_impl *vi) { if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) { - char *sql; - int rc; - - this->vm_impls[vi->get_name()] = vi; - - sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s " - "USING log_vtab_impl(%s)", - vi->get_name().c_str(), - vi->get_name().c_str()); - rc = sqlite3_exec(this->vm_db, - sql, - NULL, - NULL, - NULL); - assert(rc == SQLITE_OK); - - sqlite3_free(sql); + char *sql; + int rc; + + this->vm_impls[vi->get_name()] = vi; + + sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s " + "USING log_vtab_impl(%s)", + vi->get_name().c_str(), + vi->get_name().c_str()); + rc = sqlite3_exec(this->vm_db, + sql, + NULL, + NULL, + NULL); + assert(rc == SQLITE_OK); + + sqlite3_free(sql); + } +} + +void log_vtab_manager::unregister_vtab(std::string name) { + if (this->vm_impls.find(name) != this->vm_impls.end()) { + char *sql; + int rc; + + sql = sqlite3_mprintf("DROP TABLE %s ", name.c_str()); + rc = sqlite3_exec(this->vm_db, + sql, + NULL, + NULL, + NULL); + assert(rc == SQLITE_OK); + + sqlite3_free(sql); } + + this->vm_impls.erase(name); } diff --git a/src/log_vtab_impl.hh b/src/log_vtab_impl.hh index f2985fcd..ffbb0a59 100644 --- a/src/log_vtab_impl.hh +++ b/src/log_vtab_impl.hh @@ -41,11 +41,9 @@ enum { VT_COL_LINE_NUMBER, - VT_COL_PATH, VT_COL_LOG_TIME, VT_COL_IDLE_MSECS, VT_COL_LEVEL, - VT_COL_RAW_LINE, VT_COL_MAX }; @@ -59,11 +57,12 @@ struct log_cursor { class log_vtab_impl { public: struct vtab_column { - vtab_column(const char *name, const char *type) - : vc_name(name), vc_type(type) { }; + vtab_column(const char *name, int type, const char *collator=NULL) + : vc_name(name), vc_type(type), vc_collator(collator) { }; const char *vc_name; - const char *vc_type; + int vc_type; + const char *vc_collator; }; log_vtab_impl(const std::string name) @@ -84,6 +83,11 @@ public: content_line_t cl(lss.at(lc.lc_curr_line)); logfile *lf = lss.find(cl); + logfile::iterator lf_iter = lf->begin() + cl; + + if (lf_iter->get_level() & logline::LEVEL_CONTINUED) { + return false; + } log_format *format = lf->get_format(); if (format != NULL && format->get_name() == this->vi_name) @@ -94,22 +98,32 @@ public: virtual void get_columns(std::vector &cols) { }; - virtual void extract(const std::string &line, - int column, - sqlite3_context *ctx) { + virtual void extract(logfile *lf, + const std::string &line, + std::vector &values) { + log_format *format = lf->get_format(); + string_attrs_t sa; + + format->annotate(line, sa, values); }; + int vi_column_count; private: const std::string vi_name; }; +typedef int (*sql_progress_callback_t)(const log_cursor &lc); + class log_vtab_manager { public: - log_vtab_manager(sqlite3 *db, logfile_sub_source &lss); + log_vtab_manager(sqlite3 *db, + logfile_sub_source &lss, + sql_progress_callback_t); logfile_sub_source *get_source() { return &this->vm_source; }; void register_vtab(log_vtab_impl *vi); + void unregister_vtab(std::string name); log_vtab_impl *lookup_impl(std::string name) { return this->vm_impls[name]; }; diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index f39d10aa..2f7b6f1e 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -168,7 +168,7 @@ void logfile_sub_source::text_value_for_line(textview_curses &tc, size_t tab; assert(row >= 0); - assert(row < this->lss_index.size()); + assert((size_t)row < this->lss_index.size()); line = this->lss_index[row]; this->lss_token_file = this->find(line); @@ -309,13 +309,21 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv, lr.lr_end = -1; value_out[lr].insert(make_string_attr("file", this->lss_token_file)); - if (this->lss_token_date_end > 0 && - ((this->lss_token_line->get_time() / (60 * 60)) % 2) == 0) { - attrs = vc.attrs_for_role(view_colors::VCR_ALT_ROW); - lr.lr_start = time_offset_end; - lr.lr_end = this->lss_token_date_end; - - value_out[lr].insert(make_string_attr("style", attrs)); + if ((((this->lss_token_line->get_time() / (5 * 60)) % 2) == 0) && + !(this->lss_token_line->get_level() & logline::LEVEL_CONTINUED)) { + log_format *format = this->lss_token_file->get_format(); + std::vector line_values; + + format->annotate(this->lss_token_value, value_out, line_values); + + struct line_range time_range = find_string_attr_range(value_out, "timestamp"); + + if (time_range.lr_end != -1) { + time_range.lr_start += time_offset_end; + time_range.lr_end += time_offset_end; + attrs = vc.attrs_for_role(view_colors::VCR_ALT_ROW); + value_out[time_range].insert(make_string_attr("style", attrs)); + } } } @@ -394,6 +402,9 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force) ++start_line; } } + else { + this->lss_filtered_count += 1; + } start_line = con_line; action = logfile_filter::MAYBE; action_priority = -1; @@ -450,6 +461,9 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force) ++start_line; } } + else { + this->lss_filtered_count += 1; + } iter->ld_lines_indexed = iter->ld_file->size(); @@ -473,8 +487,12 @@ void logfile_sub_source::text_update_marks(vis_bookmarks &bm) bm[&BM_WARNINGS].clear(); bm[&BM_ERRORS].clear(); bm[&BM_FILES].clear(); - bm[&textview_curses::BM_USER].clear(); - bm[&textview_curses::BM_SEARCH].clear(); + + for (bookmarks::type::iterator iter = this->lss_user_marks.begin(); + iter != this->lss_user_marks.end(); + ++iter) { + bm[iter->first].clear(); + } for (; vl < (int)this->lss_index.size(); ++vl) { content_line_t cl = this->lss_index[vl]; diff --git a/src/network-extension-functions.cc b/src/network-extension-functions.cc new file mode 100644 index 00000000..f696df39 --- /dev/null +++ b/src/network-extension-functions.cc @@ -0,0 +1,169 @@ +/** + * Copyright (c) 2013, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file nextwork-extension-functions.cc + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "sqlite3.h" + +static void sql_gethostbyname(sqlite3_context *context, + int argc, sqlite3_value **argv) +{ + char buffer[INET6_ADDRSTRLEN]; + const char *name_in; + struct addrinfo *ai; + void *addr_ptr; + int rc; + + assert(argc >= 1 && argc <= 2); + + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { + sqlite3_result_null(context); + return; + } + + name_in = (const char *)sqlite3_value_text(argv[0]); + while ((rc = getaddrinfo(name_in, NULL, NULL, &ai)) == EAI_AGAIN) { + sqlite3_sleep(10); + } + if (rc != 0) { + sqlite3_result_text(context, name_in, -1, SQLITE_TRANSIENT); + return; + } + + switch (ai->ai_family) { + case AF_INET: + addr_ptr = &((struct sockaddr_in *)ai->ai_addr)->sin_addr; + break; + case AF_INET6: + addr_ptr = &((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr; + break; + } + + inet_ntop(ai->ai_family, addr_ptr, buffer, sizeof(buffer)); + + sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT); + + freeaddrinfo(ai); +} + +static void sql_gethostbyaddr(sqlite3_context *context, + int argc, sqlite3_value **argv) +{ + union { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + } sa; + const char *addr_str; + char buffer[NI_MAXHOST]; + int family, socklen; + char *addr_raw; + int rc; + + assert(argc == 1); + + if (sqlite3_value_type(argv[0]) == SQLITE_NULL) { + sqlite3_result_null(context); + return; + } + + addr_str = (const char *)sqlite3_value_text(argv[0]); + + memset(&sa, 0, sizeof(sa)); + if (strchr(addr_str, ':')) { + family = AF_INET6; + socklen = sizeof(struct sockaddr_in6); + sa.sin6.sin6_family = family; + addr_raw = (char *)&sa.sin6.sin6_addr; + } + else { + family = AF_INET; + socklen = sizeof(struct sockaddr_in); + sa.sin.sin_family = family; + addr_raw = (char *)&sa.sin.sin_addr; + } + + if (inet_pton(family, addr_str, addr_raw) != 1) { + sqlite3_result_text(context, addr_str, -1, SQLITE_TRANSIENT); + return; + } + + while ((rc = getnameinfo((struct sockaddr *)&sa, socklen, + buffer, sizeof(buffer), NULL, 0, 0)) == EAI_AGAIN) { + sqlite3_sleep(10); + } + + if (rc != 0) { + sqlite3_result_text(context, addr_str, -1, SQLITE_TRANSIENT); + return; + } + + sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT); +} + +int register_network_extension_functions(sqlite3 *db) +{ + static const struct { + const char *name; + char narg; + uint8_t text_rep; + void (*func)(sqlite3_context*,int,sqlite3_value**); + } plain_funcs[] = { + { "gethostbyname", 1, SQLITE_UTF8, sql_gethostbyname }, + { "gethostbyaddr", 1, SQLITE_UTF8, sql_gethostbyaddr }, + + { NULL } + }; + + int retval; + + for (int lpc = 0; plain_funcs[lpc].name; lpc++) { + retval = sqlite3_create_function(db, + plain_funcs[lpc].name, + plain_funcs[lpc].narg, + plain_funcs[lpc].text_rep, + NULL, + plain_funcs[lpc].func, + NULL, + NULL); + if (retval != SQLITE_OK) + return retval; + } + + return SQLITE_OK; +} diff --git a/src/pcrepp.hh b/src/pcrepp.hh index 8877bd33..37e56fac 100644 --- a/src/pcrepp.hh +++ b/src/pcrepp.hh @@ -100,7 +100,7 @@ public: iterator begin() { return pc_captures + 1; }; /** @return An iterator that refers to the end of the capture array. */ iterator end() { return pc_captures + pc_count; }; - + protected: pcre_context(capture_t *captures, int max_count) : pc_captures(captures), pc_max_count(max_count), pc_count(0) { }; @@ -110,6 +110,28 @@ protected: int pc_count; }; +struct capture_if_not { + capture_if_not(int begin) : cin_begin(begin) { }; + + bool operator()(const pcre_context::capture_t &cap) { + return cap.c_begin != this->cin_begin; + } + + int cin_begin; +}; + +inline +pcre_context::iterator skip_invalid_captures(pcre_context::iterator iter, + pcre_context::iterator pc_end) +{ + for (; iter != pc_end; ++iter) { + if (iter->c_begin == -1) + continue; + } + + return iter; +} + /** * A pcre_context that allocates storage for the capture array within the object * itself. @@ -152,6 +174,9 @@ public: }; std::string get_substr(pcre_context::const_iterator iter) const { + if (iter->c_begin == -1) { + return ""; + } return std::string(this->pi_string, iter->c_begin, iter->length()); diff --git a/src/readline_curses.cc b/src/readline_curses.cc index aab7d04b..129e127b 100644 --- a/src/readline_curses.cc +++ b/src/readline_curses.cc @@ -132,8 +132,13 @@ char *readline_context::completion_generator(const char *text, int state) for (iter = arg_possibilities->begin(); iter != arg_possibilities->end(); ++iter) { + int (*cmpfunc)(const char *, const char *, size_t); + fprintf(stderr, " cmp %s %s\n", text, iter->c_str()); - if (strncmp(text, iter->c_str(), len) == 0) { + cmpfunc = (loaded_context->is_case_sensitive() ? + strncmp : strncasecmp); + if (cmpfunc(text, iter->c_str(), len) == 0) { + fprintf(stderr, "match!!! %d %s %s\n", len, text, iter->c_str()); matches.push_back(*iter); } } @@ -143,6 +148,8 @@ char *readline_context::completion_generator(const char *text, int state) if (!matches.empty()) { retval = strdup(matches.back().c_str()); matches.pop_back(); + + fprintf(stderr, "comp gen %s\n", retval); } return retval; @@ -154,6 +161,8 @@ char **readline_context::attempted_completion(const char *text, { char **retval = NULL; + fprintf(stderr, "attempted %s\n", text); + if (loaded_context->rc_possibilities.find("*") != loaded_context->rc_possibilities.end()) { fprintf(stderr, "all poss\n"); arg_possibilities = &loaded_context->rc_possibilities["*"]; @@ -378,6 +387,9 @@ void readline_curses::start(void) rem_possibility(string(type), string(&msg[prompt_start])); } + else if (sscanf(msg, "cp:%d:%s", &context, type)) { + this->rc_contexts[context]->clear_possibilities(type); + } else { fprintf(stderr, "unhandled message: %s\n", msg); } @@ -594,6 +606,20 @@ void readline_curses::rem_possibility(int context, string type, string value) } } +void readline_curses::clear_possibilities(int context, string type) +{ + char buffer[1024]; + + snprintf(buffer, sizeof(buffer), + "cp:%d:%s", + context, type.c_str()); + if (reliable_send(this->rc_command_pipe[RCF_MASTER], + buffer, + strlen(buffer) + 1) == -1) { + perror("clear_possiblity: write failed"); + } +} + void readline_curses::do_update(void) { if (this->rc_active_context == -1) { diff --git a/src/readline_curses.hh b/src/readline_curses.hh index 69ef9e83..8387b8b7 100644 --- a/src/readline_curses.hh +++ b/src/readline_curses.hh @@ -63,8 +63,11 @@ public: std::vector &args); typedef std::map command_map_t; - readline_context(const std::string &name, command_map_t *commands = NULL) - : rc_name(name) + readline_context(const std::string &name, + command_map_t *commands = NULL, + bool case_sensitive = true) + : rc_name(name), + rc_case_sensitive(case_sensitive) { char *home; @@ -97,6 +100,16 @@ public: void load(void) { + char buffer[128]; + /* + * XXX Need to keep the input on a single line since the display screws + * up if it wraps around. + */ + snprintf(buffer, sizeof(buffer), + "set completion-ignore-case %s", + this->rc_case_sensitive ? "off" : "on"); + rl_parse_and_bind(buffer); // NOTE: buffer is modified + loaded_context = this; rl_attempted_completion_function = attempted_completion; history_set_history_state(&this->rc_history); @@ -126,6 +139,15 @@ public: this->rc_possibilities[type].erase(value); }; + void clear_possibilities(std::string type) + { + this->rc_possibilities[type].clear(); + }; + + bool is_case_sensitive(void) const { + return this->rc_case_sensitive; + }; + private: static char **attempted_completion(const char *text, int start, int end); static char *completion_generator(const char *text, int state); @@ -137,6 +159,7 @@ private: HISTORY_STATE rc_history; std::map > rc_possibilities; std::map > rc_prototypes; + bool rc_case_sensitive; }; /** @@ -211,6 +234,7 @@ public: void add_possibility(int context, std::string type, std::string value); void rem_possibility(int context, std::string type, std::string value); + void clear_possibilities(int context, std::string type); private: enum { diff --git a/src/statusview_curses.cc b/src/statusview_curses.cc index f78507b9..a84e170e 100644 --- a/src/statusview_curses.cc +++ b/src/statusview_curses.cc @@ -37,7 +37,7 @@ using namespace std; void statusview_curses::do_update(void) { - int top, attrs, field, field_count, left = 1, right; + int top, attrs, field, field_count, left = 1, right; view_colors &vc = view_colors::singleton(); unsigned long width, height; @@ -60,6 +60,7 @@ void statusview_curses::do_update(void) int x; val = sf.get_value(); + if (sf.is_right_justified()) { right -= 1 + sf.get_width(); x = right; diff --git a/src/statusview_curses.hh b/src/statusview_curses.hh index e9c091ba..7bc71240 100644 --- a/src/statusview_curses.hh +++ b/src/statusview_curses.hh @@ -53,7 +53,8 @@ public: sf_right_justify(false), sf_cylon(false), sf_cylon_pos(0), - sf_role(role) { }; + sf_role(role), + sf_share(0) { }; virtual ~status_field() { }; @@ -77,6 +78,14 @@ public: value = abbrev; } } + + if (this->sf_right_justify) { + int padding = this->sf_width - value.size(); + + if (padding > 2) { + value.insert(0, padding, ' '); + } + } this->sf_value = value; @@ -136,13 +145,23 @@ public: /** @param width The maximum display width, in characters. */ size_t get_width() const { return this->sf_width; }; + /** @param width The maximum display width, in characters. */ + void set_min_width(int width) { this->sf_min_width = width; }; + /** @param width The maximum display width, in characters. */ + size_t get_min_width() const { return this->sf_min_width; }; + + void set_share(int share) { this->sf_share = share; }; + int get_share() const { return this->sf_share; }; + protected: size_t sf_width; /*< The maximum display width, in chars. */ + size_t sf_min_width; /*< The maximum display width, in chars. */ bool sf_right_justify; bool sf_cylon; size_t sf_cylon_pos; attr_line_t sf_value; /*< The value to display for this field. */ view_colors::role_t sf_role; /*< The color role for this field. */ + int sf_share; }; /** @@ -187,6 +206,39 @@ public: void set_window(WINDOW *win) { this->sc_window = win; }; WINDOW *get_window() { return this->sc_window; }; + void window_change(void) { + int field_count = this->sc_source->statusview_fields(); + int remaining, total_shares = 0; + unsigned long width, height; + + getmaxyx(this->sc_window, height, width); + remaining = width - 4; + for (int field = 0; field < field_count; field++) { + status_field &sf = this->sc_source->statusview_value_for_field(field); + + remaining -= sf.get_share() ? sf.get_min_width() : sf.get_width(); + remaining -= 1; + total_shares += sf.get_share(); + } + + if (remaining < 2) { + remaining = 0; + } + + for (int field = 0; field < field_count; field++) { + status_field &sf = this->sc_source->statusview_value_for_field(field); + + if (sf.get_share()) { + int actual_width; + + actual_width = sf.get_min_width(); + actual_width += remaining / (sf.get_share() / total_shares); + + sf.set_width(actual_width); + } + } + }; + void do_update(void); private: diff --git a/src/strnatcmp.c b/src/strnatcmp.c new file mode 100644 index 00000000..775022ae --- /dev/null +++ b/src/strnatcmp.c @@ -0,0 +1,209 @@ +/* -*- mode: c; c-file-style: "k&r" -*- + + strnatcmp.c -- Perform 'natural order' comparisons of strings in C. + Copyright (C) 2000, 2004 by Martin Pool + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + + +/* partial change history: + * + * 2004-10-10 mbp: Lift out character type dependencies into macros. + * + * Eric Sosman pointed out that ctype functions take a parameter whose + * value must be that of an unsigned int, even on platforms that have + * negative chars in their default char type. + */ + +#include +#include +#include +#include + +#include "strnatcmp.h" + + +/* These are defined as macros to make it easier to adapt this code to + * different characters types or comparison functions. */ +static inline int +nat_isdigit(nat_char a) +{ + return isdigit((unsigned char) a); +} + + +static inline int +nat_isspace(nat_char a) +{ + return isspace((unsigned char) a); +} + + +static inline nat_char +nat_toupper(nat_char a) +{ + return toupper((unsigned char) a); +} + + + +static int +compare_right(int a_len, nat_char const *a, int b_len, nat_char const *b) +{ + int bias = 0; + + /* The longest run of digits wins. That aside, the greatest + value wins, but we can't know that it will until we've scanned + both numbers to know that they have the same magnitude, so we + remember it in BIAS. */ + for (;; a++, b++, a_len--, b_len--) { + if (a_len == 0 && b_len == 0) + return 0; + if (a_len == 0) + return -1; + if (b_len == 0) + return 1; + if (!nat_isdigit(*a) && !nat_isdigit(*b)) + return bias; + else if (!nat_isdigit(*a)) + return -1; + else if (!nat_isdigit(*b)) + return +1; + else if (*a < *b) { + if (!bias) + bias = -1; + } else if (*a > *b) { + if (!bias) + bias = +1; + } else if (!*a && !*b) + return bias; + } + + return 0; +} + + +static int +compare_left(int a_len, nat_char const *a, int b_len, nat_char const *b) +{ + /* Compare two left-aligned numbers: the first to have a + different value wins. */ + for (;; a++, b++, a_len--, b_len--) { + if (a_len == 0 && b_len == 0) + return 0; + if (a_len == 0) + return -1; + if (b_len == 0) + return 1; + if (!nat_isdigit(*a) && !nat_isdigit(*b)) + return 0; + else if (!nat_isdigit(*a)) + return -1; + else if (!nat_isdigit(*b)) + return +1; + else if (*a < *b) + return -1; + else if (*a > *b) + return +1; + } + + return 0; +} + + +static int strnatcmp0(int a_len, nat_char const *a, + int b_len, nat_char const *b, + int fold_case) +{ + int ai, bi; + nat_char ca, cb; + int fractional, result; + + assert(a && b); + ai = bi = 0; + while (1) { + if (ai >= a_len) + ca = 0; + else + ca = a[ai]; + if (bi >= b_len) + cb = 0; + else + cb = b[bi]; + + /* skip over leading spaces or zeros */ + while (nat_isspace(ca)) { + ai += 1; + if (ai >= a_len) + ca = 0; + else + ca = a[ai]; + } + + while (nat_isspace(cb)) { + bi += 1; + if (bi >= b_len) + cb = 0; + else + cb = b[bi]; + } + + /* process run of digits */ + if (nat_isdigit(ca) && nat_isdigit(cb)) { + fractional = (ca == '0' || cb == '0'); + + if (fractional) { + if ((result = compare_left(a_len - ai, a+ai, b_len - bi, b+bi)) != 0) + return result; + } else { + if ((result = compare_right(a_len - ai, a+ai, b_len - bi, b+bi)) != 0) + return result; + } + } + + if (!ca && !cb) { + /* The strings compare the same. Perhaps the caller + will want to call strcmp to break the tie. */ + return 0; + } + + if (fold_case) { + ca = nat_toupper(ca); + cb = nat_toupper(cb); + } + + if (ca < cb) + return -1; + else if (ca > cb) + return +1; + + ++ai; ++bi; + } +} + + + +int strnatcmp(int a_len, nat_char const *a, int b_len, nat_char const *b) { + return strnatcmp0(a_len, a, b_len, b, 0); +} + + +/* Compare, recognizing numeric string and ignoring case. */ +int strnatcasecmp(int a_len, nat_char const *a, int b_len, nat_char const *b) { + return strnatcmp0(a_len, a, b_len, b, 1); +} diff --git a/src/strnatcmp.h b/src/strnatcmp.h new file mode 100644 index 00000000..fe1ea19b --- /dev/null +++ b/src/strnatcmp.h @@ -0,0 +1,31 @@ +/* -*- mode: c; c-file-style: "k&r" -*- + + strnatcmp.c -- Perform 'natural order' comparisons of strings in C. + Copyright (C) 2000, 2004 by Martin Pool + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. +*/ + + +/* CUSTOMIZATION SECTION + * + * You can change this typedef, but must then also change the inline + * functions in strnatcmp.c */ +typedef char nat_char; + +int strnatcmp(int a_len, nat_char const *a, int b_len, nat_char const *b); +int strnatcasecmp(int a_len, nat_char const *a, int b_len, nat_char const *b); diff --git a/src/textview_curses.cc b/src/textview_curses.cc index 9b8787c4..1cb493a2 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -149,7 +149,7 @@ public: } } while (rc > 0); - for (int lpc = 0; lpc < range_queue.size(); lpc++) { + for (size_t lpc = 0; lpc < range_queue.size(); lpc++) { sa[range_queue[lpc]].insert(attr_queue[lpc]); } }; diff --git a/src/top_status_source.hh b/src/top_status_source.hh index f450cb29..b17d62f1 100644 --- a/src/top_status_source.hh +++ b/src/top_status_source.hh @@ -65,7 +65,8 @@ public: this->tss_fields[TSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS); this->tss_fields[TSF_FORMAT].set_width(15); this->tss_fields[TSF_FORMAT].right_justify(true); - this->tss_fields[TSF_FILENAME].set_width(35); // XXX + this->tss_fields[TSF_FILENAME].set_min_width(35); // XXX + this->tss_fields[TSF_FILENAME].set_share(1); this->tss_fields[TSF_FILENAME].right_justify(true); }; diff --git a/src/view_curses.cc b/src/view_curses.cc index 4f078006..4493df7b 100644 --- a/src/view_curses.cc +++ b/src/view_curses.cc @@ -2,10 +2,10 @@ * Copyright (c) 2007-2012, Timothy Stack * * All rights reserved. - * + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * * Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright notice, @@ -14,7 +14,7 @@ * * Neither the name of Timothy Stack nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. - * + * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -55,7 +55,7 @@ void view_curses::mvwattrline(WINDOW *window, size_t exp_index = 0; string full_line; - assert(lr.lr_end != -1); + assert(lr.lr_end >= 0); line_width = lr.length(); buffer = (char *)alloca(line_width + 1); @@ -95,13 +95,18 @@ void view_curses::mvwattrline(WINDOW *window, whline(window, ' ', lr.lr_end - full_line.size()); wattroff(window, attrs); + std::vector graphic_range; + std::vector graphic_in; + for (iter = sa.begin(); iter != sa.end(); ++iter) { struct line_range attr_range = iter->first; std::map::iterator tab_iter; - + assert(attr_range.lr_start >= 0); assert(attr_range.lr_end >= -1); + fprintf(stderr, "attr %d %d\n", attr_range.lr_start, attr_range.lr_end); + tab_iter = tab_list.lower_bound(attr_range.lr_start); if (tab_iter != tab_list.end()) attr_range.lr_start += (tab_iter->second - tab_iter->first) - 1; @@ -111,7 +116,7 @@ void view_curses::mvwattrline(WINDOW *window, if (tab_iter != tab_list.end()) attr_range.lr_end += (tab_iter->second - tab_iter->first) - 1; } - + attr_range.lr_start = max(0, attr_range.lr_start - lr.lr_start); if (attr_range.lr_end == -1) { attr_range.lr_end = line_width; @@ -128,19 +133,36 @@ void view_curses::mvwattrline(WINDOW *window, attrs = 0; for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) { - if (am_iter->first == "style") { - attrs |= am_iter->second.sa_int; - } - } - - /* This silliness is brought to you by a buggy old curses lib. */ - mvwinnstr(window, y, x + attr_range.lr_start, buffer, awidth); - wattron(window, attrs); - mvwaddnstr(window, y, x + attr_range.lr_start, buffer, awidth); - wattroff(window, attrs); - } - - attrs = text_attrs; /* Reset attrs to regular text. */ + if (am_iter->first == "style") { + attrs |= am_iter->second.sa_int; + } + } + + if (attrs != 0) { + fprintf(stderr, "text %d %d %x\n", y, x + attr_range.lr_start, attrs); + /* This silliness is brought to you by a buggy old curses lib. */ + mvwinnstr(window, y, x + attr_range.lr_start, buffer, awidth); + wattron(window, attrs); + mvwaddnstr(window, y, x + attr_range.lr_start, buffer, awidth); + wattroff(window, attrs); + } + for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) { + if (am_iter->first == "graphic") { + graphic_range.push_back(attr_range); + graphic_in.push_back(am_iter->second.sa_int | attrs); + } + } + } + + attrs = text_attrs; /* Reset attrs to regular text. */ + } + + for (size_t lpc = 0; lpc < graphic_range.size(); lpc++) { + for (int lpc2 = graphic_range[lpc].lr_start; + lpc2 < graphic_range[lpc].lr_end; + lpc2++) { + mvwaddch(window, y, lpc2, graphic_in[lpc]); + } } } @@ -180,6 +202,8 @@ view_colors::view_colors() this->vc_role_colors[VCR_DIFF_ADD] = COLOR_PAIR(VC_GREEN); this->vc_role_colors[VCR_DIFF_SECTION] = COLOR_PAIR(VC_MAGENTA); + this->vc_role_colors[VCR_SHADOW] = COLOR_PAIR(VC_GRAY); + for (lpc = 0; lpc < VCR__MAX; lpc++) { this->vc_role_reverse_colors[lpc] = this->vc_role_colors[lpc] | A_REVERSE; @@ -211,7 +235,7 @@ void view_colors::init(void) init_pair(VC_CYAN, COLOR_CYAN, COLOR_BLACK); init_pair(VC_GREEN, COLOR_GREEN, COLOR_BLACK); init_pair(VC_MAGENTA, COLOR_MAGENTA, COLOR_BLACK); - + init_pair(VC_BLUE_ON_WHITE, COLOR_BLUE, COLOR_WHITE); init_pair(VC_CYAN_ON_BLACK, COLOR_CYAN, COLOR_BLACK); init_pair(VC_GREEN_ON_WHITE, COLOR_GREEN, COLOR_WHITE); @@ -224,12 +248,14 @@ void view_colors::init(void) init_pair(VC_BLACK_ON_WHITE, COLOR_BLACK, COLOR_WHITE); init_pair(VC_RED_ON_WHITE, COLOR_RED, COLOR_WHITE); init_pair(VC_YELLOW_ON_WHITE, COLOR_YELLOW, COLOR_WHITE); - + init_pair(VC_WHITE_ON_GREEN, COLOR_WHITE, COLOR_GREEN); + + init_pair(VC_GRAY, COLOR_BLACK, COLOR_BLACK); } } -view_colors::role_t view_colors::next_highlight(void) +view_colors::role_t view_colors::next_highlight() { role_t retval = (role_t)(VCR__MAX + this->vc_next_highlight); @@ -238,3 +264,13 @@ view_colors::role_t view_colors::next_highlight(void) return retval; } + +view_colors::role_t view_colors::next_plain_highlight() +{ + role_t retval = (role_t)(VCR__MAX + this->vc_next_plain_highlight); + + this->vc_next_plain_highlight = (this->vc_next_plain_highlight + 2) % + (HL_COLOR_COUNT * 2); + + return retval; +} diff --git a/src/view_curses.hh b/src/view_curses.hh index dbf2c9dd..4e80c516 100644 --- a/src/view_curses.hh +++ b/src/view_curses.hh @@ -92,6 +92,8 @@ struct line_range { bool operator<(const struct line_range &rhs) const { if (this->lr_start < rhs.lr_start) return true; else if (this->lr_start > rhs.lr_start) return false; + + if (this->lr_end == rhs.lr_end) return false; if (this->lr_end < rhs.lr_end) return true; return false; @@ -143,12 +145,31 @@ typedef std::multimap attrs_map_t; /** A map of line ranges to attributes for that range. */ typedef std::map string_attrs_t; +inline struct line_range +find_string_attr_range(const string_attrs_t &sa, const std::string &name) { + struct line_range retval = { -1, -1 }; + + for (string_attrs_t::const_iterator iter = sa.begin(); + iter != sa.end(); + ++iter) { + attrs_map_t::const_iterator prefix_iter; + + if ((prefix_iter = iter->second.find(name)) != iter->second.end()) { + retval = iter->first; + break; + } + } + + return retval; +} + /** * A line that has attributes. */ class attr_line_t { public: attr_line_t() { }; + attr_line_t(const std::string &str) : al_string(str) { }; /** @return The string itself. */ std::string &get_string() { return this->al_string; }; @@ -319,6 +340,8 @@ public: VCR_DIFF_ADD, /*< Added line in a diff. */ VCR_DIFF_SECTION, /*< Section marker in a diff. */ + VCR_SHADOW, + VCR__MAX } role_t; @@ -357,7 +380,9 @@ public: * method will iterate through eight-or-so attributes combinations so there * is some variety in how text is highlighted. */ - role_t next_highlight(void); + role_t next_highlight(); + + role_t next_plain_highlight(); enum { VC_EMPTY = 0, /* XXX Dead color pair, doesn't work. */ @@ -381,6 +406,8 @@ public: VC_RED_ON_WHITE, VC_WHITE_ON_GREEN, + + VC_GRAY, }; private: @@ -397,6 +424,7 @@ private: int vc_role_reverse_colors[VCR__MAX + (HL_COLOR_COUNT * 2)]; /** The index of the next highlight color to use. */ int vc_next_highlight; + int vc_next_plain_highlight; }; /** diff --git a/test/Makefile.am b/test/Makefile.am index d978624a..839ab717 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -110,6 +110,7 @@ drive_data_scanner_SOURCES = \ ../src/data_parser.cc \ ../src/data_scanner.cc \ drive_data_scanner.cc +drive_data_scanner_LDADD = -lcrypto drive_view_colors_SOURCES = \ ../src/view_curses.cc \ @@ -151,6 +152,8 @@ dist_noinst_DATA = \ datafile_simple.3 \ datafile_simple.4 \ datafile_simple.5 \ + datafile_simple.6 \ + datafile_simple.7 \ listview_output.0 \ listview_output.1 \ listview_output.2 \ diff --git a/test/Makefile.in b/test/Makefile.in index 5c8f3558..ad5ff9d0 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -86,7 +86,6 @@ CONFIG_CLEAN_VPATH_FILES = am_drive_data_scanner_OBJECTS = data_parser.$(OBJEXT) \ data_scanner.$(OBJEXT) drive_data_scanner.$(OBJEXT) drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS) -drive_data_scanner_LDADD = $(LDADD) drive_data_scanner_DEPENDENCIES = am_drive_grep_proc_OBJECTS = line_buffer.$(OBJEXT) grep_proc.$(OBJEXT) \ drive_grep_proc.$(OBJEXT) @@ -651,6 +650,7 @@ drive_data_scanner_SOURCES = \ ../src/data_scanner.cc \ drive_data_scanner.cc +drive_data_scanner_LDADD = -lcrypto drive_view_colors_SOURCES = \ ../src/view_curses.cc \ drive_view_colors.cc @@ -690,6 +690,8 @@ dist_noinst_DATA = \ datafile_simple.3 \ datafile_simple.4 \ datafile_simple.5 \ + datafile_simple.6 \ + datafile_simple.7 \ listview_output.0 \ listview_output.1 \ listview_output.2 \ diff --git a/test/datafile_simple.0 b/test/datafile_simple.0 index dfe23d9b..8530a530 100644 --- a/test/datafile_simple.0 +++ b/test/datafile_simple.0 @@ -1,17 +1,14 @@ a=1 b=2 c=3,4 - key 8:9 ^ - sep 9:10 ^ - num 10:11 ^ - num 12:13 ^ - row 10:13 ^-^ -pair 8:13 ^---^ - key 4:5 ^ - sep 5:6 ^ - num 6:7 ^ - row 6:7 ^ -pair 4:7 ^-^ - key 0:1 ^ - sep 1:2 ^ - num 2:3 ^ - row 2:3 ^ -pair 0:3 ^-^ + key 0:1 ^ a + num 2:3 ^ 1 + val 2:3 ^ 1 +pair 0:3 ^-^ a=1 + key 4:5 ^ b + num 6:7 ^ 2 + val 6:7 ^ 2 +pair 4:7 ^-^ b=2 + key 8:9 ^ c + num 10:11 ^ 3 + num 12:13 ^ 4 + val 10:13 ^-^ 3,4 +pair 8:13 ^---^ c=3,4 diff --git a/test/datafile_simple.1 b/test/datafile_simple.1 index b51bc116..ec0f6a1b 100644 --- a/test/datafile_simple.1 +++ b/test/datafile_simple.1 @@ -1,7 +1,7 @@ current speed: 38 mph - key 0:13 ^-----------^ - sep 13:14 ^ - num 15:17 ^^ -word 18:21 ^-^ - row 15:21 ^----^ -pair 0:17 ^---------------^ + key 0:0 + key 0:13 ^-----------^ current speed +pair 0:13 ^-----------^ current speed + key 15:15 ^ + num 15:17 ^^ 38 +pair 15:17 ^^ 38 diff --git a/test/datafile_simple.2 b/test/datafile_simple.2 index 16eb963b..f672dcf3 100644 --- a/test/datafile_simple.2 +++ b/test/datafile_simple.2 @@ -1,9 +1,16 @@ 1,2,3,4,five,six,7 - num 0:1 ^ - num 2:3 ^ - num 4:5 ^ - num 6:7 ^ -word 8:12 ^--^ -word 13:16 ^-^ - num 17:18 ^ - row 0:18 ^----------------^ + key 0:0 + num 0:1 ^ 1 +pair 0:1 ^ 1 + key 2:2 ^ + num 2:3 ^ 2 +pair 2:3 ^ 2 + key 4:4 ^ + num 4:5 ^ 3 +pair 4:5 ^ 3 + key 6:6 ^ + num 6:7 ^ 4 +pair 6:7 ^ 4 + key 17:17 ^ + num 17:18 ^ 7 +pair 17:18 ^ 7 diff --git a/test/datafile_simple.3 b/test/datafile_simple.3 index cf89b8a1..717b7588 100644 --- a/test/datafile_simple.3 +++ b/test/datafile_simple.3 @@ -1,9 +1,16 @@ 1 2 3 4 five six 7 - num 0:1 ^ - num 2:3 ^ - num 4:5 ^ - num 6:7 ^ -word 8:12 ^--^ -word 13:16 ^-^ - num 17:18 ^ - row 0:18 ^----------------^ + key 0:0 + num 0:1 ^ 1 +pair 0:1 ^ 1 + key 2:2 ^ + num 2:3 ^ 2 +pair 2:3 ^ 2 + key 4:4 ^ + num 4:5 ^ 3 +pair 4:5 ^ 3 + key 6:6 ^ + num 6:7 ^ 4 +pair 6:7 ^ 4 + key 17:17 ^ + num 17:18 ^ 7 +pair 17:18 ^ 7 diff --git a/test/datafile_simple.4 b/test/datafile_simple.4 index 483c1ce9..f1975fb8 100644 --- a/test/datafile_simple.4 +++ b/test/datafile_simple.4 @@ -1,6 +1,5 @@ the-value: "Hello, World!" - key 0:9 ^-------^ - sep 9:10 ^ -quot 12:25 ^-----------^ - row 12:25 ^-----------^ -pair 0:25 ^-----------------------^ + key 0:9 ^-------^ the-value +quot 12:25 ^-----------^ Hello, World! + val 12:25 ^-----------^ Hello, World! +pair 0:25 ^-----------------------^ the-value: "Hello, World! diff --git a/test/datafile_simple.5 b/test/datafile_simple.5 index f570327e..f734fa05 100644 --- a/test/datafile_simple.5 +++ b/test/datafile_simple.5 @@ -1,6 +1,5 @@ this is a url: http://www.example.com/foo-bar - key 0:13 ^-----------^ - sep 13:14 ^ - url 15:45 ^----------------------------^ - row 15:45 ^----------------------------^ -pair 0:45 ^-------------------------------------------^ + key 0:13 ^-----------^ this is a url + url 15:45 ^----------------------------^ http://www.example.com/foo-bar + val 15:45 ^----------------------------^ http://www.example.com/foo-bar +pair 0:45 ^-------------------------------------------^ this is a url: http://www.example.com/foo-bar diff --git a/test/datafile_simple.6 b/test/datafile_simple.6 index d7d6de4e..df5f1426 100644 --- a/test/datafile_simple.6 +++ b/test/datafile_simple.6 @@ -1,11 +1,9 @@ qualified:name: foo=1 bar=2 - key 22:25 ^-^ - sep 25:26 ^ - num 26:27 ^ - row 26:27 ^ -pair 22:27 ^---^ - key 16:19 ^-^ - sep 19:20 ^ - num 20:21 ^ - row 20:21 ^ -pair 16:21 ^---^ + key 16:19 ^-^ foo + num 20:21 ^ 1 + val 20:21 ^ 1 +pair 16:21 ^---^ foo=1 + key 22:25 ^-^ bar + num 26:27 ^ 2 + val 26:27 ^ 2 +pair 22:27 ^---^ bar=2 diff --git a/test/datafile_simple.7 b/test/datafile_simple.7 new file mode 100644 index 00000000..164a41b4 --- /dev/null +++ b/test/datafile_simple.7 @@ -0,0 +1,12 @@ + func(arg1="a", arg2="b") + key 5:5 ^ + key 5:9 ^--^ arg1 +quot 11:12 ^ a + val 11:12 ^ a +pair 5:12 ^-----^ arg1="a + key 15:19 ^--^ arg2 +quot 21:22 ^ b + val 21:22 ^ b +pair 15:22 ^-----^ arg2="b + grp 5:22 ^---------------^ arg1="a", arg2="b +pair 5:22 ^---------------^ arg1="a", arg2="b diff --git a/test/datafile_simple.8 b/test/datafile_simple.8 new file mode 100644 index 00000000..e76d5c7f --- /dev/null +++ b/test/datafile_simple.8 @@ -0,0 +1,31 @@ + Succeeded authorizing right 'system.privilege.taskport.debug' by client '/usr/libexec/taskgated' [76339] for authorization created by '/usr/libexec/taskgated' [77395] (100003,1) + key 29:29 ^ +quot 29:60 ^-----------------------------^ system.privilege.taskport.debug +pair 29:60 ^-----------------------------^ system.privilege.taskport.debug + key 73:73 ^ +quot 73:95 ^--------------------^ /usr/libexec/taskgated +pair 73:95 ^--------------------^ /usr/libexec/taskgated + key 98:98 ^ + key 98:98 ^ + num 98:103 ^---^ 76339 +pair 98:103 ^---^ 76339 + grp 98:103 ^---^ 76339 +pair 98:103 ^---^ 76339 + key 135:135 ^ +quot 135:157 ^--------------------^ /usr/libexec/taskgated +pair 135:157 ^--------------------^ /usr/libexec/taskgated + key 160:160 ^ + key 160:160 ^ + num 160:165 ^---^ 77395 +pair 160:165 ^---^ 77395 + grp 160:165 ^---^ 77395 +pair 160:165 ^---^ 77395 + key 168:168 ^ + key 168:168 ^ + num 168:174 ^----^ 100003 +pair 168:174 ^----^ 100003 + key 175:175 ^ + num 175:176 ^ 1 +pair 175:176 ^ 1 + grp 168:176 ^------^ 100003,1 +pair 168:176 ^------^ 100003,1 diff --git a/test/datafile_syslog.0 b/test/datafile_syslog.0 index 4d8b909a..361c76a0 100644 --- a/test/datafile_syslog.0 +++ b/test/datafile_syslog.0 @@ -1,32 +1,20 @@ - Nov 3 09:47:02 veridian sudo: timstack : TTY=pts/6 ; PWD=/auto/wstimstack/rpms/lbuild/test ; USER=root ; COMMAND=/usr/bin/tail /var/log/messages - key 106:113 ^-----^ - sep 113:114 ^ -path 114:127 ^-----------^ -path 128:145 ^---------------^ - row 114:145 ^-----------------------------^ -pair 106:127 ^-------------------^ - key 94:98 ^--^ - sep 98:99 ^ -word 99:103 ^--^ - row 99:103 ^--^ -pair 94:103 ^-------^ - key 54:57 ^-^ - sep 57:58 ^ -path 58:91 ^-------------------------------^ - row 58:91 ^-------------------------------^ -pair 54:91 ^-----------------------------------^ - key 42:45 ^-^ - sep 45:46 ^ -word 46:49 ^-^ -path 49:51 ^^ - row 46:51 ^---^ -pair 42:51 ^-------^ - key 16:29 ^-----------^ - sep 29:30 ^ -word 31:39 ^------^ - row 31:39 ^------^ -pair 16:39 ^---------------------^ -word 0:3 ^-^ - num 5:6 ^ -time 7:15 ^------^ -date 0:15 ^-------------^ + timstack : TTY=pts/6 ; PWD=/auto/wstimstack/rpms/lbuild/test ; USER=root ; COMMAND=/usr/bin/tail /var/log/messages + key 11:14 ^-^ TTY + sym 15:18 ^-^ pts + num 19:20 ^ 6 + val 15:20 ^---^ pts/6 +pair 11:20 ^-------^ TTY=pts/6 + key 23:26 ^-^ PWD +path 27:60 ^-------------------------------^ /auto/wstimstack/rpms/lbuild/test + val 27:60 ^-------------------------------^ /auto/wstimstack/rpms/lbuild/test +pair 23:60 ^-----------------------------------^ PWD=/auto/wstimstack/rpms/lbuild/test + key 63:67 ^--^ USER +word 68:72 ^--^ root + val 68:72 ^--^ root +pair 63:72 ^-------^ USER=root + key 75:82 ^-----^ COMMAND +path 83:96 ^-----------^ /usr/bin/tail +wspc 96:97 ^ +path 97:114 ^---------------^ /var/log/messages + val 83:114 ^-----------------------------^ /usr/bin/tail /var/log/messages +pair 75:114 ^-------------------------------------^ COMMAND=/usr/bin/tail /var/log/messages diff --git a/test/datafile_syslog.1 b/test/datafile_syslog.1 index 9b49337f..928f11a6 100644 --- a/test/datafile_syslog.1 +++ b/test/datafile_syslog.1 @@ -1,33 +1,21 @@ - Jun 18 16:13:52 Tim-Stacks-iMac Safari[81045]: INSERT-HANG-DETECTED: Tx time:3.093364, # of Inserts: 89, # of bytes written: 465365, Did shrink: NO - key 137:143 ^----^ - sep 143:144 ^ -word 145:147 ^^ - row 145:147 ^^ -pair 137:147 ^--------^ - key 107:123 ^--------------^ - sep 123:124 ^ - num 125:131 ^----^ -word 133:136 ^-^ - row 125:136 ^---------^ -pair 107:136 ^---------------------------^ - key 89:99 ^--------^ - sep 99:100 ^ - num 101:103 ^^ -word 105:106 ^ - row 101:106 ^---^ -pair 89:106 ^---------------^ - key 72:76 ^--^ - sep 76:77 ^ - num 77:85 ^------^ -word 87:88 ^ - row 77:88 ^---------^ -pair 72:88 ^--------------^ - key 47:67 ^------------------^ - sep 67:68 ^ -word 69:71 ^^ - row 69:71 ^^ -pair 47:71 ^----------------------^ -word 0:3 ^-^ - num 4:6 ^^ -time 7:15 ^------^ -date 0:15 ^-------------^ + INSERT-HANG-DETECTED: Tx time:3.093364, # of Inserts: 89, # of bytes written: 465365, Did shrink: NO + key 0:20 ^------------------^ INSERT-HANG-DETECTED +word 22:24 ^^ Tx + val 22:24 ^^ Tx +pair 0:24 ^----------------------^ INSERT-HANG-DETECTED: Tx + key 25:29 ^--^ time + num 30:38 ^------^ 3.093364 + val 30:38 ^------^ 3.093364 +pair 25:38 ^-----------^ time:3.093364 + key 40:52 ^----------^ # of Inserts + num 54:56 ^^ 89 + val 54:56 ^^ 89 +pair 40:56 ^--------------^ # of Inserts: 89 + key 58:76 ^----------------^ # of bytes written + num 78:84 ^----^ 465365 + val 78:84 ^----^ 465365 +pair 58:84 ^------------------------^ # of bytes written: 465365 + key 86:96 ^--------^ Did shrink + sym 98:100 ^^ NO + val 98:100 ^^ NO +pair 86:100 ^------------^ Did shrink: NO diff --git a/test/drive_data_scanner.cc b/test/drive_data_scanner.cc index 3c702aaa..e45035f2 100644 --- a/test/drive_data_scanner.cc +++ b/test/drive_data_scanner.cc @@ -105,12 +105,10 @@ int main(int argc, char *argv[]) } data_scanner ds(line.substr(13)); - data_token_t token; - data_parser dp(&ds); dp.parse(); - dp.print(out); + dp.print(out, dp.dp_pairs); fclose(out); sprintf(cmd, "diff -u %s %s", argv[0], TMP_NAME);