[sqlite] Major improvements to the sqlite integration.

This is a checkpoint of the improvements to the sqlite integration.  The
data_parser stuff should be much better now and I've tried to improve
other parts of the user experience as well.
pull/69/head
Timothy Stack 11 years ago
parent b04e6bfc78
commit 3128dc772c

4
configure vendored

@ -3751,8 +3751,8 @@ fi
#CFLAGS=`echo $CFLAGS | sed 's/-O2//g'` # CFLAGS=`echo $CFLAGS | sed 's/-O2//g'`
#CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'` # CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'`
# Check whether --enable-static was given. # Check whether --enable-static was given.
if test "${enable_static+set}" = set; then : if test "${enable_static+set}" = set; then :

@ -35,8 +35,8 @@ AC_SUBST(abssrcdir)
AC_PROG_CXX AC_PROG_CXX
#CFLAGS=`echo $CFLAGS | sed 's/-O2//g'` # CFLAGS=`echo $CFLAGS | sed 's/-O2//g'`
#CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'` # CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'`
AC_ARG_ENABLE([static], AC_ARG_ENABLE([static],
AS_HELP_STRING([--disable-static], AS_HELP_STRING([--disable-static],

@ -23,7 +23,8 @@ LDADD = \
$(READLINE_LIBS) \ $(READLINE_LIBS) \
$(CURSES_LIB) \ $(CURSES_LIB) \
$(SQLITE3_LIBS) \ $(SQLITE3_LIBS) \
-lpcrecpp -lpcrecpp \
-lcrypto
noinst_HEADERS = \ noinst_HEADERS = \
auto_fd.hh \ auto_fd.hh \
@ -32,6 +33,7 @@ noinst_HEADERS = \
bookmarks.hh \ bookmarks.hh \
bottom_status_source.hh \ bottom_status_source.hh \
byte_array.hh \ byte_array.hh \
column_namer.hh \
data_scanner.hh \ data_scanner.hh \
data_parser.hh \ data_parser.hh \
db_sub_source.hh \ db_sub_source.hh \
@ -55,6 +57,7 @@ noinst_HEADERS = \
sequence_matcher.hh \ sequence_matcher.hh \
sequence_sink.hh \ sequence_sink.hh \
statusview_curses.hh \ statusview_curses.hh \
strnatcmp.h \
strong_int.hh \ strong_int.hh \
termios_guard.hh \ termios_guard.hh \
textfile_sub_source.hh \ textfile_sub_source.hh \
@ -69,6 +72,8 @@ noinst_HEADERS = \
libdiag_a_SOURCES = \ libdiag_a_SOURCES = \
bookmarks.cc \ bookmarks.cc \
collation-functions.cc \
extension-functions.c \
grep_proc.cc \ grep_proc.cc \
hist_source.cc \ hist_source.cc \
line_buffer.cc \ line_buffer.cc \
@ -77,12 +82,14 @@ libdiag_a_SOURCES = \
log_format.cc \ log_format.cc \
logfile.cc \ logfile.cc \
logfile_sub_source.cc \ logfile_sub_source.cc \
network-extension-functions.cc \
data_scanner.cc \ data_scanner.cc \
data_parser.cc \ data_parser.cc \
readline_curses.cc \ readline_curses.cc \
sequence_matcher.cc \ sequence_matcher.cc \
statusview_curses.cc \ statusview_curses.cc \
piper_proc.cc \ piper_proc.cc \
strnatcmp.c \
textview_curses.cc \ textview_curses.cc \
view_curses.cc \ view_curses.cc \
vt52_curses.cc \ vt52_curses.cc \

@ -77,16 +77,19 @@ am__v_AR_0 = @echo " AR " $@;
am__v_AR_1 = am__v_AR_1 =
libdiag_a_AR = $(AR) $(ARFLAGS) libdiag_a_AR = $(AR) $(ARFLAGS)
libdiag_a_LIBADD = libdiag_a_LIBADD =
am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) grep_proc.$(OBJEXT) \ am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) \
hist_source.$(OBJEXT) line_buffer.$(OBJEXT) \ collation-functions.$(OBJEXT) extension-functions.$(OBJEXT) \
listview_curses.$(OBJEXT) lnav_commands.$(OBJEXT) \ grep_proc.$(OBJEXT) hist_source.$(OBJEXT) \
log_format.$(OBJEXT) logfile.$(OBJEXT) \ line_buffer.$(OBJEXT) listview_curses.$(OBJEXT) \
logfile_sub_source.$(OBJEXT) data_scanner.$(OBJEXT) \ lnav_commands.$(OBJEXT) log_format.$(OBJEXT) logfile.$(OBJEXT) \
logfile_sub_source.$(OBJEXT) \
network-extension-functions.$(OBJEXT) data_scanner.$(OBJEXT) \
data_parser.$(OBJEXT) readline_curses.$(OBJEXT) \ data_parser.$(OBJEXT) readline_curses.$(OBJEXT) \
sequence_matcher.$(OBJEXT) statusview_curses.$(OBJEXT) \ sequence_matcher.$(OBJEXT) statusview_curses.$(OBJEXT) \
piper_proc.$(OBJEXT) textview_curses.$(OBJEXT) \ piper_proc.$(OBJEXT) strnatcmp.$(OBJEXT) \
view_curses.$(OBJEXT) vt52_curses.$(OBJEXT) \ textview_curses.$(OBJEXT) view_curses.$(OBJEXT) \
log_vtab_impl.$(OBJEXT) xterm_mouse.$(OBJEXT) vt52_curses.$(OBJEXT) log_vtab_impl.$(OBJEXT) \
xterm_mouse.$(OBJEXT)
libdiag_a_OBJECTS = $(am_libdiag_a_OBJECTS) libdiag_a_OBJECTS = $(am_libdiag_a_OBJECTS)
am__installdirs = "$(DESTDIR)$(bindir)" am__installdirs = "$(DESTDIR)$(bindir)"
PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
@ -300,7 +303,8 @@ LDADD = \
$(READLINE_LIBS) \ $(READLINE_LIBS) \
$(CURSES_LIB) \ $(CURSES_LIB) \
$(SQLITE3_LIBS) \ $(SQLITE3_LIBS) \
-lpcrecpp -lpcrecpp \
-lcrypto
noinst_HEADERS = \ noinst_HEADERS = \
auto_fd.hh \ auto_fd.hh \
@ -309,6 +313,7 @@ noinst_HEADERS = \
bookmarks.hh \ bookmarks.hh \
bottom_status_source.hh \ bottom_status_source.hh \
byte_array.hh \ byte_array.hh \
column_namer.hh \
data_scanner.hh \ data_scanner.hh \
data_parser.hh \ data_parser.hh \
db_sub_source.hh \ db_sub_source.hh \
@ -332,6 +337,7 @@ noinst_HEADERS = \
sequence_matcher.hh \ sequence_matcher.hh \
sequence_sink.hh \ sequence_sink.hh \
statusview_curses.hh \ statusview_curses.hh \
strnatcmp.h \
strong_int.hh \ strong_int.hh \
termios_guard.hh \ termios_guard.hh \
textfile_sub_source.hh \ textfile_sub_source.hh \
@ -346,6 +352,8 @@ noinst_HEADERS = \
libdiag_a_SOURCES = \ libdiag_a_SOURCES = \
bookmarks.cc \ bookmarks.cc \
collation-functions.cc \
extension-functions.c \
grep_proc.cc \ grep_proc.cc \
hist_source.cc \ hist_source.cc \
line_buffer.cc \ line_buffer.cc \
@ -354,12 +362,14 @@ libdiag_a_SOURCES = \
log_format.cc \ log_format.cc \
logfile.cc \ logfile.cc \
logfile_sub_source.cc \ logfile_sub_source.cc \
network-extension-functions.cc \
data_scanner.cc \ data_scanner.cc \
data_parser.cc \ data_parser.cc \
readline_curses.cc \ readline_curses.cc \
sequence_matcher.cc \ sequence_matcher.cc \
statusview_curses.cc \ statusview_curses.cc \
piper_proc.cc \ piper_proc.cc \
strnatcmp.c \
textview_curses.cc \ textview_curses.cc \
view_curses.cc \ view_curses.cc \
vt52_curses.cc \ vt52_curses.cc \
@ -487,8 +497,10 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bin2c.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bin2c.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bookmarks.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bookmarks.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/collation-functions.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_parser.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_parser.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_scanner.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_scanner.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/extension-functions.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/grep_proc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/grep_proc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hist_source.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hist_source.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/line_buffer.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/line_buffer.Po@am__quote@
@ -499,10 +511,12 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log_vtab_impl.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log_vtab_impl.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile_sub_source.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile_sub_source.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/network-extension-functions.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/piper_proc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/piper_proc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readline_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readline_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sequence_matcher.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sequence_matcher.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statusview_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statusview_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/strnatcmp.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textview_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textview_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/view_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/view_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vt52_curses.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vt52_curses.Po@am__quote@

@ -38,19 +38,28 @@
#include <exception> #include <exception>
typedef void (*free_func_t)(void *);
/** /**
* Resource management class for memory allocated by a custom allocator. * Resource management class for memory allocated by a custom allocator.
* *
* @param T The object type. * @param T The object type.
* @param auto_free The function to call to free the managed object. * @param auto_free The function to call to free the managed object.
*/ */
template<class T, void (*auto_free)(void *) = free> template<class T, free_func_t default_free = free>
class auto_mem { class auto_mem {
public: public:
auto_mem(T *ptr = NULL) : am_ptr(ptr) { }; auto_mem(T *ptr = NULL) : am_ptr(ptr), am_free_func(default_free) { };
auto_mem(auto_mem &am)
: am_ptr(am.release()), am_free_func(am.am_free_func)
{
};
auto_mem(auto_mem &am) : am_ptr(am.release()) { }; template<typename F>
auto_mem(F free_func)
: am_ptr(NULL), am_free_func((void (*)(void *))free_func) { };
~auto_mem() { this->reset(); }; ~auto_mem() { this->reset(); };
@ -82,14 +91,15 @@ public:
void reset(T *ptr = NULL) { void reset(T *ptr = NULL) {
if (this->am_ptr != ptr) { if (this->am_ptr != ptr) {
auto_free(this->am_ptr); this->am_free_func(this->am_ptr);
this->am_ptr = ptr; this->am_ptr = ptr;
} }
}; };
private: private:
T *am_ptr; T *am_ptr;
void (*am_free_func)(void *);
}; };
#endif #endif

@ -32,6 +32,11 @@
#include <string> #include <string>
#include "grep_proc.hh"
#include "textview_curses.hh"
#include "logfile_sub_source.hh"
#include "status_controllers.hh"
class bottom_status_source class bottom_status_source
: public status_data_source, : public status_data_source,
public grep_proc_control public grep_proc_control
@ -52,6 +57,7 @@ public:
BSF_ERRORS, BSF_ERRORS,
BSF_FILTERED, BSF_FILTERED,
BSF_LOADING, BSF_LOADING,
BSF_HELP,
BSF__MAX BSF__MAX
} field_t; } field_t;
@ -64,17 +70,20 @@ public:
bss_error(80, view_colors::VCR_ALERT_STATUS), bss_error(80, view_colors::VCR_ALERT_STATUS),
bss_hit_spinner(0), bss_hit_spinner(0),
bss_load_percent(0) { bss_load_percent(0) {
this->bss_fields[BSF_LINE_NUMBER].set_width(8); this->bss_fields[BSF_LINE_NUMBER].set_width(10);
this->bss_fields[BSF_PERCENT].set_width(4); this->bss_fields[BSF_PERCENT].set_width(4);
this->bss_fields[BSF_HITS].set_width(16); this->bss_fields[BSF_HITS].set_width(16);
this->bss_fields[BSF_HITS].set_cylon(true); this->bss_fields[BSF_HITS].set_cylon(true);
this->bss_fields[BSF_WARNINGS].set_width(10); this->bss_fields[BSF_WARNINGS].set_width(10);
this->bss_fields[BSF_ERRORS].set_width(10); this->bss_fields[BSF_ERRORS].set_width(10);
this->bss_fields[BSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS); this->bss_fields[BSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS);
this->bss_fields[BSF_FILTERED].set_width(14); this->bss_fields[BSF_FILTERED].set_width(20);
this->bss_fields[BSF_LOADING].set_width(13); this->bss_fields[BSF_LOADING].set_width(13);
this->bss_fields[BSF_LOADING].set_cylon(true); this->bss_fields[BSF_LOADING].set_cylon(true);
this->bss_fields[BSF_LOADING].right_justify(true); this->bss_fields[BSF_LOADING].right_justify(true);
this->bss_fields[BSF_HELP].set_width(14);
this->bss_fields[BSF_HELP].set_value("?:View Help");
this->bss_fields[BSF_HELP].right_justify(true);
}; };
virtual ~bottom_status_source() { }; virtual ~bottom_status_source() { };
@ -111,10 +120,12 @@ public:
void update_line_number(listview_curses *lc) { void update_line_number(listview_curses *lc) {
status_field &sf = this->bss_fields[BSF_LINE_NUMBER]; status_field &sf = this->bss_fields[BSF_LINE_NUMBER];
if (lc->get_inner_height() == 0) if (lc->get_inner_height() == 0) {
sf.set_value("L0"); sf.set_value("L0");
else }
sf.set_value("L%d", (int)lc->get_top()); else {
sf.set_value("L%'d", (int)lc->get_top());
}
}; };
void update_percent(listview_curses *lc) { void update_percent(listview_curses *lc) {
@ -153,7 +164,7 @@ public:
bookmark_vector<vis_line_t>::iterator iter; bookmark_vector<vis_line_t>::iterator iter;
iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height); iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height);
sfw.set_value("%9dW", distance(iter, bv.end())); sfw.set_value("%'9dW", distance(iter, bv.end()));
} }
else { else {
sfw.clear(); sfw.clear();
@ -164,7 +175,7 @@ public:
bookmark_vector<vis_line_t>::iterator iter; bookmark_vector<vis_line_t>::iterator iter;
iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height); iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height);
sfe.set_value("%9dE", distance(iter, bv.end())); sfe.set_value("%'9dE", distance(iter, bv.end()));
} }
else { else {
sfe.clear(); sfe.clear();
@ -189,7 +200,7 @@ public:
} }
sf.set_role(new_role); sf.set_role(new_role);
this->bss_error.clear(); this->bss_error.clear();
sf.set_value("%9d hits", tc->get_match_count()); sf.set_value("%'9d hits", tc->get_match_count());
}; };
void update_loading(off_t off, size_t total) { void update_loading(off_t off, size_t total) {
@ -218,7 +229,7 @@ public:
if (lss.get_filtered_count() == 0) if (lss.get_filtered_count() == 0)
sf.clear(); sf.clear();
else else
sf.set_value("%d Not Shown", lss.get_filtered_count()); sf.set_value("%'9d Not Shown", lss.get_filtered_count());
}; };
private: private:

@ -43,7 +43,11 @@ struct byte_array {
bool operator<(const byte_array &other) const { bool operator<(const byte_array &other) const {
return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) < 0; return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) < 0;
}; };
bool operator!=(const byte_array &other) const {
return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) != 0;
};
unsigned char ba_data[BYTE_COUNT]; unsigned char ba_data[BYTE_COUNT];
}; };

@ -0,0 +1,161 @@
/**
* Copyright (c) 2013, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file logfile_sub_source.hh
*/
#include <stdio.h>
#include <string.h>
#include <sqlite3.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <algorithm>
extern "C" {
#include "strnatcmp.h"
}
#define MAX_ADDR_LEN 128
static int strncmp2(int a_len, const char *a_str,
int b_len, const char *b_str)
{
int retval = strncmp(a_str, b_str, std::min(a_len, b_len));
if (retval == 0) {
if (a_len < b_len) {
retval = -1;
}
else {
retval = 1;
}
}
return retval;
}
static int try_inet_pton(int p_len, const char *p, char *n)
{
static int family[] = { AF_INET6, AF_INET, AF_MAX };
char buf[MAX_ADDR_LEN];
int retval = AF_MAX;
strncpy(buf, p, p_len);
buf[p_len] = '\0';
for (int lpc = 0; family[lpc] != AF_MAX; lpc++) {
if (inet_pton(family[lpc], buf, n) == 1) {
retval = family[lpc];
break;
}
}
return retval;
}
static int convert_v6_to_v4(int family, char *n)
{
struct in6_addr *ia = (struct in6_addr *)n;
if (family == AF_INET6 &&
(IN6_IS_ADDR_V4COMPAT(ia) ||
IN6_IS_ADDR_V4MAPPED(ia))) {
family = AF_INET;
memmove(n, n + 12, sizeof(struct in_addr));
}
return family;
}
static
int ipaddress(void *ptr,
int a_len, const void *a_in,
int b_len, const void *b_in)
{
char a_addr[sizeof(struct in6_addr)], b_addr[sizeof(struct in6_addr)];
const char *a_str = (const char *)a_in, *b_str = (const char *)b_in;
int a_family, b_family, retval;
if (a_len > MAX_ADDR_LEN || b_len > MAX_ADDR_LEN) {
return strncmp2(a_len, a_str, b_len, b_str);
}
a_family = try_inet_pton(a_len, a_str, a_addr);
b_family = try_inet_pton(b_len, b_str, b_addr);
if (a_family == AF_MAX && b_family != AF_MAX) {
retval = -1;
}
else if (a_family != AF_MAX && b_family == AF_MAX) {
retval = 1;
}
else {
a_family = convert_v6_to_v4(a_family, a_addr);
b_family = convert_v6_to_v4(b_family, b_addr);
if (a_family == b_family) {
retval = memcmp(a_addr, b_addr,
a_family == AF_INET ?
sizeof(struct in_addr) :
sizeof(struct in6_addr));
}
else if (a_family == AF_INET) {
retval = -1;
}
else {
retval = 1;
}
}
return retval;
}
static
int sql_strnatcmp(void *ptr,
int a_len, const void *a_in,
int b_len, const void *b_in)
{
return strnatcmp(a_len, (char *)a_in, b_len, (char *)b_in);
}
static
int sql_strnatcasecmp(void *ptr,
int a_len, const void *a_in,
int b_len, const void *b_in)
{
return strnatcasecmp(a_len, (char *)a_in, b_len, (char *)b_in);
}
int register_collation_functions(sqlite3 *db)
{
sqlite3_create_collation(db, "ipaddress", SQLITE_UTF8, NULL, ipaddress);
sqlite3_create_collation(db, "natural", SQLITE_UTF8, NULL, sql_strnatcmp);
sqlite3_create_collation(db, "naturalnocase", SQLITE_UTF8, NULL, sql_strnatcasecmp);
return 0;
}

@ -0,0 +1,86 @@
/**
* Copyright (c) 2013, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file column_namer.hh
*/
#include <map>
#include <string>
#include <vector>
#include <algorithm>
class column_namer {
public:
column_namer() {
this->cn_builtin_names.push_back("col");
};
bool existing_name(const std::string &in_name) {
if (find(this->cn_builtin_names.begin(),
this->cn_builtin_names.end(),
in_name) != this->cn_builtin_names.end()) {
return true;
}
else if (find(this->cn_names.begin(),
this->cn_names.end(),
in_name) != this->cn_names.end()) {
return true;
}
return false;
};
std::string add_column(const std::string &in_name) {
std::string base_name = in_name, retval;
size_t buf_size;
char *buffer;
int num = 0;
buf_size = in_name.length() + 64;
buffer = (char *)alloca(buf_size);
if (in_name == "") {
base_name = "col";
}
retval = base_name;
while (this->existing_name(retval)) {
snprintf(buffer, buf_size, "%s_%d", base_name.c_str(), num);
retval = buffer;
num += 1;
}
this->cn_names.push_back(retval);
return retval;
};
std::vector<std::string> cn_builtin_names;
std::vector<std::string> cn_names;
};

@ -33,259 +33,88 @@
using namespace std; using namespace std;
static data_token_t PATTERN_KEY[] = { data_format data_parser::FORMAT_SEMI(DT_COMMA, DT_SEMI);
DT_STRING, data_format data_parser::FORMAT_COMMA(DT_INVALID, DT_COMMA);
DT_NUMBER, data_format data_parser::FORMAT_PLAIN(DT_INVALID, DT_INVALID);
DT_HEX_NUMBER,
// DT_QUALIFIED_NAME,
};
static data_token_t UPTO_SEPARATOR[] = { data_format_state_t dfs_semi_next(data_format_state_t state,
DT_SEPARATOR, data_token_t next_token)
DT_LINE,
};
static data_token_t UPTO_NT[] = {
DNT_PAIR,
DNT_ROW,
DT_SEPARATOR,
};
static data_token_t PATTERN_PAIR[] = {
DNT_ROW,
DT_SEPARATOR,
// DNT_KEY,
};
static data_token_t PATTERN_ROW[] = {
DT_ANY,
DT_COMMA,
DNT_ROW,
};
static data_token_t PATTERN_DATE_TIME[] = {
DT_TIME,
DT_NUMBER,
DT_STRING,
};
static data_token_t PATTERN_QUAL[] = {
DNT_KEY,
DT_SEPARATOR,
DNT_KEY,
};
bool data_parser::reducePattern(std::list<element> &reduction,
const data_token_t *pattern_start,
const data_token_t *pattern_end,
bool repeating)
{
size_t pattern_size = (pattern_end - pattern_start);
bool found, retval = false;
reduction.clear();
do {
found = false;
if (pattern_size <= this->dp_stack.size() &&
std::equal(pattern_start, pattern_end,
this->dp_stack.begin(),
element_cmp())) {
std::list<element>::iterator match_end = this->dp_stack.begin();
advance(match_end, pattern_size);
reduction.splice(reduction.end(),
this->dp_stack,
this->dp_stack.begin(),
match_end);
retval = found = true;
}
} while (found && repeating);
reduction.reverse();
return retval;
}
void data_parser::reduceQual(const struct element &lookahead)
{
std::list<element> reduction;
if (this->reducePattern(reduction,
PATTERN_QUAL,
PATTERN_QUAL +
sizeof(PATTERN_QUAL) / sizeof(data_token_t))) {
// printf("qual hit\n");
this->dp_qual.splice(this->dp_qual.end(), reduction);
}
}
void data_parser::reduceRow(void)
{
std::list<element> reduction;
if (this->reducePattern(reduction,
PATTERN_ROW,
PATTERN_ROW +
sizeof(PATTERN_ROW) / sizeof(data_token_t))) {
std::list<element>::iterator match_end;
if (reduction.back().e_sub_elements != NULL)
reduction.front().assign_elements(*reduction.back().e_sub_elements);
else
reduction.front().e_sub_elements->push_back(reduction.back());
reduction.front().update_capture();
match_end = reduction.begin();
++match_end;
this->dp_stack.splice(this->dp_stack.begin(),
reduction,
reduction.begin(),
match_end);
}
}
void data_parser::reducePair(void)
{ {
std::list<element> reduction; data_format_state_t retval = state;
if (this->reducePattern(reduction, switch (state) {
PATTERN_DATE_TIME, case DFS_INIT:
PATTERN_DATE_TIME + switch (next_token) {
sizeof(PATTERN_DATE_TIME) / sizeof(data_token_t))) { case DT_COMMA:
this->dp_stack.push_front(element(reduction, DNT_DATE_TIME)); case DT_SEMI:
this->dp_stack.front().assign_elements(reduction); retval = DFS_ERROR;
} break;
default: retval = DFS_KEY; break;
this->reduceRow(); }
if (this->reduceUpTo(reduction, break;
UPTO_SEPARATOR, case DFS_KEY:
UPTO_SEPARATOR + switch (next_token) {
sizeof(UPTO_SEPARATOR) / sizeof(data_token_t)) && case DT_SEPARATOR: retval = DFS_VALUE; break;
!reduction.empty()) { case DT_SEMI: retval = DFS_ERROR; break;
if (reduction.front().e_token == DNT_ROW) { default: break;
reduction.reverse(); }
this->dp_stack.splice(this->dp_stack.begin(), reduction); break;
} case DFS_VALUE:
else { switch (next_token) {
this->dp_stack.push_front(element(reduction, DNT_ROW)); case DT_SEMI: retval = DFS_INIT; break;
this->dp_stack.front().assign_elements(reduction); default: break;
}
break;
case DFS_ERROR: retval = DFS_ERROR; break;
} }
}
if (this->reducePattern(reduction, return retval;
PATTERN_PAIR,
PATTERN_PAIR +
sizeof(PATTERN_PAIR) / sizeof(data_token_t))) {
if (this->dp_qual.empty()) {
this->dp_stack.splice(this->dp_stack.begin(), reduction);
}
else {
reduction.push_front(this->dp_qual.back());
this->dp_qual.pop_back();
this->dp_stack.push_front(element(reduction, DNT_PAIR));
this->dp_stack.front().assign_elements(reduction);
}
}
// this->print(stdout);
} }
#define DEB 0 data_format_state_t dfs_comma_next(data_format_state_t state,
data_token_t next_token)
void data_parser::reduce(const element &lookahead)
{ {
std::list<element> reduction; data_format_state_t retval = state;
bool push_lookahead = true;
switch (state) {
switch (lookahead.e_token) { case DFS_INIT:
case DT_INVALID: switch (next_token) {
case DT_WHITE: case DT_COMMA:
this->reducePair(); case DT_SEMI:
push_lookahead = false; retval = DFS_ERROR;
break; break;
default:
case DT_GARBAGE: retval = DFS_KEY;
push_lookahead = false; break;
break; }
break;
case DT_LINE: case DFS_KEY:
this->reduceRow(); switch (next_token) {
if (!this->reduceUpTo(reduction, case DT_SEPARATOR:
UPTO_NT, retval = DFS_VALUE;
UPTO_NT + break;
sizeof(UPTO_NT) / sizeof(data_token_t))) { case DT_SEMI:
reduction.splice(reduction.begin(), this->dp_stack); case DT_COMMA:
reduction.reverse(); retval = DFS_ERROR;
} break;
if (!reduction.empty()) { default: break;
if (this->dp_stack.front().e_token == DNT_ROW) { }
this->dp_stack.front().assign_elements(reduction); break;
} case DFS_VALUE:
else if (this->dp_stack.front().e_token == DNT_PAIR) { switch (next_token) {
this->dp_stack.front().e_sub_elements->back().assign_elements(reduction); case DT_COMMA:
} retval = DFS_INIT;
else { break;
this->dp_stack.push_front(element(reduction, DNT_ROW)); case DT_SEPARATOR:
this->dp_stack.front().assign_elements(reduction); retval = DFS_VALUE;
} break;
default: break;
}
break;
case DFS_ERROR:
retval = DFS_ERROR;
break;
} }
this->reducePair(); return retval;
push_lookahead = false;
break;
case DT_COMMA:
this->reduceRow();
if (!this->dp_stack.empty() &&
this->dp_stack.front().e_token != DNT_ROW) {
if (this->dp_stack.front().e_token == DT_SEPARATOR) {
push_lookahead = false;
}
else if (this->dp_stack.front().e_token == DNT_PAIR) {
std::list<element>::iterator pair_iter = this->dp_stack.begin();
this->dp_qual.push_front(this->dp_stack.front().e_sub_elements->front());
this->dp_stack.front().e_sub_elements->pop_front();
this->dp_stack.front().e_sub_elements->reverse();
this->dp_stack.splice(this->dp_stack.begin(),
*this->dp_stack.front().e_sub_elements);
this->dp_stack.erase(pair_iter);
}
else {
std::list<element>::iterator next_elem = this->dp_stack.begin();
advance(next_elem, 1);
reduction.splice(reduction.end(),
this->dp_stack,
this->dp_stack.begin(),
next_elem);
this->dp_stack.push_front(element(reduction, DNT_ROW));
this->dp_stack.front().assign_elements(reduction);
}
}
break;
case DT_SEPARATOR:
if (this->reduceAnyOf(reduction,
PATTERN_KEY,
PATTERN_KEY +
sizeof(PATTERN_KEY) / sizeof(data_token_t))) {
this->reducePair();
if (this->dp_stack.front().e_token == DT_SEPARATOR)
this->dp_stack.pop_front();
this->dp_qual.push_back(element(reduction, DNT_KEY));
// this->reduceQual(lookahead);
}
break;
default:
break;
}
if (push_lookahead) {
this->dp_stack.push_front(lookahead);
}
// this->print(stdout);
} }

@ -32,49 +32,152 @@
#include <stdio.h> #include <stdio.h>
#include <openssl/sha.h>
#include <list> #include <list>
#include <vector>
#include <iterator>
#include <algorithm> #include <algorithm>
#include "pcrepp.hh" #include "pcrepp.hh"
#include "byte_array.hh"
#include "data_scanner.hh" #include "data_scanner.hh"
template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate> /**
ForwardIterator1 find_first_not_of(ForwardIterator1 first, ForwardIterator1 last, * Switch to the 'parser' view mode when the user hits ';' so they
ForwardIterator2 s_first, ForwardIterator2 s_last, * can easily see what columns are available.
BinaryPredicate p) *
* select * from logline;
* select itemfrom(csv_key, 0) from logline;
* select itemfrom(csv_key, -1) from logline;
* select itemfrom(dict_key, "key") from logline;
* select itemfrom(dict_key, "key[0]") from logline;
* select itemfrom(csv_key, 0:3) from logline; support splices ?
*
* Add a command to create a logline table with a given name so the user can
* do joins across the tables:
*
* create-logline-table sudo_logline
* select * from logline, sudo_logline where sudo_logline.COMMAND=logline.COMMAND;
*
* select timestmap / 60 as minute, sc_status, count(*) from access_log
* group by minute, sc_status
* order by minute, sc_status desc;
* (use group_concat() here?)
*
* The "itemfrom()" function parses the group and lets you specify an
* expression to query the contents.
*
* For 'report-on' command:
* 'report-on PWD'
* select PWD,count(*) as amount from logline group by PWD order by amount desc;
* 'report-on num_col num_col2'
* select avg(num_col),stddev(num_col),... from logline;
* Instead of a command, we should automatically create views with the
* relevant select statements.
*
* Add a tojson() aggregate function to sqlite:
* select foo,tojson(bar) group by foo;
*
* 1 ["a", "b", "c"]
* 2 ["d", "e", "f"]
*
* We should automatically detect sqlite files provided on the command line
* and attach the database.
*
* add a 'metadata' view that has all the metadata crud (sql tables/log)
*
* Add support for sqlite_log that writes to a temp file and is displayed in the
* metadata view.
*
* Add a function that bookmarks all lines in the log view based on line_numbers
* in the sql query result.
*
* select line_number from logline where A="b";
* hit 'y/Y' to move forward and backwards through sql results
* hit 'R' key and all the lines are bookmarked
*
* add path manipulation functions like basename, dirname, splitext
*
* use the vt52_curses emulation to embed a editor for editing queries. For
* example, you could hit 'ctrl+;' and it would split the window in half with
* the bottom being used for nano. When the file was written, lnav should
* notice and do a 'prepare' on the sql to make sure it is correct.
*
* Maybe add other tables for accessing lnav state. For example, you could do
* a query to find log lines of interest and then insert their line numbers
* into the 'bookmarks' table to create new user bookmarks.
*/
template<class Container, class UnaryPredicate>
void strip(Container &container, UnaryPredicate p)
{ {
for (; first != last; ++first) { while (!container.empty() && p(container.front())) {
bool found = false; container.pop_front();
}
for (ForwardIterator2 it = s_first; it != s_last; ++it) { while (!container.empty() && p(container.back())) {
if (p(*first, *it)) { container.pop_back();
found = true; }
}
}
if (!found)
return first;
}
return last;
} }
enum data_format_state_t {
DFS_ERROR = -1,
DFS_INIT,
DFS_KEY,
DFS_VALUE,
};
struct data_format {
data_format(data_token_t appender = DT_INVALID,
data_token_t terminator = DT_INVALID)
: df_appender(appender), df_terminator(terminator)
{
};
const data_token_t df_appender;
const data_token_t df_terminator;
};
data_format_state_t dfs_semi_next(data_format_state_t state,
data_token_t next_token);
data_format_state_t dfs_comma_next(data_format_state_t state,
data_token_t next_token);
class data_parser { class data_parser {
public: public:
static data_format FORMAT_SEMI;
static data_format FORMAT_COMMA;
static data_format FORMAT_PLAIN;
typedef byte_array<20> schema_id_t;
struct element;
typedef std::list<element> element_list_t;
struct element { struct element {
element() : e_token(DT_INVALID), e_sub_elements(NULL) { }; element() : e_token(DT_INVALID), e_sub_elements(NULL) { };
element(std::list<element> &subs, data_token_t token) element(element_list_t &subs,
data_token_t token,
bool assign_subs_elements = true)
: e_capture(subs.front().e_capture.c_begin, : e_capture(subs.front().e_capture.c_begin,
subs.back().e_capture.c_end), subs.back().e_capture.c_end),
e_token(token), e_token(token),
e_sub_elements(NULL) { e_sub_elements(NULL) {
if (assign_subs_elements) {
this->assign_elements(subs);
}
}; };
element(const element &other) { element(const element &other) {
assert(other.e_sub_elements == NULL); // assert(other.e_sub_elements == NULL);
this->e_capture = other.e_capture; this->e_capture = other.e_capture;
this->e_token = other.e_token; this->e_token = other.e_token;
this->e_sub_elements = NULL; this->e_sub_elements = NULL;
if (other.e_sub_elements != NULL)
this->assign_elements(*other.e_sub_elements);
}; };
~element() { ~element() {
@ -84,10 +187,19 @@ public:
} }
}; };
void assign_elements(std::list<element> &subs) { element& operator=(const element &other) {
this->e_capture = other.e_capture;
this->e_token = other.e_token;
this->e_sub_elements = NULL;
if (other.e_sub_elements != NULL)
this->assign_elements(*other.e_sub_elements);
return *this;
};
void assign_elements(element_list_t &subs) {
if (this->e_sub_elements == NULL) if (this->e_sub_elements == NULL)
this->e_sub_elements = new std::list<element>(); this->e_sub_elements = new element_list_t();
this->e_sub_elements->splice(this->e_sub_elements->end(), subs); this->e_sub_elements->swap(subs);
this->update_capture(); this->update_capture();
}; };
@ -100,9 +212,22 @@ public:
} }
}; };
data_token_t value_token(void) const {
data_token_t retval = DT_INVALID;
if (this->e_token == DNT_VALUE &&
this->e_sub_elements != NULL &&
this->e_sub_elements->size() == 1) {
retval = this->e_sub_elements->front().e_token;
}
return retval;
};
void print(FILE *out, pcre_input &pi, int offset = 0) { void print(FILE *out, pcre_input &pi, int offset = 0) {
int lpc;
if (this->e_sub_elements != NULL) { if (this->e_sub_elements != NULL) {
for (std::list<data_parser::element>::iterator iter2 = for (element_list_t::iterator iter2 =
this->e_sub_elements->begin(); this->e_sub_elements->begin();
iter2 != this->e_sub_elements->end(); iter2 != this->e_sub_elements->end();
++iter2) { ++iter2) {
@ -114,7 +239,7 @@ public:
data_scanner::token2name(this->e_token), data_scanner::token2name(this->e_token),
this->e_capture.c_begin, this->e_capture.c_begin,
this->e_capture.c_end); this->e_capture.c_end);
for (int lpc = 0; lpc < this->e_capture.c_end; lpc++) { for (lpc = 0; lpc < this->e_capture.c_end; lpc++) {
if (lpc == this->e_capture.c_begin) if (lpc == this->e_capture.c_begin)
fputc('^', out); fputc('^', out);
else if (lpc == (this->e_capture.c_end - 1)) else if (lpc == (this->e_capture.c_end - 1))
@ -124,13 +249,18 @@ public:
else else
fputc(' ', out); fputc(' ', out);
} }
fputc('\n', out); for (; lpc < (int)pi.pi_length; lpc++) {
fputc(' ', out);
}
std::string sub = pi.get_substr(&this->e_capture);
fprintf(out, " %s\n", sub.c_str());
}; };
pcre_context::capture_t e_capture; pcre_context::capture_t e_capture;
data_token_t e_token; data_token_t e_token;
std::list<element> *e_sub_elements; element_list_t *e_sub_elements;
}; };
struct element_cmp { struct element_cmp {
@ -154,94 +284,343 @@ public:
data_token_t ei_token; data_token_t ei_token;
}; };
data_parser(data_scanner *ds) : dp_scanner(ds) { }; data_parser(data_scanner *ds) : dp_format(NULL), dp_scanner(ds) { };
void parse(void) { void pairup(schema_id_t *schema, element_list_t &pairs_out, element_list_t &in_list) {
pcre_context_static<30> pc; element_list_t el_stack, free_row, key_comps, value, prefix;
struct element elem; SHA_CTX context;
while (this->dp_scanner->tokenize(pc, elem.e_token)) { for (element_list_t::iterator iter = in_list.begin();
elem.e_capture = *(pc.begin()); iter != in_list.end();
++iter) {
if (iter->e_token == DNT_GROUP) {
element_list_t group_pairs;
this->reduce(elem); this->pairup(NULL, group_pairs, *iter->e_sub_elements);
if (!group_pairs.empty()) {
iter->assign_elements(group_pairs);
}
}
if (iter->e_token == DT_SEPARATOR) {
element_list_t::iterator key_iter = key_comps.end();
bool found = false;
--key_iter;
for (;
key_iter != key_comps.begin() && !found;
--key_iter) {
if (key_iter->e_token == this->dp_format->df_appender) {
++key_iter;
value.splice(value.end(),
key_comps,
key_comps.begin(),
key_iter);
key_comps.splice(key_comps.begin(),
key_comps,
key_comps.end());
key_comps.resize(1);
found = true;
}
else if (key_iter->e_token == this->dp_format->df_terminator) {
std::vector<element> key_copy;
value.splice(value.end(),
key_comps,
key_comps.begin(),
key_iter);
++key_iter;
key_comps.pop_front();
strip(key_comps, element_if(DT_WHITE));
found = true;
}
}
if (!found && !el_stack.empty() && !key_comps.empty()) {
element_list_t::iterator value_iter;
value.splice(value.end(),
key_comps,
key_comps.begin(),
key_comps.end());
value_iter = value.end();
std::advance(value_iter, -1);
key_comps.splice(key_comps.begin(),
value,
value_iter);
key_comps.resize(1);
}
strip(value, element_if(DT_WHITE));
value.remove_if(element_if(DT_COMMA));
if (!value.empty()) {
el_stack.push_back(element(value, DNT_VALUE));
}
strip(key_comps, element_if(DT_WHITE));
if (!key_comps.empty()) {
el_stack.push_back(element(key_comps, DNT_KEY, false));
}
key_comps.clear();
value.clear();
}
else {
key_comps.push_back(*iter);
}
} }
};
void reduce(const element &elem); if (el_stack.empty()) {
free_row.splice(free_row.begin(),
key_comps, key_comps.begin(), key_comps.end());
}
else {
value.splice(value.begin(),
key_comps,
key_comps.begin(),
key_comps.end());
strip(value, element_if(DT_WHITE));
value.remove_if(element_if(DT_COMMA));
if (!value.empty()) {
el_stack.push_back(element(value, DNT_VALUE));
}
}
SHA_Init(&context);
while (!el_stack.empty()) {
element_list_t::iterator kv_iter = el_stack.begin();
if (kv_iter->e_token == DNT_VALUE) {
free_row.push_back(el_stack.front());
}
if (kv_iter->e_token != DNT_KEY) {
el_stack.pop_front();
continue;
}
++kv_iter;
if (kv_iter == el_stack.end()) {
el_stack.pop_front();
continue;
}
bool reducePattern(std::list<element> &reduction, if (kv_iter->e_token != DNT_VALUE) {
const data_token_t *pattern_start, el_stack.pop_front();
const data_token_t *pattern_end, continue;
bool repeating = false); }
bool reduceAnyOf(std::list<element> &reduction, std::string key_val = this->get_element_string(el_stack.front());
const data_token_t *possibilities_start, element_list_t pair_subs;
const data_token_t *possibilities_end) {
std::list<element>::iterator iter;
bool retval = false;
reduction.clear(); if (schema != NULL) {
SHA_Update(&context, key_val.c_str(), key_val.length());
}
iter = find_first_not_of(this->dp_stack.begin(), ++kv_iter;
this->dp_stack.end(), pair_subs.splice(pair_subs.begin(),
possibilities_start, possibilities_end, el_stack,
element_cmp()); el_stack.begin(),
if (iter != this->dp_stack.begin()) { kv_iter);
reduction.splice(reduction.end(), pairs_out.push_back(element(pair_subs, DNT_PAIR));
this->dp_stack, }
this->dp_stack.begin(),
iter);
retval = true; if (pairs_out.size() == 1) {
element &pair = pairs_out.front();
element &value = pair.e_sub_elements->back();
if (value.e_token == DNT_VALUE &&
value.e_sub_elements != NULL &&
value.e_sub_elements->size() > 1) {
prefix.splice(prefix.begin(),
*pair.e_sub_elements,
pair.e_sub_elements->begin());
free_row.clear();
free_row.splice(free_row.begin(),
*value.e_sub_elements,
value.e_sub_elements->begin(),
value.e_sub_elements->end());
pairs_out.clear();
SHA_Init(&context);
}
} }
reduction.reverse(); if (pairs_out.empty() && !free_row.empty()) {
while (!free_row.empty()) {
return retval; switch (free_row.front().e_token) {
case DNT_GROUP:
case DT_NUMBER:
case DT_SYMBOL:
case DT_HEX_NUMBER:
case DT_OCTAL_NUMBER:
case DT_VERSION_NUMBER:
case DT_QUOTED_STRING:
case DT_IPV4_ADDRESS:
case DT_IPV6_ADDRESS:
case DT_MAC_ADDRESS:
case DT_UUID:
case DT_URL:
case DT_PATH:
case DT_TIME:
case DT_PERCENTAGE: {
element_list_t pair_subs;
struct element blank;
blank.e_capture.c_begin = blank.e_capture.c_end = free_row.front().e_capture.c_begin;
blank.e_token = DNT_KEY;
pair_subs.push_back(blank);
pair_subs.push_back(free_row.front());
pairs_out.push_back(element(pair_subs, DNT_PAIR));
}
break;
default: {
std::string key_val = this->get_element_string(free_row.front());
SHA_Update(&context, key_val.c_str(), key_val.length());
}
break;
}
free_row.pop_front();
}
}
if (!prefix.empty()) {
element_list_t pair_subs;
struct element blank;
blank.e_capture.c_begin = blank.e_capture.c_end = prefix.front().e_capture.c_begin;
blank.e_token = DNT_KEY;
pair_subs.push_back(blank);
pair_subs.push_back(prefix.front());
pairs_out.push_front(element(pair_subs, DNT_PAIR));
}
if (schema != NULL)
SHA_Final(this->dp_schema_id.ba_data, &context);
}; };
void discover_format(void) {
pcre_context_static<30> pc;
int hist[DT_TERMINAL_MAX];
struct element elem;
bool reduceUpTo(std::list<element> &reduction, this->dp_group_token.push_back(DT_INVALID);
const data_token_t *possibilities_start, this->dp_group_stack.resize(1);
const data_token_t *possibilities_end) {
std::list<element>::iterator iter;
bool retval = false;
reduction.clear(); data_format_state_t semi_state = DFS_INIT;
data_format_state_t comma_state = DFS_INIT;
iter = std::find_first_of(this->dp_stack.begin(), this->dp_stack.end(), memset(hist, 0, sizeof(hist));
possibilities_start, possibilities_end, while (this->dp_scanner->tokenize(pc, elem.e_token)) {
element_cmp()); pcre_context::iterator pc_iter;
if (iter != this->dp_stack.end()) {
reduction.splice(reduction.end(), pc_iter = std::find_if(pc.begin(), pc.end(), capture_if_not(-1));
this->dp_stack, assert(pc_iter != pc.end());
this->dp_stack.begin(),
iter); elem.e_capture = *pc_iter;
assert(elem.e_capture.c_begin != -1);
assert(elem.e_capture.c_end != -1);
semi_state = dfs_semi_next(semi_state, elem.e_token);
comma_state = dfs_comma_next(comma_state, elem.e_token);
hist[elem.e_token] += 1;
switch (elem.e_token) {
case DT_LPAREN:
case DT_LANGLE:
case DT_LCURLY:
case DT_LSQUARE:
this->dp_group_token.push_back(elem.e_token);
this->dp_group_stack.push_back(element_list_t());
break;
case DT_RPAREN:
case DT_RANGLE:
case DT_RCURLY:
case DT_RSQUARE:
if (this->dp_group_token.back() == (elem.e_token - 1)) {
this->dp_group_token.pop_back();
std::list<element_list_t>::reverse_iterator riter = this->dp_group_stack.rbegin();
++riter;
if (!this->dp_group_stack.back().empty()) {
(*riter).push_back(element(this->dp_group_stack.back(), DNT_GROUP));
}
this->dp_group_stack.pop_back();
}
else {
this->dp_group_stack.back().push_back(elem);
}
break;
default:
this->dp_group_stack.back().push_back(elem);
break;
}
}
while (this->dp_group_stack.size() > 1) {
this->dp_group_token.pop_back();
retval = true; std::list<element_list_t>::reverse_iterator riter = this->dp_group_stack.rbegin();
++riter;
if (!this->dp_group_stack.back().empty()) {
(*riter).push_back(element(this->dp_group_stack.back(), DNT_GROUP));
}
this->dp_group_stack.pop_back();
} }
reduction.reverse(); if (semi_state != DFS_ERROR && hist[DT_SEMI]) {
this->dp_format = &FORMAT_SEMI;
}
else if (comma_state != DFS_ERROR) {
this->dp_format = &FORMAT_COMMA;
}
else {
this->dp_format = &FORMAT_PLAIN;
}
};
void parse(void) {
return retval; this->discover_format();
this->pairup(&this->dp_schema_id,
this->dp_pairs,
this->dp_group_stack.front());
for (element_list_t::iterator iter = this->dp_pairs.begin();
iter != this->dp_pairs.end();
++iter) {
if (iter->e_token == DNT_PAIR) {
element_list_t &pair_subs = *iter->e_sub_elements;
std::string key_val = this->get_element_string(pair_subs.front());
}
}
}; };
void reduceQual(const struct element &lookahead); std::string get_element_string(element &elem) {
void reduceRow(void); pcre_input &pi = this->dp_scanner->get_input();
void reducePair(void);
return pi.get_substr(&elem.e_capture);
};
void print(FILE *out) { void print(FILE *out, element_list_t &el) {
fprintf(out, " %s\n", fprintf(out, " %s\n",
this->dp_scanner->get_input().get_string()); this->dp_scanner->get_input().get_string());
for (std::list<data_parser::element>::iterator iter = this->dp_stack.begin(); for (element_list_t::iterator iter = el.begin();
iter != this->dp_stack.end(); iter != el.end();
++iter) { ++iter) {
iter->print(out, this->dp_scanner->get_input()); iter->print(out, this->dp_scanner->get_input());
} }
}; };
std::list<element> dp_qual; std::vector<data_token_t> dp_group_token;
std::list<element> dp_stack; std::list<element_list_t> dp_group_stack;
element_list_t dp_errors;
element_list_t dp_pairs;
schema_id_t dp_schema_id;
data_format *dp_format;
private: private:
data_scanner *dp_scanner; data_scanner *dp_scanner;

@ -29,6 +29,8 @@
#include "config.h" #include "config.h"
#include <arpa/inet.h>
#include "pcrepp.hh" #include "pcrepp.hh"
#include "data_scanner.hh" #include "data_scanner.hh"
@ -38,33 +40,50 @@ static struct {
const char *name; const char *name;
pcrepp pcre; pcrepp pcre;
} MATCHERS[DT_TERMINAL_MAX] = { } MATCHERS[DT_TERMINAL_MAX] = {
{ "url", pcrepp("([\\w]+://[^\\s'\"\\[\\](){}]+)"), }, { "quot", pcrepp("(?:u|r)?\"((?:\\\\.|[^\"])+)\"|"
{ "path", pcrepp("(?<![\\w\\d-_])((?:/|\\./|\\.\\./)[\\w\\d\\.-_\\~/]+)"), }, "(?:u|r)?'((?:\\\\.|[^'])+)'"), },
{ "url", pcrepp("([\\w]+://[^\\s'\"\\[\\](){}]+[a-zA-Z0-9\\-=&])"), },
{ "path", pcrepp("(?<![\\w\\d-_])((?:/|\\./|\\.\\./)[\\w\\.\\-_\\~/]+)"), },
{ "mac", pcrepp("([0-9a-fA-F][0-9a-fA-F](?::[0-9a-fA-F][0-9a-fA-F]){5})"), },
{ "time", pcrepp("\\b(\\d?\\d:\\d\\d(:\\d\\d)?(:\\d\\d)?([,.]\\d{3})?)\\b"), }, // XXX be more specific { "time", pcrepp("\\b(\\d?\\d:\\d\\d(:\\d\\d)?(:\\d\\d)?([,.]\\d{3})?)\\b"), }, // XXX be more specific
{ "mac", pcrepp("([0-9a-fA-F][0-9a-fA-F](?::[0-9a-fA-F][0-9a-fA-F]){5,5})"), },
{ "quot", pcrepp("u?\"([^\"]+)\"|u'([^']+(?:'\\w[^']*)*)'"), },
// { "qual", pcrepp("([^\\s:=]+:[^\\s:=,]+(?!,)(?::[^\\s:=,]+)*)"), }, // { "qual", pcrepp("([^\\s:=]+:[^\\s:=,]+(?!,)(?::[^\\s:=,]+)*)"), },
{ "ipv6", pcrepp("(::|[:\\da-fA-f\\.]+[a-fA-f\\d])"), },
{ "sep", pcrepp("(:|=)"), }, { "sep", pcrepp("(:|=)"), },
{ "comm", pcrepp("(,|/)"), }, { "comm", pcrepp("(,|/)"), },
{ "semi", pcrepp("(;)"), },
{ "lcurly", pcrepp("({)"), },
{ "rcurly", pcrepp("(})"), },
{ "lsquare", pcrepp("(\\[)"), },
{ "rsquare", pcrepp("(\\])"), },
{ "lparen", pcrepp("(\\()"), },
{ "rparen", pcrepp("(\\))"), },
{ "langle", pcrepp("(\\<)"), },
{ "rangle", pcrepp("(\\>)"), },
{ "ipv4", pcrepp("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})"), }, { "ipv4", pcrepp("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})"), },
{ "uuid", pcrepp("([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})"), },
{ "vers", pcrepp("([0-9]+(?:\\.[0-9]+){2,}\\b)"), }, { "vers", pcrepp("([0-9]+(?:\\.[0-9]+){2,}\\b)"), },
{ "oct", pcrepp("(-?0[0-7]+\\b)"), }, { "oct", pcrepp("(-?0[0-7]+\\b)"), },
{ "pcnt", pcrepp("(-?[0-9]+(\\.[0-9]+)?[ ]*%\\b)"), }, { "pcnt", pcrepp("(-?[0-9]+(\\.[0-9]+)?[ ]*%\\b)"), },
{ "num", pcrepp("(-?[0-9]+(\\.[0-9]+)?([eE][-+][0-9]+)?\\b)"), }, { "num", pcrepp("(-?[0-9]+(\\.[0-9]+)?([eE][-+][0-9]+)?\\b)"), },
{ "hex", pcrepp("(-?(?:0x|[0-9])[0-9a-fA-F]+\\b)"), }, { "hex", pcrepp("(-?(?:0x|[0-9])[0-9a-fA-F]+\\b)"), },
{ "word", pcrepp("([^\"';\\s:=,/(){}\\[\\]]+)"), }, { "word", pcrepp("([a-zA-Z][a-z']+(?=[\\s\\(\\)!\\*:;'\\\"\\?,]|\\.\\s|$))"), },
{ "sym", pcrepp("([^\";\\s:=,/(){}\\[\\]]+)"), },
{ "line", pcrepp("(\r?\n|\r|;)"), }, { "line", pcrepp("(\r?\n|\r|;)"), },
{ "whit", pcrepp("([ \r\t]+)"), }, { "wspc", pcrepp("([ \r\t]+)"), },
{ "dot", pcrepp("(\\.)"), }, { "dot", pcrepp("(\\.)"), },
{ "gbg", pcrepp("(.)"), }, { "gbg", pcrepp("(.)"), },
}; };
const char *DNT_NAMES[] = { const char *DNT_NAMES[DNT_MAX - DNT_KEY] = {
"key", "key",
"pair", "pair",
"val", "val",
@ -74,6 +93,7 @@ const char *DNT_NAMES[] = {
"var", "var",
"rang", "rang",
"date", "date",
"grp",
}; };
const char *data_scanner::token2name(data_token_t token) const char *data_scanner::token2name(data_token_t token)
@ -88,6 +108,24 @@ const char *data_scanner::token2name(data_token_t token)
return DNT_NAMES[token - DNT_KEY]; return DNT_NAMES[token - DNT_KEY];
} }
static
bool find_string_end(const char *str, size_t &start, size_t length, char term)
{
for (; start < length; start++) {
if (str[start] == term) {
start += 1;
return true;
}
if (str[start] == '\\') {
if (start + 1 >= length) {
return false;
}
start += 1;
}
}
return false;
}
bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out) bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out)
{ {
int lpc; int lpc;
@ -102,14 +140,74 @@ bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out)
this->ds_pcre_input.pi_next_offset += 1; this->ds_pcre_input.pi_next_offset += 1;
token_out = DT_LINE; token_out = DT_LINE;
return true; return false;
} }
for (lpc = 0; lpc < DT_TERMINAL_MAX; lpc++) { for (lpc = 0; lpc < DT_TERMINAL_MAX; lpc++) {
if (MATCHERS[lpc].pcre.match(pc, this->ds_pcre_input, PCRE_ANCHORED)) { switch (lpc) {
token_out = data_token_t(lpc); case DT_QUOTED_STRING: {
break; pcre_input &pi = this->ds_pcre_input;
} const char *str = pi.get_string();
size_t str_start, str_end;
bool found = false;
pi.pi_offset = pi.pi_next_offset;
str_end = str_start = pi.pi_offset + 1;
switch (str[pi.pi_offset]) {
case 'u':
case 'r':
if (pi.pi_offset + 1 < pi.pi_length &&
(str[pi.pi_offset + 1] == '\'' ||
str[pi.pi_offset + 1] == '\"')) {
str_start += 1;
str_end += 1;
found = find_string_end(str,
str_end,
pi.pi_length,
str[pi.pi_offset]);
}
break;
case '\'':
case '\"':
found = find_string_end(str,
str_end,
pi.pi_length,
str[pi.pi_offset]);
break;
}
if (found) {
token_out = data_token_t(DT_QUOTED_STRING);
pi.pi_next_offset = str_end;
pc.all()[0].c_begin = pi.pi_offset;
pc.all()[0].c_end = str_end;
pc.all()[1].c_begin = str_start;
pc.all()[1].c_end = str_end - 1;
pc.set_count(2);
return true;
}
}
break;
default:
if (MATCHERS[lpc].pcre.match(pc, this->ds_pcre_input, PCRE_ANCHORED)) {
switch (lpc) {
case DT_IPV6_ADDRESS: {
std::string addr = this->ds_pcre_input.get_substr(pc.all());
char buf[sizeof(struct in6_addr)];
if (inet_pton(AF_INET6, addr.c_str(), buf) == 1) {
token_out = data_token_t(lpc);
return true;
}
this->ds_pcre_input.pi_next_offset = this->ds_pcre_input.pi_offset;
break;
}
default:
token_out = data_token_t(lpc);
return true;
}
}
break;
}
} }
assert((0 <= token_out && token_out < DT_TERMINAL_MAX)); assert((0 <= token_out && token_out < DT_TERMINAL_MAX));

@ -36,18 +36,33 @@
enum data_token_t { enum data_token_t {
DT_INVALID = -1, DT_INVALID = -1,
DT_URL = 0, DT_QUOTED_STRING = 0,
DT_URL,
DT_PATH, DT_PATH,
DT_TIME,
DT_MAC_ADDRESS, DT_MAC_ADDRESS,
DT_QUOTED_STRING, DT_TIME,
DT_IPV6_ADDRESS,
// DT_QUALIFIED_NAME, // DT_QUALIFIED_NAME,
DT_SEPARATOR, DT_SEPARATOR,
DT_COMMA, DT_COMMA,
DT_SEMI,
DT_LCURLY,
DT_RCURLY,
DT_IP_ADDRESS, DT_LSQUARE,
DT_RSQUARE,
DT_LPAREN,
DT_RPAREN,
DT_LANGLE,
DT_RANGLE,
DT_IPV4_ADDRESS,
DT_UUID,
DT_VERSION_NUMBER, DT_VERSION_NUMBER,
DT_OCTAL_NUMBER, DT_OCTAL_NUMBER,
@ -55,7 +70,8 @@ enum data_token_t {
DT_NUMBER, DT_NUMBER,
DT_HEX_NUMBER, DT_HEX_NUMBER,
DT_STRING, DT_WORD,
DT_SYMBOL,
DT_LINE, DT_LINE,
DT_WHITE, DT_WHITE,
DT_DOT, DT_DOT,
@ -73,6 +89,9 @@ enum data_token_t {
DNT_VARIABLE_KEY, DNT_VARIABLE_KEY,
DNT_ROWRANGE, DNT_ROWRANGE,
DNT_DATE_TIME, DNT_DATE_TIME,
DNT_GROUP,
DNT_MAX,
DT_ANY = 100, DT_ANY = 100,
}; };
@ -84,6 +103,9 @@ public:
data_scanner(const std::string &line) : data_scanner(const std::string &line) :
ds_line(line), ds_line(line),
ds_pcre_input(ds_line.c_str()) { ds_pcre_input(ds_line.c_str()) {
if (!line.empty() && line[line.length() - 1] == '.') {
this->ds_pcre_input.pi_length -= 1;
}
}; };
bool tokenize(pcre_context &pc, data_token_t &token_out); bool tokenize(pcre_context &pc, data_token_t &token_out);

@ -33,7 +33,9 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "listview_curses.hh"
#include "hist_source.hh" #include "hist_source.hh"
#include "log_vtab_impl.hh"
class db_label_source : public hist_source::label_source { class db_label_source : public hist_source::label_source {
public: public:
@ -43,16 +45,6 @@ public:
void hist_label_for_group(int group, std::string &label_out) { void hist_label_for_group(int group, std::string &label_out) {
label_out.clear(); label_out.clear();
for (int lpc = 0; lpc < (int)this->dls_headers.size(); lpc++) {
int before, total_fill =
this->dls_column_sizes[lpc] - this->dls_headers[lpc].length();
before = total_fill / 2;
total_fill -= before;
label_out.append(before, ' ');
label_out.append(this->dls_headers[lpc]);
label_out.append(total_fill, ' ');
}
}; };
void hist_label_for_bucket(int bucket_start_value, void hist_label_for_bucket(int bucket_start_value,
@ -67,11 +59,42 @@ public:
if (bucket_start_value >= (int)this->dls_rows.size()) if (bucket_start_value >= (int)this->dls_rows.size())
return; return;
for (int lpc = 0; lpc < (int)this->dls_rows[bucket_start_value].size(); lpc++) { for (int lpc = 0; lpc < (int)this->dls_rows[bucket_start_value].size(); lpc++) {
label_out.append(this->dls_column_sizes[lpc] - this->dls_rows[bucket_start_value][lpc].length(), ' '); int padding = (this->dls_column_sizes[lpc] -
this->dls_rows[bucket_start_value][lpc].length() -
1);
if (this->dls_column_types[lpc] != SQLITE3_TEXT) {
label_out.append(padding, ' ');
}
label_out.append(this->dls_rows[bucket_start_value][lpc]); label_out.append(this->dls_rows[bucket_start_value][lpc]);
if (this->dls_column_types[lpc] == SQLITE3_TEXT) {
label_out.append(padding, ' ');
}
label_out.append(1, ' ');
} }
}; };
void hist_attrs_for_bucket(int bucket_start_value,
const hist_source::bucket_t &bucket,
string_attrs_t &sa) {
struct line_range lr = { 0, 0 };
struct line_range lr2 = { 0, -1 };
if (bucket_start_value >= (int)this->dls_rows.size())
return;
for (size_t lpc = 0; lpc < this->dls_column_sizes.size() - 1; lpc++) {
if (bucket_start_value % 2 == 0) {
sa[lr2].insert(make_string_attr("style", A_BOLD));
}
lr.lr_start += this->dls_column_sizes[lpc] - 1;
lr.lr_end = lr.lr_start + 1;
sa[lr].insert(make_string_attr("graphic", ACS_VLINE));
lr.lr_start += 1;
}
}
// TODO: add support for left and right justification... numbers should
// be right justified and strings should be left.
void push_column(const char *colstr) { void push_column(const char *colstr) {
int index = this->dls_rows.back().size(); int index = this->dls_rows.back().size();
@ -83,7 +106,7 @@ public:
std::max(this->dls_column_sizes[index], strlen(colstr) + 1); std::max(this->dls_column_sizes[index], strlen(colstr) + 1);
}; };
void push_header(const std::string &colstr) { void push_header(const std::string &colstr, int type, bool graphable) {
int index = this->dls_headers.size(); int index = this->dls_headers.size();
this->dls_headers.push_back(colstr); this->dls_headers.push_back(colstr);
@ -92,17 +115,74 @@ public:
} }
this->dls_column_sizes[index] = this->dls_column_sizes[index] =
std::max(this->dls_column_sizes[index], colstr.length() + 1); std::max(this->dls_column_sizes[index], colstr.length() + 1);
this->dls_column_types.push_back(type);
this->dls_headers_to_graph.push_back(graphable);
} }
void clear(void) { void clear(void) {
this->dls_headers.clear(); this->dls_headers.clear();
this->dls_headers_to_graph.clear();
this->dls_column_types.clear();
this->dls_rows.clear(); this->dls_rows.clear();
this->dls_column_sizes.clear(); this->dls_column_sizes.clear();
} }
std::vector< std::string > dls_headers; std::vector< std::string > dls_headers;
std::vector< bool > dls_headers_to_graph;
std::vector<int> dls_column_types;
std::vector< std::vector< std::string > > dls_rows; std::vector< std::vector< std::string > > dls_rows;
std::vector< size_t > dls_column_sizes; std::vector< size_t > dls_column_sizes;
}; };
class db_overlay_source : public list_overlay_source {
public:
db_overlay_source() : dos_labels(NULL) { };
size_t list_overlay_count(const listview_curses &lv) {
return 1;
};
bool list_value_for_overlay(const listview_curses &lv,
vis_line_t y,
attr_line_t &value_out) {
view_colors &vc = view_colors::singleton();
if (y != 0) {
return false;
}
std::string &line = value_out.get_string();
db_label_source *dls = this->dos_labels;
for (size_t lpc = 0;
lpc < this->dos_labels->dls_column_sizes.size();
lpc++) {
int before, total_fill =
dls->dls_column_sizes[lpc] - dls->dls_headers[lpc].length();
struct line_range header_range = { line.length(), line.length() + dls->dls_column_sizes[lpc] };
int attrs = vc.attrs_for_role(this->dos_hist_source->get_role_for_type(bucket_type_t(lpc))) | A_UNDERLINE;
if (!this->dos_labels->dls_headers_to_graph[lpc]) {
attrs = A_UNDERLINE;
}
value_out.get_attrs()[header_range].insert(make_string_attr("style", attrs));
before = total_fill / 2;
total_fill -= before;
line.append(before, ' ');
line.append(dls->dls_headers[lpc]);
line.append(total_fill, ' ');
}
struct line_range lr = { 0, -1 };
value_out.get_attrs()[lr].insert(make_string_attr("style", A_BOLD | A_UNDERLINE));
return true;
};
db_label_source *dos_labels;
hist_source *dos_hist_source;
};
#endif #endif

File diff suppressed because it is too large Load Diff

@ -57,9 +57,8 @@ On color displays, the lines will be highlighted as follows:
* Errors will be colored in red; * Errors will be colored in red;
* warnings will be yellow; * warnings will be yellow;
* boundaries between days will be underlined; and * boundaries between days will be underlined; and
* various color highlights will be applied to: SQL keywords, XML * various color highlights will be applied to: IP addresses, SQL keywords,
tags, file and line numbers in Java backtraces, and XML tags, file and line numbers in Java backtraces, and quoted strings.
quoted strings.
To give you an idea of where you are in the file spatially, the right To give you an idea of where you are in the file spatially, the right
side of the display has a proportionally sized 'scrollbar' that side of the display has a proportionally sized 'scrollbar' that
@ -221,6 +220,9 @@ through the file.
that can be used in queries. See the SQL section that can be used in queries. See the SQL section
below for more information. below for more information.
y/Y Move forward/backward through the log view based on the
"line_number" column in the SQL result view.
v Switch to/from the SQL result view. v Switch to/from the SQL result view.
V Switch between the log and SQL result views while V Switch between the log and SQL result views while
@ -232,6 +234,11 @@ through the file.
to the log view and move to the line number that was to the log view and move to the line number that was
selected in the "line_number" column. selected in the "line_number" column.
p Enable or disable the display of the fields that the
log message parser knows about or has discovered.
This overlay is temporarily enabled when the semicolon
key (;) is pressed so that it is easier to write queries.
MOUSE SUPPORT (experimental) MOUSE SUPPORT (experimental)
---------------------------- ----------------------------
@ -244,6 +251,8 @@ environment variable to "mouse".
COMMANDS COMMANDS
-------- --------
help Switch to this help text view.
unix-time <secs-or-date> unix-time <secs-or-date>
Convert a unix-timestamp in seconds to a Convert a unix-timestamp in seconds to a
human-readable form or vice-versa. human-readable form or vice-versa.
@ -297,7 +306,7 @@ COMMANDS
filter-related commands can be added to the session file. filter-related commands can be added to the session file.
SQL QUERIES SQL QUERIES (experimental)
----------- -----------
Lnav has support for performing SQL queries on log files using the Lnav has support for performing SQL queries on log files using the
@ -312,24 +321,46 @@ being accessed in any loaded Apache log files, you can execute:
The query result view shows the results as text and graphs any number The query result view shows the results as text and graphs any number
values found in the result, much like the histogram view. values found in the result, much like the histogram view.
The basic set of log file tables are: The builtin set of log tables are listed below. Note that only the
log messages that match a particular format can be queried by a
particular table. You can find the file format and table name for
the top log message by looking in the upper right hand corner of the
log file view.
syslog_log, generic_log The log table names are as follows:
Contains any syslog lines and any lines picked up
by the generic log file parser. access_log Apache common access log format
syslog_log Syslog format
glog_log Google glog format
strace_log Strace log format
generic_log 'Generic' log format. This table contains messages
from files that have a very simple format with a
leading timestamp followed by the message.
The columns available for the top log line in the view will
automatically be displayed after pressing the semicolon (;) key.
All log tables contain at least the following columns:
line_number The line number in the file, starting at zero. line_number The line number in the file, starting at zero.
path The full path to the file.
log_time The time of the log entry. log_time The time of the log entry.
idle_msecs The amount of time, in milliseconds, between the
current log message and the previous one.
level The log level (e.g. info, error, etc...). level The log level (e.g. info, error, etc...).
path The full path to the file.
raw_line The raw line of text. raw_line The raw line of text.
The following tables include the basic columns as listed above and The following tables include the basic columns as listed above and
include a few more columns since the log file format is more include a few more columns since the log file format is more
structured. structured.
access_log Contains Apache log file lines. The column names syslog_log
are the same as those in the Microsoft LogParser.
log_hostname The hostname the message was received from.
log_procname The name of the process that sent the message.
log_pid The process ID of the process that sent the message.
access_log (The column names are the same as those in the
Microsoft LogParser tool.)
c_ip The client IP address. c_ip The client IP address.
cs_username The client user name. cs_username The client user name.
@ -342,8 +373,9 @@ structured.
cs_referrer The URL of the referring page. cs_referrer The URL of the referring page.
cs_user_agent The user agent string. cs_user_agent The user agent string.
strace_log Contains strace log file lines. Currently, you strace_log (Currently, you need to run strace with the
need to run strace with the "-tt -T" options. "-tt -T" options so there are timestamps for
each function call.)
funcname The name of the syscall. funcname The name of the syscall.
result The result code. result The result code.
@ -360,3 +392,51 @@ example of a top ten query into the "/tmp/topten.db" file, you can do:
;create table topten as select cs_uri_stem, count(*) as total ;create table topten as select cs_uri_stem, count(*) as total
from access_log group by cs_uri_stem order by total desc from access_log group by cs_uri_stem order by total desc
limit 10; limit 10;
DYNAMIC LOG LINE TABLE (experimental)
----------------------
(NOTE: This feature is still very new and not completely reliable yet,
use with care.)
For log formats that lack message structure, lnav can parse the log
message and attempt to extract any data fields that it finds. This
feature is available through the "logline" log table. This table is
dynamically created and defined based on the message at the top of
the log view. For example, given the following log message from "sudo",
lnav will create the "logline" table with columns for "TTY", "PWD",
"USER", and "COMMAND":
May 24 06:48:38 Tim-Stacks-iMac.local sudo[76387]: stack : TTY=ttys003 ;
PWD=/Users/stack/github/lbuild ; USER=root ;
COMMAND=/bin/echo Hello, World!
Queries executed against this table will then only return results for
other log messages that have the same format. So, if you were to
execute the following query while viewing the above line, you might
get the following results:
;select USER,COMMAND from logline;
USER | COMMAND
---- | -------------------------
root | /bin/echo Hello, World!
mal | /bin/echo Goodbye, World!
The log parser works by examining each message for key/value pairs
separated by an equal sign (=) or a colon (:). For example, in the
previous example of a "sudo" message, the parser sees the "USER=root"
string as a pair where the key is "USER" and the value is "root".
If no pairs can be found, then anything that looks like a value is
extracted and assigned a numbered column. For example, the following
line is from "dhcpd":
Sep 16 22:35:57 drill dhcpd: DHCPDISCOVER from 00:16:ce:54:4e:f3 via hme3
In this case, the lnav parser recognizes that "DHCPDISCOVER", the MAC
address and the "hme3" device name are values and not normal words. So,
it builds a table with three columns for each of these values. The
regular words in the message, like "from" and "via", are then used to
find other messages with a similar format.

@ -88,6 +88,11 @@ void hist_source::text_attrs_for_line(textview_curses &tc,
int row, int row,
string_attrs_t &value_out) string_attrs_t &value_out)
{ {
int grow = row / (this->buckets_per_group() + 1);
int brow = row % (this->buckets_per_group() + 1);
bucket_group_t bg = this->hs_group_keys[grow];
int bucket_index = brow - 1;
assert(this->hs_analyzed); assert(this->hs_analyzed);
if (this->hs_token_bucket != NULL) { if (this->hs_token_bucket != NULL) {
@ -120,9 +125,14 @@ void hist_source::text_attrs_for_line(textview_curses &tc,
lr.lr_end = lr.lr_start + amount; lr.lr_end = lr.lr_start + amount;
value_out[lr].insert(make_string_attr("style", attrs)); value_out[lr].insert(make_string_attr("style", attrs));
lr.lr_start = lr.lr_end; lr.lr_start = lr.lr_end;
} }
this->hs_label_source->hist_attrs_for_bucket((bg * this->hs_group_size) +
(bucket_index * this->hs_bucket_size),
*this->hs_token_bucket,
value_out);
} }
} }

@ -79,6 +79,10 @@ public:
virtual void hist_label_for_bucket(int bucket_start_value, virtual void hist_label_for_bucket(int bucket_start_value,
const bucket_t &bucket, const bucket_t &bucket,
std::string &label_out) { }; std::string &label_out) { };
virtual void hist_attrs_for_bucket(int bucket_start_value,
const bucket_t &bucket,
string_attrs_t &sa) { };
}; };
hist_source(); hist_source();
@ -186,6 +190,21 @@ public:
void add_value(unsigned int value, void add_value(unsigned int value,
bucket_type_t bt, bucket_type_t bt,
bucket_count_t amount = 1.0); bucket_count_t amount = 1.0);
void add_empty_value(unsigned int value) {
bucket_group_t bg;
this->hs_analyzed = false;
bg = bucket_group_t(value / this->hs_group_size);
bucket_array_t &ba = this->hs_groups[bg];
if (ba.empty()) {
ba.resize(this->buckets_per_group());
}
};
void analyze(void); void analyze(void);
protected: protected:

@ -125,11 +125,18 @@ bool listview_curses::handle_key(int ch)
void listview_curses::do_update(void) void listview_curses::do_update(void)
{ {
if (this->lv_window != NULL && this->lv_needs_update) { if (this->lv_window != NULL && this->lv_needs_update) {
vis_line_t y(this->lv_y), height, bottom, lines; vis_line_t y(this->lv_y), height, bottom, lines, row;
attr_line_t overlay_line;
vis_line_t overlay_height(0);
struct line_range lr; struct line_range lr;
unsigned long width; unsigned long width;
size_t row_count; size_t row_count;
if (this->lv_overlay_source != NULL) {
overlay_height = vis_line_t(
this->lv_overlay_source->list_overlay_count(*this));
}
this->get_dimensions(height, width); this->get_dimensions(height, width);
lr.lr_start = this->lv_left; lr.lr_start = this->lv_left;
lr.lr_end = this->lv_left + width; lr.lr_end = this->lv_left + width;
@ -139,20 +146,31 @@ void listview_curses::do_update(void)
this->lv_top = max(vis_line_t(0), vis_line_t(row_count) - height); this->lv_top = max(vis_line_t(0), vis_line_t(row_count) - height);
} }
lines = y + min(height, vis_line_t(row_count) - this->lv_top); row = this->lv_top;
lines = min(height - overlay_height,
vis_line_t(row_count) - this->lv_top);
bottom = y + height; bottom = y + height;
for (; y < lines; ++y) {
vis_line_t row = this->lv_top + y - vis_line_t(this->lv_y);
attr_line_t al;
this->lv_source->listview_value_for_row(*this, row, al);
this->mvwattrline(this->lv_window, y, 0, al, lr);
}
/* Clear out any remaining lines on the display. */
for (; y < bottom; ++y) { for (; y < bottom; ++y) {
wmove(this->lv_window, y, 0); if (this->lv_overlay_source != NULL &&
wclrtoeol(this->lv_window); this->lv_overlay_source->list_value_for_overlay(
*this,
y - vis_line_t(this->lv_y),
overlay_line)) {
this->mvwattrline(this->lv_window, y, 0, overlay_line, lr);
overlay_line.clear();
}
else if (lines > 0) {
attr_line_t al;
this->lv_source->listview_value_for_row(*this, row, al);
this->mvwattrline(this->lv_window, y, 0, al, lr);
--lines;
++row;
}
else {
wmove(this->lv_window, y, 0);
wclrtoeol(this->lv_window);
}
} }
if (this->lv_show_scrollbar) { if (this->lv_show_scrollbar) {

@ -35,6 +35,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <string> #include <string>
#include <vector>
#include <algorithm> #include <algorithm>
#include "strong_int.hh" #include "strong_int.hh"
@ -66,6 +67,31 @@ public:
attr_line_t &value_out) = 0; attr_line_t &value_out) = 0;
}; };
struct listview_overlay {
listview_overlay(int y, const attr_line_t &al) : lo_y(y), lo_line(al) { };
int get_absolute_y(int height) {
if (this->lo_y >= 0)
return this->lo_y;
return height + this->lo_y;
};
int lo_y;
attr_line_t lo_line;
};
class list_overlay_source {
public:
virtual ~list_overlay_source() { };
virtual size_t list_overlay_count(const listview_curses &lv) = 0;
virtual bool list_value_for_overlay(const listview_curses &lv,
vis_line_t y,
attr_line_t &value_out) = 0;
};
/** /**
* View that displays a list of lines that can optionally contain highlighting. * View that displays a list of lines that can optionally contain highlighting.
*/ */
@ -87,7 +113,19 @@ public:
}; };
/** @return The data source delegate. */ /** @return The data source delegate. */
list_data_source *get_data_source() { return this->lv_source; }; list_data_source *get_data_source() const { return this->lv_source; };
/** @param src The data source delegate. */
void set_overlay_source(list_overlay_source *src)
{
this->lv_overlay_source = src;
this->reload_data();
};
/** @return The overlay source delegate. */
list_overlay_source *get_overlay_source() {
return this->lv_overlay_source;
};
/** /**
* @param va The action to invoke when the view is scrolled. * @param va The action to invoke when the view is scrolled.
@ -265,6 +303,7 @@ public:
else { else {
height_out = this->lv_height; height_out = this->lv_height;
} }
}; };
/** This method should be called when the data source has changed. */ /** This method should be called when the data source has changed. */
@ -283,6 +322,7 @@ public:
protected: protected:
list_data_source *lv_source; /*< The data source delegate. */ list_data_source *lv_source; /*< The data source delegate. */
list_overlay_source *lv_overlay_source;
action lv_scroll; /*< The scroll action. */ action lv_scroll; /*< The scroll action. */
WINDOW *lv_window; /*< The window that contains this view. */ WINDOW *lv_window; /*< The window that contains this view. */
unsigned int lv_y; /*< The y offset of this view. */ unsigned int lv_y; /*< The y offset of this view. */

File diff suppressed because it is too large Load Diff

@ -120,6 +120,7 @@ struct _lnav_data {
std::set< std::pair<std::string, int> > ld_file_names; std::set< std::pair<std::string, int> > ld_file_names;
std::list<logfile *> ld_files; std::list<logfile *> ld_files;
std::list<std::string> ld_other_files;
sig_atomic_t ld_looping; sig_atomic_t ld_looping;
sig_atomic_t ld_winched; sig_atomic_t ld_winched;
unsigned long ld_flags; unsigned long ld_flags;
@ -152,6 +153,8 @@ struct _lnav_data {
hist_source ld_db_source; hist_source ld_db_source;
db_label_source ld_db_rows; db_label_source ld_db_rows;
db_overlay_source ld_db_overlay;
std::vector<std::string> ld_db_key_names;
int ld_max_fd; int ld_max_fd;
fd_set ld_read_fds; fd_set ld_read_fds;
@ -168,6 +171,7 @@ void rebuild_indexes(bool force);
std::string dotlnav_path(const char *sub); std::string dotlnav_path(const char *sub);
void ensure_view(textview_curses *expected_tc);
bool toggle_view(textview_curses *toggle_tc); bool toggle_view(textview_curses *toggle_tc);
#endif #endif

@ -159,13 +159,15 @@ static string com_save_to(string cmdline, vector<string> &args)
return "error: expecting file name"; return "error: expecting file name";
} }
snprintf(command, sizeof(command), "echo -n %s", args[1].c_str()); snprintf(command, sizeof(command), "/bin/echo -n %s", args[1].c_str());
if ((pfile = popen(command, "r")) == NULL) { if ((pfile = popen(command, "r")) == NULL) {
return "error: unable to compute file name"; return "error: unable to compute file name";
} }
if (fgets(command, sizeof(command), pfile) == 0) if (fgets(command, sizeof(command), pfile) == 0) {
perror("fgets"); perror("fgets");
return "error: unable to compute file name";
}
fclose(pfile); fclose(pfile);
pfile = NULL; pfile = NULL;
@ -280,6 +282,20 @@ static string com_graph(string cmdline, vector<string> &args)
return retval; return retval;
} }
static string com_help(string cmdline, vector<string> &args)
{
string retval = "";
if (args.size() == 0) {
}
else {
ensure_view(&lnav_data.ld_views[LNV_HELP]);
}
return retval;
}
class pcre_filter class pcre_filter
: public logfile_filter { : public logfile_filter {
public: public:
@ -507,6 +523,7 @@ void init_lnav_commands(readline_context::command_map_t &cmd_map)
cmd_map["current-time"] = com_current_time; cmd_map["current-time"] = com_current_time;
cmd_map["goto"] = com_goto; cmd_map["goto"] = com_goto;
cmd_map["graph"] = com_graph; cmd_map["graph"] = com_graph;
cmd_map["help"] = com_help;
cmd_map["highlight"] = com_highlight; cmd_map["highlight"] = com_highlight;
cmd_map["filter-in"] = com_filter; cmd_map["filter-in"] = com_filter;
cmd_map["filter-out"] = com_filter; cmd_map["filter-out"] = com_filter;

@ -41,6 +41,8 @@
#include <vector> #include <vector>
#include <memory> #include <memory>
#include "view_curses.hh"
/** /**
* Metadata for a single line in a log file. * Metadata for a single line in a log file.
*/ */
@ -139,6 +141,65 @@ private:
uint8_t ll_module; uint8_t ll_module;
}; };
class logline_value {
public:
enum kind_t {
VALUE_TEXT,
VALUE_INTEGER,
VALUE_FLOAT,
};
logline_value(std::string name, int64_t i)
: lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i) { };
logline_value(std::string name, double i)
: lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i) { };
logline_value(std::string name, std::string s)
: lv_name(name), lv_kind(VALUE_TEXT), lv_string(s) { };
logline_value(std::string name, kind_t kind, std::string s)
: lv_name(name), lv_kind(kind) {
switch (kind) {
case VALUE_TEXT:
this->lv_string = s;
break;
case VALUE_INTEGER:
sscanf(s.c_str(), "%qd", &this->lv_number.i);
break;
case VALUE_FLOAT:
sscanf(s.c_str(), "%lf", &this->lv_number.d);
break;
}
};
const std::string to_string() {
char buffer[128];
switch (this->lv_kind) {
case VALUE_TEXT:
return this->lv_string;
case VALUE_INTEGER:
snprintf(buffer, sizeof(buffer), "%qd", this->lv_number.i);
break;
case VALUE_FLOAT:
snprintf(buffer, sizeof(buffer), "%lf", this->lv_number.d);
break;
}
return std::string(buffer);
};
std::string lv_name;
kind_t lv_kind;
union value_u {
int64_t i;
double d;
value_u() : i(0) { };
value_u(int64_t i) : i(i) { };
value_u(double d) : d(d) { };
} lv_number;
std::string lv_string;
};
/** /**
* Base class for implementations of log format parsers. * Base class for implementations of log format parsers.
*/ */
@ -199,6 +260,11 @@ public:
* @param line The log line to edit. * @param line The log line to edit.
*/ */
virtual void scrub(std::string &line) { }; virtual void scrub(std::string &line) { };
virtual void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const
{ };
virtual std::auto_ptr<log_format> specialized(void) = 0; virtual std::auto_ptr<log_format> specialized(void) = 0;

@ -76,6 +76,17 @@ static string scrub_rdns(const string &str)
} }
class access_log_format : public log_format { class access_log_format : public log_format {
static pcrepp &value_pattern(void) {
static pcrepp VALUE_PATTERN(
"^([\\w\\.-]+) [\\w\\.-]+ ([\\w\\.-]+) "
"\\[([^\\]]+)\\] \"(?:\\-|(\\w+) ([^ \\?]+)(\\?[^ ]+)? "
"([\\w/\\.]+))\" (\\d+) "
"(\\d+|-)(?: \"([^\"]+)\" \"([^\"]+)\")?.*");
return VALUE_PATTERN;
};
string get_name() { return "access_log"; }; string get_name() { return "access_log"; };
bool scan(vector < logline > &dst, bool scan(vector < logline > &dst,
@ -126,12 +137,75 @@ class access_log_format : public log_format {
return retval; return retval;
}; };
void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
pcre_context_static<30> pc;
pcre_input pi(line);
if (value_pattern().match(pc, pi)) {
static struct {
const char *name;
logline_value::kind_t kind;
} columns[] = {
{ "c_ip", logline_value::VALUE_TEXT },
{ "cs_username", logline_value::VALUE_TEXT },
{ "", },
{ "cs_method", logline_value::VALUE_TEXT },
{ "cs_uri_stem", logline_value::VALUE_TEXT },
{ "cs_uri_query", logline_value::VALUE_TEXT },
{ "cs_version", logline_value::VALUE_TEXT },
{ "sc_status", logline_value::VALUE_INTEGER },
{ "sc_bytes", logline_value::VALUE_INTEGER },
{ "cs_referer", logline_value::VALUE_TEXT },
{ "cs_user_agent", logline_value::VALUE_TEXT },
{ NULL },
};
pcre_context::iterator iter;
struct line_range lr;
iter = pc.begin() + 3;
lr.lr_start = iter->c_begin;
lr.lr_end = iter->c_end;
sa[lr].insert(make_string_attr("timestamp", 0));
lr.lr_start = 0;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = line.length();
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
for (int lpc = 0; columns[lpc].name; lpc++) {
if (columns[lpc].name[0] == '\0')
continue;
values.push_back(logline_value(columns[lpc].name,
columns[lpc].kind,
pi.get_substr(pc.begin() + lpc)));
}
}
else {
fprintf(stderr, "bad match! %s\n", line.c_str());
}
};
}; };
log_format::register_root_format<access_log_format> access_log_instance; log_format::register_root_format<access_log_format> access_log_instance;
class syslog_log_format : public log_format { class syslog_log_format : public log_format {
static const int TIMESTAMP_LENGTH = 16;
static pcrepp &repeated_pattern(void) {
static pcrepp REPEATED_PATTERN("last message repeated \\d+ times");
return REPEATED_PATTERN;
}
static pcrepp &scrub_pattern(void) { static pcrepp &scrub_pattern(void) {
static pcrepp SCRUB_PATTERN("(\\w+\\s[\\s\\d]\\d \\d+:\\d+:\\d+) [\\.\\-\\w]+( .*)"); static pcrepp SCRUB_PATTERN("(\\w+\\s[\\s\\d]\\d \\d+:\\d+:\\d+) [\\.\\-\\w]+( .*)");
@ -169,6 +243,90 @@ class syslog_log_format : public log_format {
} }
}; };
void annotate(const string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
bool found_hostname = false, found_procname = false, found_prefix = false;
struct line_range lr, hostname_range = { 0, 0 };
struct line_range procname_range = { 0, 0 };
int log_pid = -1;
lr.lr_start = 0;
lr.lr_end = TIMESTAMP_LENGTH;
sa[lr].insert(make_string_attr("timestamp", 0));
for (size_t lpc = TIMESTAMP_LENGTH; lpc < line.size(); lpc++) {
if (line[lpc] == ' ' && !found_hostname) {
hostname_range.lr_start = TIMESTAMP_LENGTH;
hostname_range.lr_end = lpc;
sa[hostname_range].insert(make_string_attr("log_hostname", 0));
found_hostname = true;
}
if (line[lpc] == ' ' && found_hostname && !found_procname) {
bool done = false;
procname_range.lr_start = procname_range.lr_end = lpc + 1;
while (!done) {
switch (line[lpc]) {
case '\0':
done = true;
break;
case ':':
case '[':
procname_range.lr_end = lpc;
done = true;
break;
default:
lpc += 1;
break;
}
}
sa[procname_range].insert(make_string_attr("log_procname", 0));
found_procname = true;
}
if (line[lpc] == '[' && log_pid == -1) {
const char *line_c_str = line.c_str();
sscanf(&line_c_str[lpc + 1], "%d", &log_pid);
}
if (line[lpc] == ':') {
lr.lr_start = 0;
lr.lr_end = lpc + 1;
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = lpc + 1;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
found_prefix = true;
break;
}
}
if (!found_prefix) {
lr.lr_start = 0;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = line.length();
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
hostname_range.lr_start = 0;
hostname_range.lr_end = 0;
}
values.push_back(logline_value("log_hostname", line.substr(hostname_range.lr_start, hostname_range.length())));
values.push_back(logline_value("log_procname", line.substr(procname_range.lr_start, procname_range.length())));
values.push_back(logline_value("log_pid", (int64_t)log_pid));
};
bool scan(vector < logline > &dst, bool scan(vector < logline > &dst,
off_t offset, off_t offset,
char *prefix, char *prefix,
@ -192,7 +350,10 @@ class syslog_log_format : public log_format {
logline::level_t ll = logline::LEVEL_UNKNOWN; logline::level_t ll = logline::LEVEL_UNKNOWN;
time_t log_gmt; time_t log_gmt;
if (error_pattern().match(context, pi)) { if (repeated_pattern().match(context, pi)) {
ll = logline::LEVEL_CONTINUED;
}
else if (error_pattern().match(context, pi)) {
ll = logline::LEVEL_ERROR; ll = logline::LEVEL_ERROR;
} }
else if (warning_pattern().match(context, pi)) { else if (warning_pattern().match(context, pi)) {
@ -281,6 +442,24 @@ class generic_log_format : public log_format {
return SCRUB_PATTERN; return SCRUB_PATTERN;
} }
static const char**get_log_formats() {
static const char *log_fmt[] = {
"%63[0-9: ,.-]%63[^:]%n",
"%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]%n",
"%63[a-zA-Z0-9:.,-] %63[^\n]%n",
"%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]%n",
"%63[a-zA-Z0-9: .,-] %63[^\n]%n",
"[%63[0-9: .-] %*s %63[^\n]%n",
"[%63[a-zA-Z0-9: -+/]] %63[^\n]%n",
"[%63[a-zA-Z0-9: -+/]] [%63[a-zA-Z]]%n",
"[%63[a-zA-Z0-9: .-+/] %*s %63[^\n]%n",
"[%63[a-zA-Z0-9: -+/]] (%*d) %63[^\n]%n",
NULL
};
return log_fmt;
};
string get_name() { return "generic_log"; }; string get_name() { return "generic_log"; };
void scrub(string &line) { void scrub(string &line) {
@ -303,19 +482,6 @@ class generic_log_format : public log_format {
off_t offset, off_t offset,
char *prefix, char *prefix,
int len) { int len) {
static const char *log_fmt[] = {
"%63[0-9: ,.-]%63[^:]",
"%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]",
"%63[a-zA-Z0-9:.,-] %63[^\n]",
"%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]",
"%63[a-zA-Z0-9: .,-] %63[^\n]",
"[%63[0-9: .-] %*s %63[^\n]",
"[%63[a-zA-Z0-9: -+/]] %63[^\n]",
"[%63[a-zA-Z0-9: -+/]] [%63[a-zA-Z]]",
"[%63[a-zA-Z0-9: .-+/] %*s %63[^\n]",
"[%63[a-zA-Z0-9: -+/]] (%*d) %63[^\n]",
NULL
};
bool retval = false; bool retval = false;
struct tm log_time; struct tm log_time;
@ -323,9 +489,10 @@ class generic_log_format : public log_format {
time_t line_time; time_t line_time;
char level[64]; char level[64];
char *last_pos; char *last_pos;
int prefix_len;
if ((last_pos = this->log_scanf(prefix, if ((last_pos = this->log_scanf(prefix,
log_fmt, get_log_formats(),
2, 2,
NULL, NULL,
timestr, timestr,
@ -333,7 +500,8 @@ class generic_log_format : public log_format {
line_time, line_time,
timestr, timestr,
level)) != NULL) { level,
&prefix_len)) != NULL) {
uint16_t millis = 0; uint16_t millis = 0;
/* Try to pull out the milliseconds value. */ /* Try to pull out the milliseconds value. */
@ -352,6 +520,30 @@ class generic_log_format : public log_format {
return retval; return retval;
}; };
void annotate(const string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
const char *fmt = get_log_formats()[this->lf_fmt_lock];
char timestr[64 + 32] = "";
char level[64] = "";
struct line_range lr;
int prefix_len;
sscanf(line.c_str(), fmt, timestr, level, &prefix_len);
lr.lr_start = 0 ? fmt[0] == '%' : 1;
lr.lr_end = lr.lr_start + strlen(timestr);
sa[lr].insert(make_string_attr("timestamp", 0));
lr.lr_start = 0;
lr.lr_end = prefix_len;
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = prefix_len;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
};
auto_ptr<log_format> specialized() { auto_ptr<log_format> specialized() {
auto_ptr<log_format> retval((log_format *) auto_ptr<log_format> retval((log_format *)
new generic_log_format(*this)); new generic_log_format(*this));
@ -363,6 +555,16 @@ class generic_log_format : public log_format {
log_format::register_root_format<generic_log_format> generic_log_instance; log_format::register_root_format<generic_log_format> generic_log_instance;
class glog_log_format : public log_format { class glog_log_format : public log_format {
static pcrepp &value_pattern(void) {
static pcrepp VALUE_PATTERN(
"\\s*(?:[IWECF])\\d+ \\d+:\\d+:\\d+\\.(\\d+)" // level, date
"\\s*([0-9]*)" // thread
"\\s*(.*):(\\d*)\\]" // filename:number
"\\s*(.*)");
return VALUE_PATTERN;
};
string get_name() { return "glog_log"; }; string get_name() { return "glog_log"; };
bool scan(vector < logline > &dst, bool scan(vector < logline > &dst,
@ -419,11 +621,68 @@ class glog_log_format : public log_format {
return retval; return retval;
}; };
void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
pcre_context_static<30> pc;
pcre_input pi(line);
if (value_pattern().match(pc, pi)) {
static struct {
const char *name;
logline_value::kind_t kind;
} columns[] = {
{ "micros", logline_value::VALUE_INTEGER },
{ "thread", logline_value::VALUE_TEXT },
{ "src_file", logline_value::VALUE_TEXT },
{ "src_line", logline_value::VALUE_INTEGER },
{ "message", logline_value::VALUE_TEXT },
{ NULL }
};
pcre_context::iterator iter;
struct line_range lr;
lr.lr_start = 1;
lr.lr_end = 21;
sa[lr].insert(make_string_attr("timestamp", 0));
iter = pc.begin() + 4;
lr.lr_start = 0;
lr.lr_end = iter->c_begin;
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = iter->c_begin;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
for (int lpc = 0; columns[lpc].name; lpc++) {
if (columns[lpc].name[0] == '\0')
continue;
values.push_back(logline_value(columns[lpc].name,
columns[lpc].kind,
pi.get_substr(pc.begin() + lpc)));
}
}
else {
fprintf(stderr, "bad match! %s\n", line.c_str());
}
};
}; };
log_format::register_root_format<glog_log_format> glog_instance; log_format::register_root_format<glog_log_format> glog_instance;
class strace_log_format : public log_format { class strace_log_format : public log_format {
static pcrepp &value_pattern(void) {
static pcrepp VALUE_PATTERN(
"[0-9:.]* ([a-zA-Z_][a-zA-Z_0-9]*)\\("
"(.*)\\)"
"\\s+= ([-xa-fA-F\\d\\?]+).*(?:<(\\d+\\.\\d+)>)?");
return VALUE_PATTERN;
};
string get_name() { return "strace_log"; }; string get_name() { return "strace_log"; };
bool scan(vector < logline > &dst, bool scan(vector < logline > &dst,
@ -486,6 +745,54 @@ class strace_log_format : public log_format {
return retval; return retval;
}; };
void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
pcre_context_static<30> pc;
pcre_input pi(line);
if (value_pattern().match(pc, pi)) {
static struct {
const char *name;
logline_value::kind_t kind;
} columns[] = {
{ "funcname", logline_value::VALUE_TEXT },
{ "args", logline_value::VALUE_TEXT },
{ "result", logline_value::VALUE_TEXT },
{ "duration", logline_value::VALUE_TEXT },
{ NULL },
};
pcre_context::iterator iter;
struct line_range lr;
iter = pc.begin() + 3;
lr.lr_start = iter->c_begin;
lr.lr_end = iter->c_end;
sa[lr].insert(make_string_attr("timestamp", 0));
lr.lr_start = 0;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = line.length();
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
for (int lpc = 0; columns[lpc].name; lpc++) {
if (columns[lpc].name[0] == '\0')
continue;
values.push_back(logline_value(columns[lpc].name,
columns[lpc].kind,
pi.get_substr(pc.begin() + lpc)));
}
}
else {
fprintf(stderr, "bad match! %s\n", line.c_str());
}
};
}; };
log_format::register_root_format<strace_log_format> strace_log_instance; log_format::register_root_format<strace_log_format> strace_log_instance;

@ -35,25 +35,52 @@
using namespace std; using namespace std;
static struct log_cursor log_cursor_latest;
static sql_progress_callback_t vtab_progress_callback;
static const char *type_to_string(int type)
{
switch (type) {
case SQLITE_FLOAT:
return "float";
case SQLITE_INTEGER:
return "integer";
case SQLITE_TEXT:
return "text";
}
assert("Invalid sqlite type");
return NULL;
}
static string declare_table_statement(log_vtab_impl *vi) static string declare_table_statement(log_vtab_impl *vi)
{ {
std::vector<log_vtab_impl::vtab_column> cols; std::vector<log_vtab_impl::vtab_column> cols;
std::vector<log_vtab_impl::vtab_column>::const_iterator iter; std::vector<log_vtab_impl::vtab_column>::const_iterator iter;
std::ostringstream oss; std::ostringstream oss;
oss << "CREATE TABLE unused (\n" oss << "CREATE TABLE unused (\n"
<< " line_number int,\n" << " line_number text,\n"
<< " path text,\n"
<< " log_time datetime,\n" << " log_time datetime,\n"
<< " idle_msecs int,\n" << " idle_msecs int,\n"
<< " level text,\n" << " level text,\n";
<< " raw_line text";
vi->get_columns(cols); vi->get_columns(cols);
vi->vi_column_count = cols.size();
for (iter = cols.begin(); iter != cols.end(); iter++) { for (iter = cols.begin(); iter != cols.end(); iter++) {
oss << ",\n"; auto_mem<char, sqlite3_free> coldecl;
oss << " " << iter->vc_name << " " << iter->vc_type;
coldecl = sqlite3_mprintf(" %Q %s collate %Q,\n",
iter->vc_name,
type_to_string(iter->vc_type),
iter->vc_collator == NULL ?
"BINARY" : iter->vc_collator);
oss << coldecl;
} }
oss << "\n);"; oss << " path text collate naturalnocase,\n"
<< " raw_line text\n"
<< ");";
return oss.str(); return oss.str();
} }
@ -68,6 +95,7 @@ struct vtab {
struct vtab_cursor { struct vtab_cursor {
sqlite3_vtab_cursor base; sqlite3_vtab_cursor base;
struct log_cursor log_cursor; struct log_cursor log_cursor;
std::vector<logline_value> line_values;
}; };
static int vt_destructor(sqlite3_vtab *p_svt); static int vt_destructor(sqlite3_vtab *p_svt);
@ -95,6 +123,9 @@ static int vt_create( sqlite3 *db,
/* Declare the vtable's structure */ /* Declare the vtable's structure */
p_vt->vi = vm->lookup_impl(argv[3]); p_vt->vi = vm->lookup_impl(argv[3]);
if (p_vt->vi == NULL) {
return SQLITE_ERROR;
}
p_vt->lss = vm->get_source(); p_vt->lss = vm->get_source();
rc = sqlite3_declare_vtab(db, declare_table_statement(p_vt->vi).c_str()); rc = sqlite3_declare_vtab(db, declare_table_statement(p_vt->vi).c_str());
@ -138,8 +169,7 @@ static int vt_open(sqlite3_vtab *p_svt, sqlite3_vtab_cursor **pp_cursor)
vtab* p_vt = (vtab*)p_svt; vtab* p_vt = (vtab*)p_svt;
p_vt->base.zErrMsg = NULL; p_vt->base.zErrMsg = NULL;
vtab_cursor *p_cur = vtab_cursor *p_cur = (vtab_cursor*)new vtab_cursor();
(vtab_cursor*)sqlite3_malloc(sizeof(vtab_cursor));
*pp_cursor = (sqlite3_vtab_cursor*)p_cur; *pp_cursor = (sqlite3_vtab_cursor*)p_cur;
@ -156,7 +186,7 @@ static int vt_close(sqlite3_vtab_cursor *cur)
vtab_cursor *p_cur = (vtab_cursor*)cur; vtab_cursor *p_cur = (vtab_cursor*)cur;
/* Free cursor struct. */ /* Free cursor struct. */
sqlite3_free(p_cur); delete p_cur;
return SQLITE_OK; return SQLITE_OK;
} }
@ -175,7 +205,13 @@ static int vt_next(sqlite3_vtab_cursor *cur)
vtab *vt = (vtab *)cur->pVtab; vtab *vt = (vtab *)cur->pVtab;
bool done = false; bool done = false;
vc->line_values.clear();
do { do {
log_cursor_latest = vc->log_cursor;
if (vtab_progress_callback(log_cursor_latest)) {
done = true;
break;
}
done = vt->vi->next(vc->log_cursor, *vt->lss); done = vt->vi->next(vc->log_cursor, *vt->lss);
} }
while (!done); while (!done);
@ -202,16 +238,6 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
sqlite3_result_int64( ctx, vc->log_cursor.lc_curr_line ); sqlite3_result_int64( ctx, vc->log_cursor.lc_curr_line );
} }
break; break;
case VT_COL_PATH:
{
const string &fn = lf->get_filename();
sqlite3_result_text( ctx,
fn.c_str(),
fn.length(),
SQLITE_STATIC );
}
break;
case VT_COL_LOG_TIME: case VT_COL_LOG_TIME:
{ {
time_t line_time; time_t line_time;
@ -252,25 +278,57 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
SQLITE_STATIC); SQLITE_STATIC);
} }
break; break;
case VT_COL_RAW_LINE:
{
string line;
lf->read_line(ll, line);
sqlite3_result_text(ctx,
line.c_str(),
line.length(),
SQLITE_TRANSIENT);
}
break;
default: default:
{ if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) {
logfile::iterator line_iter; int post_col_number = col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1;
string line, value;
if (post_col_number == 0)
line_iter = lf->begin() + cl; {
lf->read_line(line_iter, line); const string &fn = lf->get_filename();
vt->vi->extract(line, col - VT_COL_MAX, ctx);
sqlite3_result_text( ctx,
fn.c_str(),
fn.length(),
SQLITE_STATIC );
}
else {
string line;
lf->read_line(ll, line);
sqlite3_result_text(ctx,
line.c_str(),
line.length(),
SQLITE_TRANSIENT);
}
}
else {
if (vc->line_values.empty()) {
logfile::iterator line_iter;
string line, value;
line_iter = lf->begin() + cl;
lf->read_line(line_iter, line);
vt->vi->extract(lf, line, vc->line_values);
}
{
logline_value &lv = vc->line_values[col - VT_COL_MAX];
switch (lv.lv_kind) {
case logline_value::VALUE_TEXT:
sqlite3_result_text(ctx,
lv.lv_string.c_str(),
lv.lv_string.length(),
SQLITE_TRANSIENT);
break;
case logline_value::VALUE_INTEGER:
sqlite3_result_int64(ctx, lv.lv_number.i);
break;
case logline_value::VALUE_FLOAT:
sqlite3_result_double(ctx, lv.lv_number.d);
break;
}
}
} }
break; break;
} }
@ -322,30 +380,64 @@ static sqlite3_module vtab_module = {
NULL, /* xFindFunction - function overloading */ NULL, /* xFindFunction - function overloading */
}; };
log_vtab_manager::log_vtab_manager(sqlite3 *memdb, logfile_sub_source &lss) static int progress_callback(void *ptr)
{
int retval = 0;
if (vtab_progress_callback != NULL) {
retval = vtab_progress_callback(log_cursor_latest);
}
return retval;
}
log_vtab_manager::log_vtab_manager(sqlite3 *memdb,
logfile_sub_source &lss,
sql_progress_callback_t pc)
: vm_db(memdb), vm_source(lss) : vm_db(memdb), vm_source(lss)
{ {
sqlite3_create_module(this->vm_db, "log_vtab_impl", &vtab_module, this); sqlite3_create_module(this->vm_db, "log_vtab_impl", &vtab_module, this);
vtab_progress_callback = pc;
sqlite3_progress_handler(memdb, 10, progress_callback, NULL);
} }
void log_vtab_manager::register_vtab(log_vtab_impl *vi) { void log_vtab_manager::register_vtab(log_vtab_impl *vi) {
if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) { if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) {
char *sql; char *sql;
int rc; int rc;
this->vm_impls[vi->get_name()] = vi; this->vm_impls[vi->get_name()] = vi;
sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s " sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s "
"USING log_vtab_impl(%s)", "USING log_vtab_impl(%s)",
vi->get_name().c_str(), vi->get_name().c_str(),
vi->get_name().c_str()); vi->get_name().c_str());
rc = sqlite3_exec(this->vm_db, rc = sqlite3_exec(this->vm_db,
sql, sql,
NULL, NULL,
NULL, NULL,
NULL); NULL);
assert(rc == SQLITE_OK); assert(rc == SQLITE_OK);
sqlite3_free(sql); sqlite3_free(sql);
}
}
void log_vtab_manager::unregister_vtab(std::string name) {
if (this->vm_impls.find(name) != this->vm_impls.end()) {
char *sql;
int rc;
sql = sqlite3_mprintf("DROP TABLE %s ", name.c_str());
rc = sqlite3_exec(this->vm_db,
sql,
NULL,
NULL,
NULL);
assert(rc == SQLITE_OK);
sqlite3_free(sql);
} }
this->vm_impls.erase(name);
} }

@ -41,11 +41,9 @@
enum { enum {
VT_COL_LINE_NUMBER, VT_COL_LINE_NUMBER,
VT_COL_PATH,
VT_COL_LOG_TIME, VT_COL_LOG_TIME,
VT_COL_IDLE_MSECS, VT_COL_IDLE_MSECS,
VT_COL_LEVEL, VT_COL_LEVEL,
VT_COL_RAW_LINE,
VT_COL_MAX VT_COL_MAX
}; };
@ -59,11 +57,12 @@ struct log_cursor {
class log_vtab_impl { class log_vtab_impl {
public: public:
struct vtab_column { struct vtab_column {
vtab_column(const char *name, const char *type) vtab_column(const char *name, int type, const char *collator=NULL)
: vc_name(name), vc_type(type) { }; : vc_name(name), vc_type(type), vc_collator(collator) { };
const char *vc_name; const char *vc_name;
const char *vc_type; int vc_type;
const char *vc_collator;
}; };
log_vtab_impl(const std::string name) log_vtab_impl(const std::string name)
@ -84,6 +83,11 @@ public:
content_line_t cl(lss.at(lc.lc_curr_line)); content_line_t cl(lss.at(lc.lc_curr_line));
logfile *lf = lss.find(cl); logfile *lf = lss.find(cl);
logfile::iterator lf_iter = lf->begin() + cl;
if (lf_iter->get_level() & logline::LEVEL_CONTINUED) {
return false;
}
log_format *format = lf->get_format(); log_format *format = lf->get_format();
if (format != NULL && format->get_name() == this->vi_name) if (format != NULL && format->get_name() == this->vi_name)
@ -94,22 +98,32 @@ public:
virtual void get_columns(std::vector<vtab_column> &cols) { }; virtual void get_columns(std::vector<vtab_column> &cols) { };
virtual void extract(const std::string &line, virtual void extract(logfile *lf,
int column, const std::string &line,
sqlite3_context *ctx) { std::vector<logline_value> &values) {
log_format *format = lf->get_format();
string_attrs_t sa;
format->annotate(line, sa, values);
}; };
int vi_column_count;
private: private:
const std::string vi_name; const std::string vi_name;
}; };
typedef int (*sql_progress_callback_t)(const log_cursor &lc);
class log_vtab_manager { class log_vtab_manager {
public: public:
log_vtab_manager(sqlite3 *db, logfile_sub_source &lss); log_vtab_manager(sqlite3 *db,
logfile_sub_source &lss,
sql_progress_callback_t);
logfile_sub_source *get_source() { return &this->vm_source; }; logfile_sub_source *get_source() { return &this->vm_source; };
void register_vtab(log_vtab_impl *vi); void register_vtab(log_vtab_impl *vi);
void unregister_vtab(std::string name);
log_vtab_impl *lookup_impl(std::string name) { log_vtab_impl *lookup_impl(std::string name) {
return this->vm_impls[name]; return this->vm_impls[name];
}; };

@ -168,7 +168,7 @@ void logfile_sub_source::text_value_for_line(textview_curses &tc,
size_t tab; size_t tab;
assert(row >= 0); assert(row >= 0);
assert(row < this->lss_index.size()); assert((size_t)row < this->lss_index.size());
line = this->lss_index[row]; line = this->lss_index[row];
this->lss_token_file = this->find(line); this->lss_token_file = this->find(line);
@ -309,13 +309,21 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv,
lr.lr_end = -1; lr.lr_end = -1;
value_out[lr].insert(make_string_attr("file", this->lss_token_file)); value_out[lr].insert(make_string_attr("file", this->lss_token_file));
if (this->lss_token_date_end > 0 && if ((((this->lss_token_line->get_time() / (5 * 60)) % 2) == 0) &&
((this->lss_token_line->get_time() / (60 * 60)) % 2) == 0) { !(this->lss_token_line->get_level() & logline::LEVEL_CONTINUED)) {
attrs = vc.attrs_for_role(view_colors::VCR_ALT_ROW); log_format *format = this->lss_token_file->get_format();
lr.lr_start = time_offset_end; std::vector<logline_value> line_values;
lr.lr_end = this->lss_token_date_end;
format->annotate(this->lss_token_value, value_out, line_values);
value_out[lr].insert(make_string_attr("style", attrs));
struct line_range time_range = find_string_attr_range(value_out, "timestamp");
if (time_range.lr_end != -1) {
time_range.lr_start += time_offset_end;
time_range.lr_end += time_offset_end;
attrs = vc.attrs_for_role(view_colors::VCR_ALT_ROW);
value_out[time_range].insert(make_string_attr("style", attrs));
}
} }
} }
@ -394,6 +402,9 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
++start_line; ++start_line;
} }
} }
else {
this->lss_filtered_count += 1;
}
start_line = con_line; start_line = con_line;
action = logfile_filter::MAYBE; action = logfile_filter::MAYBE;
action_priority = -1; action_priority = -1;
@ -450,6 +461,9 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
++start_line; ++start_line;
} }
} }
else {
this->lss_filtered_count += 1;
}
iter->ld_lines_indexed = iter->ld_file->size(); iter->ld_lines_indexed = iter->ld_file->size();
@ -473,8 +487,12 @@ void logfile_sub_source::text_update_marks(vis_bookmarks &bm)
bm[&BM_WARNINGS].clear(); bm[&BM_WARNINGS].clear();
bm[&BM_ERRORS].clear(); bm[&BM_ERRORS].clear();
bm[&BM_FILES].clear(); bm[&BM_FILES].clear();
bm[&textview_curses::BM_USER].clear();
bm[&textview_curses::BM_SEARCH].clear(); for (bookmarks<content_line_t>::type::iterator iter = this->lss_user_marks.begin();
iter != this->lss_user_marks.end();
++iter) {
bm[iter->first].clear();
}
for (; vl < (int)this->lss_index.size(); ++vl) { for (; vl < (int)this->lss_index.size(); ++vl) {
content_line_t cl = this->lss_index[vl]; content_line_t cl = this->lss_index[vl];

@ -0,0 +1,169 @@
/**
* Copyright (c) 2013, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file nextwork-extension-functions.cc
*/
#include <stdio.h>
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <arpa/inet.h>
#include "sqlite3.h"
static void sql_gethostbyname(sqlite3_context *context,
int argc, sqlite3_value **argv)
{
char buffer[INET6_ADDRSTRLEN];
const char *name_in;
struct addrinfo *ai;
void *addr_ptr;
int rc;
assert(argc >= 1 && argc <= 2);
if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
sqlite3_result_null(context);
return;
}
name_in = (const char *)sqlite3_value_text(argv[0]);
while ((rc = getaddrinfo(name_in, NULL, NULL, &ai)) == EAI_AGAIN) {
sqlite3_sleep(10);
}
if (rc != 0) {
sqlite3_result_text(context, name_in, -1, SQLITE_TRANSIENT);
return;
}
switch (ai->ai_family) {
case AF_INET:
addr_ptr = &((struct sockaddr_in *)ai->ai_addr)->sin_addr;
break;
case AF_INET6:
addr_ptr = &((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr;
break;
}
inet_ntop(ai->ai_family, addr_ptr, buffer, sizeof(buffer));
sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT);
freeaddrinfo(ai);
}
static void sql_gethostbyaddr(sqlite3_context *context,
int argc, sqlite3_value **argv)
{
union {
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
} sa;
const char *addr_str;
char buffer[NI_MAXHOST];
int family, socklen;
char *addr_raw;
int rc;
assert(argc == 1);
if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
sqlite3_result_null(context);
return;
}
addr_str = (const char *)sqlite3_value_text(argv[0]);
memset(&sa, 0, sizeof(sa));
if (strchr(addr_str, ':')) {
family = AF_INET6;
socklen = sizeof(struct sockaddr_in6);
sa.sin6.sin6_family = family;
addr_raw = (char *)&sa.sin6.sin6_addr;
}
else {
family = AF_INET;
socklen = sizeof(struct sockaddr_in);
sa.sin.sin_family = family;
addr_raw = (char *)&sa.sin.sin_addr;
}
if (inet_pton(family, addr_str, addr_raw) != 1) {
sqlite3_result_text(context, addr_str, -1, SQLITE_TRANSIENT);
return;
}
while ((rc = getnameinfo((struct sockaddr *)&sa, socklen,
buffer, sizeof(buffer), NULL, 0, 0)) == EAI_AGAIN) {
sqlite3_sleep(10);
}
if (rc != 0) {
sqlite3_result_text(context, addr_str, -1, SQLITE_TRANSIENT);
return;
}
sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT);
}
int register_network_extension_functions(sqlite3 *db)
{
static const struct {
const char *name;
char narg;
uint8_t text_rep;
void (*func)(sqlite3_context*,int,sqlite3_value**);
} plain_funcs[] = {
{ "gethostbyname", 1, SQLITE_UTF8, sql_gethostbyname },
{ "gethostbyaddr", 1, SQLITE_UTF8, sql_gethostbyaddr },
{ NULL }
};
int retval;
for (int lpc = 0; plain_funcs[lpc].name; lpc++) {
retval = sqlite3_create_function(db,
plain_funcs[lpc].name,
plain_funcs[lpc].narg,
plain_funcs[lpc].text_rep,
NULL,
plain_funcs[lpc].func,
NULL,
NULL);
if (retval != SQLITE_OK)
return retval;
}
return SQLITE_OK;
}

@ -100,7 +100,7 @@ public:
iterator begin() { return pc_captures + 1; }; iterator begin() { return pc_captures + 1; };
/** @return An iterator that refers to the end of the capture array. */ /** @return An iterator that refers to the end of the capture array. */
iterator end() { return pc_captures + pc_count; }; iterator end() { return pc_captures + pc_count; };
protected: protected:
pcre_context(capture_t *captures, int max_count) pcre_context(capture_t *captures, int max_count)
: pc_captures(captures), pc_max_count(max_count), pc_count(0) { }; : pc_captures(captures), pc_max_count(max_count), pc_count(0) { };
@ -110,6 +110,28 @@ protected:
int pc_count; int pc_count;
}; };
struct capture_if_not {
capture_if_not(int begin) : cin_begin(begin) { };
bool operator()(const pcre_context::capture_t &cap) {
return cap.c_begin != this->cin_begin;
}
int cin_begin;
};
inline
pcre_context::iterator skip_invalid_captures(pcre_context::iterator iter,
pcre_context::iterator pc_end)
{
for (; iter != pc_end; ++iter) {
if (iter->c_begin == -1)
continue;
}
return iter;
}
/** /**
* A pcre_context that allocates storage for the capture array within the object * A pcre_context that allocates storage for the capture array within the object
* itself. * itself.
@ -152,6 +174,9 @@ public:
}; };
std::string get_substr(pcre_context::const_iterator iter) const { std::string get_substr(pcre_context::const_iterator iter) const {
if (iter->c_begin == -1) {
return "";
}
return std::string(this->pi_string, return std::string(this->pi_string,
iter->c_begin, iter->c_begin,
iter->length()); iter->length());

@ -132,8 +132,13 @@ char *readline_context::completion_generator(const char *text, int state)
for (iter = arg_possibilities->begin(); for (iter = arg_possibilities->begin();
iter != arg_possibilities->end(); iter != arg_possibilities->end();
++iter) { ++iter) {
int (*cmpfunc)(const char *, const char *, size_t);
fprintf(stderr, " cmp %s %s\n", text, iter->c_str()); fprintf(stderr, " cmp %s %s\n", text, iter->c_str());
if (strncmp(text, iter->c_str(), len) == 0) { cmpfunc = (loaded_context->is_case_sensitive() ?
strncmp : strncasecmp);
if (cmpfunc(text, iter->c_str(), len) == 0) {
fprintf(stderr, "match!!! %d %s %s\n", len, text, iter->c_str());
matches.push_back(*iter); matches.push_back(*iter);
} }
} }
@ -143,6 +148,8 @@ char *readline_context::completion_generator(const char *text, int state)
if (!matches.empty()) { if (!matches.empty()) {
retval = strdup(matches.back().c_str()); retval = strdup(matches.back().c_str());
matches.pop_back(); matches.pop_back();
fprintf(stderr, "comp gen %s\n", retval);
} }
return retval; return retval;
@ -154,6 +161,8 @@ char **readline_context::attempted_completion(const char *text,
{ {
char **retval = NULL; char **retval = NULL;
fprintf(stderr, "attempted %s\n", text);
if (loaded_context->rc_possibilities.find("*") != loaded_context->rc_possibilities.end()) { if (loaded_context->rc_possibilities.find("*") != loaded_context->rc_possibilities.end()) {
fprintf(stderr, "all poss\n"); fprintf(stderr, "all poss\n");
arg_possibilities = &loaded_context->rc_possibilities["*"]; arg_possibilities = &loaded_context->rc_possibilities["*"];
@ -378,6 +387,9 @@ void readline_curses::start(void)
rem_possibility(string(type), rem_possibility(string(type),
string(&msg[prompt_start])); string(&msg[prompt_start]));
} }
else if (sscanf(msg, "cp:%d:%s", &context, type)) {
this->rc_contexts[context]->clear_possibilities(type);
}
else { else {
fprintf(stderr, "unhandled message: %s\n", msg); fprintf(stderr, "unhandled message: %s\n", msg);
} }
@ -594,6 +606,20 @@ void readline_curses::rem_possibility(int context, string type, string value)
} }
} }
void readline_curses::clear_possibilities(int context, string type)
{
char buffer[1024];
snprintf(buffer, sizeof(buffer),
"cp:%d:%s",
context, type.c_str());
if (reliable_send(this->rc_command_pipe[RCF_MASTER],
buffer,
strlen(buffer) + 1) == -1) {
perror("clear_possiblity: write failed");
}
}
void readline_curses::do_update(void) void readline_curses::do_update(void)
{ {
if (this->rc_active_context == -1) { if (this->rc_active_context == -1) {

@ -63,8 +63,11 @@ public:
std::vector<std::string> &args); std::vector<std::string> &args);
typedef std::map<std::string, command_t> command_map_t; typedef std::map<std::string, command_t> command_map_t;
readline_context(const std::string &name, command_map_t *commands = NULL) readline_context(const std::string &name,
: rc_name(name) command_map_t *commands = NULL,
bool case_sensitive = true)
: rc_name(name),
rc_case_sensitive(case_sensitive)
{ {
char *home; char *home;
@ -97,6 +100,16 @@ public:
void load(void) void load(void)
{ {
char buffer[128];
/*
* XXX Need to keep the input on a single line since the display screws
* up if it wraps around.
*/
snprintf(buffer, sizeof(buffer),
"set completion-ignore-case %s",
this->rc_case_sensitive ? "off" : "on");
rl_parse_and_bind(buffer); // NOTE: buffer is modified
loaded_context = this; loaded_context = this;
rl_attempted_completion_function = attempted_completion; rl_attempted_completion_function = attempted_completion;
history_set_history_state(&this->rc_history); history_set_history_state(&this->rc_history);
@ -126,6 +139,15 @@ public:
this->rc_possibilities[type].erase(value); this->rc_possibilities[type].erase(value);
}; };
void clear_possibilities(std::string type)
{
this->rc_possibilities[type].clear();
};
bool is_case_sensitive(void) const {
return this->rc_case_sensitive;
};
private: private:
static char **attempted_completion(const char *text, int start, int end); static char **attempted_completion(const char *text, int start, int end);
static char *completion_generator(const char *text, int state); static char *completion_generator(const char *text, int state);
@ -137,6 +159,7 @@ private:
HISTORY_STATE rc_history; HISTORY_STATE rc_history;
std::map<std::string, std::set<std::string> > rc_possibilities; std::map<std::string, std::set<std::string> > rc_possibilities;
std::map<std::string, std::vector<std::string> > rc_prototypes; std::map<std::string, std::vector<std::string> > rc_prototypes;
bool rc_case_sensitive;
}; };
/** /**
@ -211,6 +234,7 @@ public:
void add_possibility(int context, std::string type, std::string value); void add_possibility(int context, std::string type, std::string value);
void rem_possibility(int context, std::string type, std::string value); void rem_possibility(int context, std::string type, std::string value);
void clear_possibilities(int context, std::string type);
private: private:
enum { enum {

@ -37,7 +37,7 @@ using namespace std;
void statusview_curses::do_update(void) void statusview_curses::do_update(void)
{ {
int top, attrs, field, field_count, left = 1, right; int top, attrs, field, field_count, left = 1, right;
view_colors &vc = view_colors::singleton(); view_colors &vc = view_colors::singleton();
unsigned long width, height; unsigned long width, height;
@ -60,6 +60,7 @@ void statusview_curses::do_update(void)
int x; int x;
val = sf.get_value(); val = sf.get_value();
if (sf.is_right_justified()) { if (sf.is_right_justified()) {
right -= 1 + sf.get_width(); right -= 1 + sf.get_width();
x = right; x = right;

@ -53,7 +53,8 @@ public:
sf_right_justify(false), sf_right_justify(false),
sf_cylon(false), sf_cylon(false),
sf_cylon_pos(0), sf_cylon_pos(0),
sf_role(role) { }; sf_role(role),
sf_share(0) { };
virtual ~status_field() { }; virtual ~status_field() { };
@ -77,6 +78,14 @@ public:
value = abbrev; value = abbrev;
} }
} }
if (this->sf_right_justify) {
int padding = this->sf_width - value.size();
if (padding > 2) {
value.insert(0, padding, ' ');
}
}
this->sf_value = value; this->sf_value = value;
@ -136,13 +145,23 @@ public:
/** @param width The maximum display width, in characters. */ /** @param width The maximum display width, in characters. */
size_t get_width() const { return this->sf_width; }; size_t get_width() const { return this->sf_width; };
/** @param width The maximum display width, in characters. */
void set_min_width(int width) { this->sf_min_width = width; };
/** @param width The maximum display width, in characters. */
size_t get_min_width() const { return this->sf_min_width; };
void set_share(int share) { this->sf_share = share; };
int get_share() const { return this->sf_share; };
protected: protected:
size_t sf_width; /*< The maximum display width, in chars. */ size_t sf_width; /*< The maximum display width, in chars. */
size_t sf_min_width; /*< The maximum display width, in chars. */
bool sf_right_justify; bool sf_right_justify;
bool sf_cylon; bool sf_cylon;
size_t sf_cylon_pos; size_t sf_cylon_pos;
attr_line_t sf_value; /*< The value to display for this field. */ attr_line_t sf_value; /*< The value to display for this field. */
view_colors::role_t sf_role; /*< The color role for this field. */ view_colors::role_t sf_role; /*< The color role for this field. */
int sf_share;
}; };
/** /**
@ -187,6 +206,39 @@ public:
void set_window(WINDOW *win) { this->sc_window = win; }; void set_window(WINDOW *win) { this->sc_window = win; };
WINDOW *get_window() { return this->sc_window; }; WINDOW *get_window() { return this->sc_window; };
void window_change(void) {
int field_count = this->sc_source->statusview_fields();
int remaining, total_shares = 0;
unsigned long width, height;
getmaxyx(this->sc_window, height, width);
remaining = width - 4;
for (int field = 0; field < field_count; field++) {
status_field &sf = this->sc_source->statusview_value_for_field(field);
remaining -= sf.get_share() ? sf.get_min_width() : sf.get_width();
remaining -= 1;
total_shares += sf.get_share();
}
if (remaining < 2) {
remaining = 0;
}
for (int field = 0; field < field_count; field++) {
status_field &sf = this->sc_source->statusview_value_for_field(field);
if (sf.get_share()) {
int actual_width;
actual_width = sf.get_min_width();
actual_width += remaining / (sf.get_share() / total_shares);
sf.set_width(actual_width);
}
}
};
void do_update(void); void do_update(void);
private: private:

@ -0,0 +1,209 @@
/* -*- mode: c; c-file-style: "k&r" -*-
strnatcmp.c -- Perform 'natural order' comparisons of strings in C.
Copyright (C) 2000, 2004 by Martin Pool <mbp sourcefrog net>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
/* partial change history:
*
* 2004-10-10 mbp: Lift out character type dependencies into macros.
*
* Eric Sosman pointed out that ctype functions take a parameter whose
* value must be that of an unsigned int, even on platforms that have
* negative chars in their default char type.
*/
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include "strnatcmp.h"
/* These are defined as macros to make it easier to adapt this code to
* different characters types or comparison functions. */
static inline int
nat_isdigit(nat_char a)
{
return isdigit((unsigned char) a);
}
static inline int
nat_isspace(nat_char a)
{
return isspace((unsigned char) a);
}
static inline nat_char
nat_toupper(nat_char a)
{
return toupper((unsigned char) a);
}
static int
compare_right(int a_len, nat_char const *a, int b_len, nat_char const *b)
{
int bias = 0;
/* The longest run of digits wins. That aside, the greatest
value wins, but we can't know that it will until we've scanned
both numbers to know that they have the same magnitude, so we
remember it in BIAS. */
for (;; a++, b++, a_len--, b_len--) {
if (a_len == 0 && b_len == 0)
return 0;
if (a_len == 0)
return -1;
if (b_len == 0)
return 1;
if (!nat_isdigit(*a) && !nat_isdigit(*b))
return bias;
else if (!nat_isdigit(*a))
return -1;
else if (!nat_isdigit(*b))
return +1;
else if (*a < *b) {
if (!bias)
bias = -1;
} else if (*a > *b) {
if (!bias)
bias = +1;
} else if (!*a && !*b)
return bias;
}
return 0;
}
static int
compare_left(int a_len, nat_char const *a, int b_len, nat_char const *b)
{
/* Compare two left-aligned numbers: the first to have a
different value wins. */
for (;; a++, b++, a_len--, b_len--) {
if (a_len == 0 && b_len == 0)
return 0;
if (a_len == 0)
return -1;
if (b_len == 0)
return 1;
if (!nat_isdigit(*a) && !nat_isdigit(*b))
return 0;
else if (!nat_isdigit(*a))
return -1;
else if (!nat_isdigit(*b))
return +1;
else if (*a < *b)
return -1;
else if (*a > *b)
return +1;
}
return 0;
}
static int strnatcmp0(int a_len, nat_char const *a,
int b_len, nat_char const *b,
int fold_case)
{
int ai, bi;
nat_char ca, cb;
int fractional, result;
assert(a && b);
ai = bi = 0;
while (1) {
if (ai >= a_len)
ca = 0;
else
ca = a[ai];
if (bi >= b_len)
cb = 0;
else
cb = b[bi];
/* skip over leading spaces or zeros */
while (nat_isspace(ca)) {
ai += 1;
if (ai >= a_len)
ca = 0;
else
ca = a[ai];
}
while (nat_isspace(cb)) {
bi += 1;
if (bi >= b_len)
cb = 0;
else
cb = b[bi];
}
/* process run of digits */
if (nat_isdigit(ca) && nat_isdigit(cb)) {
fractional = (ca == '0' || cb == '0');
if (fractional) {
if ((result = compare_left(a_len - ai, a+ai, b_len - bi, b+bi)) != 0)
return result;
} else {
if ((result = compare_right(a_len - ai, a+ai, b_len - bi, b+bi)) != 0)
return result;
}
}
if (!ca && !cb) {
/* The strings compare the same. Perhaps the caller
will want to call strcmp to break the tie. */
return 0;
}
if (fold_case) {
ca = nat_toupper(ca);
cb = nat_toupper(cb);
}
if (ca < cb)
return -1;
else if (ca > cb)
return +1;
++ai; ++bi;
}
}
int strnatcmp(int a_len, nat_char const *a, int b_len, nat_char const *b) {
return strnatcmp0(a_len, a, b_len, b, 0);
}
/* Compare, recognizing numeric string and ignoring case. */
int strnatcasecmp(int a_len, nat_char const *a, int b_len, nat_char const *b) {
return strnatcmp0(a_len, a, b_len, b, 1);
}

@ -0,0 +1,31 @@
/* -*- mode: c; c-file-style: "k&r" -*-
strnatcmp.c -- Perform 'natural order' comparisons of strings in C.
Copyright (C) 2000, 2004 by Martin Pool <mbp sourcefrog net>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
/* CUSTOMIZATION SECTION
*
* You can change this typedef, but must then also change the inline
* functions in strnatcmp.c */
typedef char nat_char;
int strnatcmp(int a_len, nat_char const *a, int b_len, nat_char const *b);
int strnatcasecmp(int a_len, nat_char const *a, int b_len, nat_char const *b);

@ -149,7 +149,7 @@ public:
} }
} while (rc > 0); } while (rc > 0);
for (int lpc = 0; lpc < range_queue.size(); lpc++) { for (size_t lpc = 0; lpc < range_queue.size(); lpc++) {
sa[range_queue[lpc]].insert(attr_queue[lpc]); sa[range_queue[lpc]].insert(attr_queue[lpc]);
} }
}; };

@ -65,7 +65,8 @@ public:
this->tss_fields[TSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS); this->tss_fields[TSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS);
this->tss_fields[TSF_FORMAT].set_width(15); this->tss_fields[TSF_FORMAT].set_width(15);
this->tss_fields[TSF_FORMAT].right_justify(true); this->tss_fields[TSF_FORMAT].right_justify(true);
this->tss_fields[TSF_FILENAME].set_width(35); // XXX this->tss_fields[TSF_FILENAME].set_min_width(35); // XXX
this->tss_fields[TSF_FILENAME].set_share(1);
this->tss_fields[TSF_FILENAME].right_justify(true); this->tss_fields[TSF_FILENAME].right_justify(true);
}; };

@ -2,10 +2,10 @@
* Copyright (c) 2007-2012, Timothy Stack * Copyright (c) 2007-2012, Timothy Stack
* *
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met: * modification, are permitted provided that the following conditions are met:
* *
* * Redistributions of source code must retain the above copyright notice, this * * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer. * list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, * * Redistributions in binary form must reproduce the above copyright notice,
@ -14,7 +14,7 @@
* * Neither the name of Timothy Stack nor the names of its contributors * * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software * may be used to endorse or promote products derived from this software
* without specific prior written permission. * without specific prior written permission.
* *
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@ -55,7 +55,7 @@ void view_curses::mvwattrline(WINDOW *window,
size_t exp_index = 0; size_t exp_index = 0;
string full_line; string full_line;
assert(lr.lr_end != -1); assert(lr.lr_end >= 0);
line_width = lr.length(); line_width = lr.length();
buffer = (char *)alloca(line_width + 1); buffer = (char *)alloca(line_width + 1);
@ -95,13 +95,18 @@ void view_curses::mvwattrline(WINDOW *window,
whline(window, ' ', lr.lr_end - full_line.size()); whline(window, ' ', lr.lr_end - full_line.size());
wattroff(window, attrs); wattroff(window, attrs);
std::vector<line_range> graphic_range;
std::vector<int> graphic_in;
for (iter = sa.begin(); iter != sa.end(); ++iter) { for (iter = sa.begin(); iter != sa.end(); ++iter) {
struct line_range attr_range = iter->first; struct line_range attr_range = iter->first;
std::map<size_t, size_t>::iterator tab_iter; std::map<size_t, size_t>::iterator tab_iter;
assert(attr_range.lr_start >= 0); assert(attr_range.lr_start >= 0);
assert(attr_range.lr_end >= -1); assert(attr_range.lr_end >= -1);
fprintf(stderr, "attr %d %d\n", attr_range.lr_start, attr_range.lr_end);
tab_iter = tab_list.lower_bound(attr_range.lr_start); tab_iter = tab_list.lower_bound(attr_range.lr_start);
if (tab_iter != tab_list.end()) if (tab_iter != tab_list.end())
attr_range.lr_start += (tab_iter->second - tab_iter->first) - 1; attr_range.lr_start += (tab_iter->second - tab_iter->first) - 1;
@ -111,7 +116,7 @@ void view_curses::mvwattrline(WINDOW *window,
if (tab_iter != tab_list.end()) if (tab_iter != tab_list.end())
attr_range.lr_end += (tab_iter->second - tab_iter->first) - 1; attr_range.lr_end += (tab_iter->second - tab_iter->first) - 1;
} }
attr_range.lr_start = max(0, attr_range.lr_start - lr.lr_start); attr_range.lr_start = max(0, attr_range.lr_start - lr.lr_start);
if (attr_range.lr_end == -1) { if (attr_range.lr_end == -1) {
attr_range.lr_end = line_width; attr_range.lr_end = line_width;
@ -128,19 +133,36 @@ void view_curses::mvwattrline(WINDOW *window,
attrs = 0; attrs = 0;
for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) { for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) {
if (am_iter->first == "style") { if (am_iter->first == "style") {
attrs |= am_iter->second.sa_int; attrs |= am_iter->second.sa_int;
} }
} }
/* This silliness is brought to you by a buggy old curses lib. */ if (attrs != 0) {
mvwinnstr(window, y, x + attr_range.lr_start, buffer, awidth); fprintf(stderr, "text %d %d %x\n", y, x + attr_range.lr_start, attrs);
wattron(window, attrs); /* This silliness is brought to you by a buggy old curses lib. */
mvwaddnstr(window, y, x + attr_range.lr_start, buffer, awidth); mvwinnstr(window, y, x + attr_range.lr_start, buffer, awidth);
wattroff(window, attrs); wattron(window, attrs);
} mvwaddnstr(window, y, x + attr_range.lr_start, buffer, awidth);
wattroff(window, attrs);
attrs = text_attrs; /* Reset attrs to regular text. */ }
for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) {
if (am_iter->first == "graphic") {
graphic_range.push_back(attr_range);
graphic_in.push_back(am_iter->second.sa_int | attrs);
}
}
}
attrs = text_attrs; /* Reset attrs to regular text. */
}
for (size_t lpc = 0; lpc < graphic_range.size(); lpc++) {
for (int lpc2 = graphic_range[lpc].lr_start;
lpc2 < graphic_range[lpc].lr_end;
lpc2++) {
mvwaddch(window, y, lpc2, graphic_in[lpc]);
}
} }
} }
@ -180,6 +202,8 @@ view_colors::view_colors()
this->vc_role_colors[VCR_DIFF_ADD] = COLOR_PAIR(VC_GREEN); this->vc_role_colors[VCR_DIFF_ADD] = COLOR_PAIR(VC_GREEN);
this->vc_role_colors[VCR_DIFF_SECTION] = COLOR_PAIR(VC_MAGENTA); this->vc_role_colors[VCR_DIFF_SECTION] = COLOR_PAIR(VC_MAGENTA);
this->vc_role_colors[VCR_SHADOW] = COLOR_PAIR(VC_GRAY);
for (lpc = 0; lpc < VCR__MAX; lpc++) { for (lpc = 0; lpc < VCR__MAX; lpc++) {
this->vc_role_reverse_colors[lpc] = this->vc_role_reverse_colors[lpc] =
this->vc_role_colors[lpc] | A_REVERSE; this->vc_role_colors[lpc] | A_REVERSE;
@ -211,7 +235,7 @@ void view_colors::init(void)
init_pair(VC_CYAN, COLOR_CYAN, COLOR_BLACK); init_pair(VC_CYAN, COLOR_CYAN, COLOR_BLACK);
init_pair(VC_GREEN, COLOR_GREEN, COLOR_BLACK); init_pair(VC_GREEN, COLOR_GREEN, COLOR_BLACK);
init_pair(VC_MAGENTA, COLOR_MAGENTA, COLOR_BLACK); init_pair(VC_MAGENTA, COLOR_MAGENTA, COLOR_BLACK);
init_pair(VC_BLUE_ON_WHITE, COLOR_BLUE, COLOR_WHITE); init_pair(VC_BLUE_ON_WHITE, COLOR_BLUE, COLOR_WHITE);
init_pair(VC_CYAN_ON_BLACK, COLOR_CYAN, COLOR_BLACK); init_pair(VC_CYAN_ON_BLACK, COLOR_CYAN, COLOR_BLACK);
init_pair(VC_GREEN_ON_WHITE, COLOR_GREEN, COLOR_WHITE); init_pair(VC_GREEN_ON_WHITE, COLOR_GREEN, COLOR_WHITE);
@ -224,12 +248,14 @@ void view_colors::init(void)
init_pair(VC_BLACK_ON_WHITE, COLOR_BLACK, COLOR_WHITE); init_pair(VC_BLACK_ON_WHITE, COLOR_BLACK, COLOR_WHITE);
init_pair(VC_RED_ON_WHITE, COLOR_RED, COLOR_WHITE); init_pair(VC_RED_ON_WHITE, COLOR_RED, COLOR_WHITE);
init_pair(VC_YELLOW_ON_WHITE, COLOR_YELLOW, COLOR_WHITE); init_pair(VC_YELLOW_ON_WHITE, COLOR_YELLOW, COLOR_WHITE);
init_pair(VC_WHITE_ON_GREEN, COLOR_WHITE, COLOR_GREEN); init_pair(VC_WHITE_ON_GREEN, COLOR_WHITE, COLOR_GREEN);
init_pair(VC_GRAY, COLOR_BLACK, COLOR_BLACK);
} }
} }
view_colors::role_t view_colors::next_highlight(void) view_colors::role_t view_colors::next_highlight()
{ {
role_t retval = (role_t)(VCR__MAX + this->vc_next_highlight); role_t retval = (role_t)(VCR__MAX + this->vc_next_highlight);
@ -238,3 +264,13 @@ view_colors::role_t view_colors::next_highlight(void)
return retval; return retval;
} }
view_colors::role_t view_colors::next_plain_highlight()
{
role_t retval = (role_t)(VCR__MAX + this->vc_next_plain_highlight);
this->vc_next_plain_highlight = (this->vc_next_plain_highlight + 2) %
(HL_COLOR_COUNT * 2);
return retval;
}

@ -92,6 +92,8 @@ struct line_range {
bool operator<(const struct line_range &rhs) const { bool operator<(const struct line_range &rhs) const {
if (this->lr_start < rhs.lr_start) return true; if (this->lr_start < rhs.lr_start) return true;
else if (this->lr_start > rhs.lr_start) return false; else if (this->lr_start > rhs.lr_start) return false;
if (this->lr_end == rhs.lr_end) return false;
if (this->lr_end < rhs.lr_end) return true; if (this->lr_end < rhs.lr_end) return true;
return false; return false;
@ -143,12 +145,31 @@ typedef std::multimap<std::string, string_attr_t> attrs_map_t;
/** A map of line ranges to attributes for that range. */ /** A map of line ranges to attributes for that range. */
typedef std::map<struct line_range, attrs_map_t> string_attrs_t; typedef std::map<struct line_range, attrs_map_t> string_attrs_t;
inline struct line_range
find_string_attr_range(const string_attrs_t &sa, const std::string &name) {
struct line_range retval = { -1, -1 };
for (string_attrs_t::const_iterator iter = sa.begin();
iter != sa.end();
++iter) {
attrs_map_t::const_iterator prefix_iter;
if ((prefix_iter = iter->second.find(name)) != iter->second.end()) {
retval = iter->first;
break;
}
}
return retval;
}
/** /**
* A line that has attributes. * A line that has attributes.
*/ */
class attr_line_t { class attr_line_t {
public: public:
attr_line_t() { }; attr_line_t() { };
attr_line_t(const std::string &str) : al_string(str) { };
/** @return The string itself. */ /** @return The string itself. */
std::string &get_string() { return this->al_string; }; std::string &get_string() { return this->al_string; };
@ -319,6 +340,8 @@ public:
VCR_DIFF_ADD, /*< Added line in a diff. */ VCR_DIFF_ADD, /*< Added line in a diff. */
VCR_DIFF_SECTION, /*< Section marker in a diff. */ VCR_DIFF_SECTION, /*< Section marker in a diff. */
VCR_SHADOW,
VCR__MAX VCR__MAX
} role_t; } role_t;
@ -357,7 +380,9 @@ public:
* method will iterate through eight-or-so attributes combinations so there * method will iterate through eight-or-so attributes combinations so there
* is some variety in how text is highlighted. * is some variety in how text is highlighted.
*/ */
role_t next_highlight(void); role_t next_highlight();
role_t next_plain_highlight();
enum { enum {
VC_EMPTY = 0, /* XXX Dead color pair, doesn't work. */ VC_EMPTY = 0, /* XXX Dead color pair, doesn't work. */
@ -381,6 +406,8 @@ public:
VC_RED_ON_WHITE, VC_RED_ON_WHITE,
VC_WHITE_ON_GREEN, VC_WHITE_ON_GREEN,
VC_GRAY,
}; };
private: private:
@ -397,6 +424,7 @@ private:
int vc_role_reverse_colors[VCR__MAX + (HL_COLOR_COUNT * 2)]; int vc_role_reverse_colors[VCR__MAX + (HL_COLOR_COUNT * 2)];
/** The index of the next highlight color to use. */ /** The index of the next highlight color to use. */
int vc_next_highlight; int vc_next_highlight;
int vc_next_plain_highlight;
}; };
/** /**

@ -110,6 +110,7 @@ drive_data_scanner_SOURCES = \
../src/data_parser.cc \ ../src/data_parser.cc \
../src/data_scanner.cc \ ../src/data_scanner.cc \
drive_data_scanner.cc drive_data_scanner.cc
drive_data_scanner_LDADD = -lcrypto
drive_view_colors_SOURCES = \ drive_view_colors_SOURCES = \
../src/view_curses.cc \ ../src/view_curses.cc \
@ -151,6 +152,8 @@ dist_noinst_DATA = \
datafile_simple.3 \ datafile_simple.3 \
datafile_simple.4 \ datafile_simple.4 \
datafile_simple.5 \ datafile_simple.5 \
datafile_simple.6 \
datafile_simple.7 \
listview_output.0 \ listview_output.0 \
listview_output.1 \ listview_output.1 \
listview_output.2 \ listview_output.2 \

@ -86,7 +86,6 @@ CONFIG_CLEAN_VPATH_FILES =
am_drive_data_scanner_OBJECTS = data_parser.$(OBJEXT) \ am_drive_data_scanner_OBJECTS = data_parser.$(OBJEXT) \
data_scanner.$(OBJEXT) drive_data_scanner.$(OBJEXT) data_scanner.$(OBJEXT) drive_data_scanner.$(OBJEXT)
drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS) drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS)
drive_data_scanner_LDADD = $(LDADD)
drive_data_scanner_DEPENDENCIES = drive_data_scanner_DEPENDENCIES =
am_drive_grep_proc_OBJECTS = line_buffer.$(OBJEXT) grep_proc.$(OBJEXT) \ am_drive_grep_proc_OBJECTS = line_buffer.$(OBJEXT) grep_proc.$(OBJEXT) \
drive_grep_proc.$(OBJEXT) drive_grep_proc.$(OBJEXT)
@ -651,6 +650,7 @@ drive_data_scanner_SOURCES = \
../src/data_scanner.cc \ ../src/data_scanner.cc \
drive_data_scanner.cc drive_data_scanner.cc
drive_data_scanner_LDADD = -lcrypto
drive_view_colors_SOURCES = \ drive_view_colors_SOURCES = \
../src/view_curses.cc \ ../src/view_curses.cc \
drive_view_colors.cc drive_view_colors.cc
@ -690,6 +690,8 @@ dist_noinst_DATA = \
datafile_simple.3 \ datafile_simple.3 \
datafile_simple.4 \ datafile_simple.4 \
datafile_simple.5 \ datafile_simple.5 \
datafile_simple.6 \
datafile_simple.7 \
listview_output.0 \ listview_output.0 \
listview_output.1 \ listview_output.1 \
listview_output.2 \ listview_output.2 \

@ -1,17 +1,14 @@
a=1 b=2 c=3,4 a=1 b=2 c=3,4
key 8:9 ^ key 0:1 ^ a
sep 9:10 ^ num 2:3 ^ 1
num 10:11 ^ val 2:3 ^ 1
num 12:13 ^ pair 0:3 ^-^ a=1
row 10:13 ^-^ key 4:5 ^ b
pair 8:13 ^---^ num 6:7 ^ 2
key 4:5 ^ val 6:7 ^ 2
sep 5:6 ^ pair 4:7 ^-^ b=2
num 6:7 ^ key 8:9 ^ c
row 6:7 ^ num 10:11 ^ 3
pair 4:7 ^-^ num 12:13 ^ 4
key 0:1 ^ val 10:13 ^-^ 3,4
sep 1:2 ^ pair 8:13 ^---^ c=3,4
num 2:3 ^
row 2:3 ^
pair 0:3 ^-^

@ -1,7 +1,7 @@
current speed: 38 mph current speed: 38 mph
key 0:13 ^-----------^ key 0:0
sep 13:14 ^ key 0:13 ^-----------^ current speed
num 15:17 ^^ pair 0:13 ^-----------^ current speed
word 18:21 ^-^ key 15:15 ^
row 15:21 ^----^ num 15:17 ^^ 38
pair 0:17 ^---------------^ pair 15:17 ^^ 38

@ -1,9 +1,16 @@
1,2,3,4,five,six,7 1,2,3,4,five,six,7
num 0:1 ^ key 0:0
num 2:3 ^ num 0:1 ^ 1
num 4:5 ^ pair 0:1 ^ 1
num 6:7 ^ key 2:2 ^
word 8:12 ^--^ num 2:3 ^ 2
word 13:16 ^-^ pair 2:3 ^ 2
num 17:18 ^ key 4:4 ^
row 0:18 ^----------------^ num 4:5 ^ 3
pair 4:5 ^ 3
key 6:6 ^
num 6:7 ^ 4
pair 6:7 ^ 4
key 17:17 ^
num 17:18 ^ 7
pair 17:18 ^ 7

@ -1,9 +1,16 @@
1 2 3 4 five six 7 1 2 3 4 five six 7
num 0:1 ^ key 0:0
num 2:3 ^ num 0:1 ^ 1
num 4:5 ^ pair 0:1 ^ 1
num 6:7 ^ key 2:2 ^
word 8:12 ^--^ num 2:3 ^ 2
word 13:16 ^-^ pair 2:3 ^ 2
num 17:18 ^ key 4:4 ^
row 0:18 ^----------------^ num 4:5 ^ 3
pair 4:5 ^ 3
key 6:6 ^
num 6:7 ^ 4
pair 6:7 ^ 4
key 17:17 ^
num 17:18 ^ 7
pair 17:18 ^ 7

@ -1,6 +1,5 @@
the-value: "Hello, World!" the-value: "Hello, World!"
key 0:9 ^-------^ key 0:9 ^-------^ the-value
sep 9:10 ^ quot 12:25 ^-----------^ Hello, World!
quot 12:25 ^-----------^ val 12:25 ^-----------^ Hello, World!
row 12:25 ^-----------^ pair 0:25 ^-----------------------^ the-value: "Hello, World!
pair 0:25 ^-----------------------^

@ -1,6 +1,5 @@
this is a url: http://www.example.com/foo-bar this is a url: http://www.example.com/foo-bar
key 0:13 ^-----------^ key 0:13 ^-----------^ this is a url
sep 13:14 ^ url 15:45 ^----------------------------^ http://www.example.com/foo-bar
url 15:45 ^----------------------------^ val 15:45 ^----------------------------^ http://www.example.com/foo-bar
row 15:45 ^----------------------------^ pair 0:45 ^-------------------------------------------^ this is a url: http://www.example.com/foo-bar
pair 0:45 ^-------------------------------------------^

@ -1,11 +1,9 @@
qualified:name: foo=1 bar=2 qualified:name: foo=1 bar=2
key 22:25 ^-^ key 16:19 ^-^ foo
sep 25:26 ^ num 20:21 ^ 1
num 26:27 ^ val 20:21 ^ 1
row 26:27 ^ pair 16:21 ^---^ foo=1
pair 22:27 ^---^ key 22:25 ^-^ bar
key 16:19 ^-^ num 26:27 ^ 2
sep 19:20 ^ val 26:27 ^ 2
num 20:21 ^ pair 22:27 ^---^ bar=2
row 20:21 ^
pair 16:21 ^---^

@ -0,0 +1,12 @@
func(arg1="a", arg2="b")
key 5:5 ^
key 5:9 ^--^ arg1
quot 11:12 ^ a
val 11:12 ^ a
pair 5:12 ^-----^ arg1="a
key 15:19 ^--^ arg2
quot 21:22 ^ b
val 21:22 ^ b
pair 15:22 ^-----^ arg2="b
grp 5:22 ^---------------^ arg1="a", arg2="b
pair 5:22 ^---------------^ arg1="a", arg2="b

@ -0,0 +1,31 @@
Succeeded authorizing right 'system.privilege.taskport.debug' by client '/usr/libexec/taskgated' [76339] for authorization created by '/usr/libexec/taskgated' [77395] (100003,1)
key 29:29 ^
quot 29:60 ^-----------------------------^ system.privilege.taskport.debug
pair 29:60 ^-----------------------------^ system.privilege.taskport.debug
key 73:73 ^
quot 73:95 ^--------------------^ /usr/libexec/taskgated
pair 73:95 ^--------------------^ /usr/libexec/taskgated
key 98:98 ^
key 98:98 ^
num 98:103 ^---^ 76339
pair 98:103 ^---^ 76339
grp 98:103 ^---^ 76339
pair 98:103 ^---^ 76339
key 135:135 ^
quot 135:157 ^--------------------^ /usr/libexec/taskgated
pair 135:157 ^--------------------^ /usr/libexec/taskgated
key 160:160 ^
key 160:160 ^
num 160:165 ^---^ 77395
pair 160:165 ^---^ 77395
grp 160:165 ^---^ 77395
pair 160:165 ^---^ 77395
key 168:168 ^
key 168:168 ^
num 168:174 ^----^ 100003
pair 168:174 ^----^ 100003
key 175:175 ^
num 175:176 ^ 1
pair 175:176 ^ 1
grp 168:176 ^------^ 100003,1
pair 168:176 ^------^ 100003,1

@ -1,32 +1,20 @@
Nov 3 09:47:02 veridian sudo: timstack : TTY=pts/6 ; PWD=/auto/wstimstack/rpms/lbuild/test ; USER=root ; COMMAND=/usr/bin/tail /var/log/messages timstack : TTY=pts/6 ; PWD=/auto/wstimstack/rpms/lbuild/test ; USER=root ; COMMAND=/usr/bin/tail /var/log/messages
key 106:113 ^-----^ key 11:14 ^-^ TTY
sep 113:114 ^ sym 15:18 ^-^ pts
path 114:127 ^-----------^ num 19:20 ^ 6
path 128:145 ^---------------^ val 15:20 ^---^ pts/6
row 114:145 ^-----------------------------^ pair 11:20 ^-------^ TTY=pts/6
pair 106:127 ^-------------------^ key 23:26 ^-^ PWD
key 94:98 ^--^ path 27:60 ^-------------------------------^ /auto/wstimstack/rpms/lbuild/test
sep 98:99 ^ val 27:60 ^-------------------------------^ /auto/wstimstack/rpms/lbuild/test
word 99:103 ^--^ pair 23:60 ^-----------------------------------^ PWD=/auto/wstimstack/rpms/lbuild/test
row 99:103 ^--^ key 63:67 ^--^ USER
pair 94:103 ^-------^ word 68:72 ^--^ root
key 54:57 ^-^ val 68:72 ^--^ root
sep 57:58 ^ pair 63:72 ^-------^ USER=root
path 58:91 ^-------------------------------^ key 75:82 ^-----^ COMMAND
row 58:91 ^-------------------------------^ path 83:96 ^-----------^ /usr/bin/tail
pair 54:91 ^-----------------------------------^ wspc 96:97 ^
key 42:45 ^-^ path 97:114 ^---------------^ /var/log/messages
sep 45:46 ^ val 83:114 ^-----------------------------^ /usr/bin/tail /var/log/messages
word 46:49 ^-^ pair 75:114 ^-------------------------------------^ COMMAND=/usr/bin/tail /var/log/messages
path 49:51 ^^
row 46:51 ^---^
pair 42:51 ^-------^
key 16:29 ^-----------^
sep 29:30 ^
word 31:39 ^------^
row 31:39 ^------^
pair 16:39 ^---------------------^
word 0:3 ^-^
num 5:6 ^
time 7:15 ^------^
date 0:15 ^-------------^

@ -1,33 +1,21 @@
Jun 18 16:13:52 Tim-Stacks-iMac Safari[81045]: INSERT-HANG-DETECTED: Tx time:3.093364, # of Inserts: 89, # of bytes written: 465365, Did shrink: NO INSERT-HANG-DETECTED: Tx time:3.093364, # of Inserts: 89, # of bytes written: 465365, Did shrink: NO
key 137:143 ^----^ key 0:20 ^------------------^ INSERT-HANG-DETECTED
sep 143:144 ^ word 22:24 ^^ Tx
word 145:147 ^^ val 22:24 ^^ Tx
row 145:147 ^^ pair 0:24 ^----------------------^ INSERT-HANG-DETECTED: Tx
pair 137:147 ^--------^ key 25:29 ^--^ time
key 107:123 ^--------------^ num 30:38 ^------^ 3.093364
sep 123:124 ^ val 30:38 ^------^ 3.093364
num 125:131 ^----^ pair 25:38 ^-----------^ time:3.093364
word 133:136 ^-^ key 40:52 ^----------^ # of Inserts
row 125:136 ^---------^ num 54:56 ^^ 89
pair 107:136 ^---------------------------^ val 54:56 ^^ 89
key 89:99 ^--------^ pair 40:56 ^--------------^ # of Inserts: 89
sep 99:100 ^ key 58:76 ^----------------^ # of bytes written
num 101:103 ^^ num 78:84 ^----^ 465365
word 105:106 ^ val 78:84 ^----^ 465365
row 101:106 ^---^ pair 58:84 ^------------------------^ # of bytes written: 465365
pair 89:106 ^---------------^ key 86:96 ^--------^ Did shrink
key 72:76 ^--^ sym 98:100 ^^ NO
sep 76:77 ^ val 98:100 ^^ NO
num 77:85 ^------^ pair 86:100 ^------------^ Did shrink: NO
word 87:88 ^
row 77:88 ^---------^
pair 72:88 ^--------------^
key 47:67 ^------------------^
sep 67:68 ^
word 69:71 ^^
row 69:71 ^^
pair 47:71 ^----------------------^
word 0:3 ^-^
num 4:6 ^^
time 7:15 ^------^
date 0:15 ^-------------^

@ -105,12 +105,10 @@ int main(int argc, char *argv[])
} }
data_scanner ds(line.substr(13)); data_scanner ds(line.substr(13));
data_token_t token;
data_parser dp(&ds); data_parser dp(&ds);
dp.parse(); dp.parse();
dp.print(out); dp.print(out, dp.dp_pairs);
fclose(out); fclose(out);
sprintf(cmd, "diff -u %s %s", argv[0], TMP_NAME); sprintf(cmd, "diff -u %s %s", argv[0], TMP_NAME);

Loading…
Cancel
Save