[highlights] allow the highlight color to be specified in log formats

Defect Number:
    Reviewed By:
   Testing Done:

NEWS

@@ -6,6 +6,8 @@ lnav v0.8.3:
      * Added an fstat() table-valued function for querying the local
        filesystem.
      * Added readlink() and realpath() SQL functions.
+     * Highlights specified in log formats can now specify the colors to use
+       for the highlighted parts of the log message.
      Fixes:
      * The help view now includes all the command-help that would pop up as
        you entered commands and SQL queries.

README

@@ -60,6 +60,14 @@ The lnav mailing list can be reached at:
   lnav@googlegroups.com
 
 
+ACKNOWLEDGEMENTS
+----------------
+
+The xterm color database was copied from:
+
+  http://jonasjacek.github.io/colors/
+
+
 SEE ALSO
 --------
 

docs/source/formats.rst

@@ -206,8 +206,22 @@ fields:
     :level: The expected error level.  An error will be raised if this level
       does not match the level parsed by lnav for this sample message.
 
-  :highlights: A list of strings that are regular expressions to be highlighted
-    in log messages.
+  :highlights: This object contains the definitions for patterns to be
+      highlighted in a log message.  Each entry should have a name and a
+      definition with the following fields:
+    :pattern: The regular expression to match in the log message body.
+    :color: The foreground color to use when highlighting the part of the
+      message that matched the pattern.  If no color is specified, one will be
+      picked automatically.  Colors can be specified using hexadecimal notation
+      by starting with a hash (e.g. #aabbcc) or using a color name as found
+      at http://jonasjacek.github.io/colors/.
+    :background-color: The background color to use when highlighting the part
+      of the message that matched the pattern.  If no background color is
+      specified, black will be used.  The background color is only considered
+      if a foreground color is specified.
+    :underline: If true, underline the part of the message that matched the
+      pattern.
+    :blink: If true, blink the part of the message that matched the pattern.
 
 Example format::
 

src/CMakeLists.txt

@@ -98,6 +98,7 @@ set(diag_STAT_SRCS
     concise_index.hh
     column_namer.hh
     curl_looper.hh
+    default-log-formats-json.hh
     doc_status_source.hh
     elem_to_json.hh
     field_overlay_source.hh
@@ -145,6 +146,7 @@ set(diag_STAT_SRCS
     url_loader.hh
     views_vtab.hh
     vtab_module.hh
+    xterm-palette.hh
 
     mapbox/recursive_wrapper.hpp
     mapbox/variant.hpp

src/Makefile.am

@@ -236,6 +236,7 @@ noinst_HEADERS = \
 	log_vtab_impl.hh \
 	log_format_impls.cc \
 	xterm_mouse.hh \
+	xterm-palette.hh \
 	yajlpp.hh \
 	spookyhash/SpookyV2.h \
 	filesystem/fwd.h \
@@ -328,6 +329,7 @@ libdiag_a_SOURCES = \
 	vtab_module.cc \
 	log_vtab_impl.cc \
 	xterm_mouse.cc \
+	xterm-palette.c \
 	yajlpp.cc \
 	spookyhash/SpookyV2.cpp
 
@@ -359,7 +361,8 @@ TEXT2C_FILES = \
 	keymap-default.o \
 	lnav-pop-view.o \
 	partition-by-boot.o \
-	search-for.o
+	search-for.o \
+	xterm-palette.o
 
 lnav_SOURCES = lnav.cc $(HELP_SRC)
 lnav_LDADD = \
@@ -384,7 +387,8 @@ DISTCLEANFILES = \
 	init-sql.c \
 	default-log-formats-json.c \
 	default-config-json.c \
-    time_fmts.cc
+    time_fmts.cc \
+    xterm-palette.c
 
 uncrusty:
 	(cd $(srcdir) && uncrustify -c ../lnav.cfg --replace $(SOURCES) \

src/data_parser.hh

@@ -42,7 +42,6 @@
 
 #include "lnav_log.hh"
 #include "lnav_util.hh"
-#include "yajlpp.hh"
 #include "pcrepp.hh"
 #include "byte_array.hh"
 #include "data_scanner.hh"

src/elem_to_json.cc

@@ -30,6 +30,7 @@
 #include "config.h"
 
 #include "elem_to_json.hh"
+#include "yajlpp.hh"
 
 using namespace std;
 

src/fs-extension-functions.cc

@@ -36,6 +36,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <stdint.h>
+#include <unistd.h>
 
 #include <string>
 #include <stddef.h>

src/grep_proc.cc

@@ -185,7 +185,6 @@ void grep_proc::child_loop(void)
                 pcre_context_static<128> pc;
                 pcre_input pi(line_value);
 
-                log_debug("wtf %s", line_value.c_str());
                 while (this->gp_pcre.match(pc, pi)) {
                     pcre_context::iterator   pc_iter;
                     pcre_context::capture_t *m;

src/highlighter.hh

@@ -32,6 +32,7 @@
 #ifndef __highlighter_hh
 #define __highlighter_hh
 
+#include "optional.hpp"
 #include "pcrepp.hh"
 #include "text_format.hh"
 #include "view_curses.hh"
@@ -133,6 +134,13 @@ struct highlighter {
         return *this;
     };
 
+    highlighter &with_color(const rgb_color &fg, const rgb_color &bg) {
+        this->h_fg = fg;
+        this->h_bg = bg;
+
+        return *this;
+    };
+
     int get_attrs() const
     {
         ensure(this->h_attrs != -1);
@@ -141,6 +149,8 @@ struct highlighter {
     };
 
     std::string h_pattern;
+    rgb_color h_fg;
+    rgb_color h_bg;
     pcre *                           h_code;
     pcre_extra *                     h_code_extra;
     int h_attrs;

src/intern_string.hh

@@ -42,6 +42,11 @@ struct string_fragment {
         : sf_string(str), sf_begin(begin), sf_end(end == -1 ? strlen(str) : end) {
     };
 
+    string_fragment(const std::string &str)
+        : sf_string(str.c_str()), sf_begin(0), sf_end(str.length()) {
+
+    }
+
     bool is_valid() const {
         return this->sf_begin != -1;
     };
@@ -91,6 +96,10 @@ struct string_fragment {
         return buf;
     };
 
+    std::string to_string() const {
+        return std::string(this->data(), this->length());
+    }
+
     void clear() {
         this->sf_begin = 0;
         this->sf_end = 0;

src/json_op.hh

@@ -37,7 +37,7 @@
 #include <string>
 
 #include "json_ptr.hh"
-#include "yajlpp.hh"
+#include "yajl/api/yajl_parse.h"
 
 class json_op {
     static int handle_null(void *ctx);

src/lnav.cc

@@ -2144,7 +2144,8 @@ static void looper(void)
         define_key("\033Od", KEY_BEG);
         define_key("\033Oc", KEY_END);
 
-        view_colors::singleton().init();
+        view_colors &vc = view_colors::singleton();
+        vc.init();
 
         {
             setup_highlights(lnav_data.ld_views[LNV_LOG].get_highlights());
@@ -2155,7 +2156,11 @@ static void looper(void)
 
             for (auto format : log_format::get_root_formats()) {
                 for (auto &hl : format->lf_highlighters) {
-                    hl.with_attrs(view_colors::singleton().attrs_for_ident(hl.h_pattern));
+                    if (hl.h_fg.empty()) {
+                        hl.with_attrs(hl.h_attrs | vc.attrs_for_ident(hl.h_pattern));
+                    } else {
+                        hl.with_attrs(hl.h_attrs | vc.ensure_color_pair(hl.h_fg, hl.h_bg));
+                    }
 
                     lnav_data.ld_views[LNV_LOG].get_highlights()[
                         "$" + format->get_name().to_string() + "-" + hl.h_pattern] = hl;

src/lnav_config.hh

@@ -39,8 +39,6 @@
 #include <vector>
 #include <unordered_map>
 
-#include "yajlpp.hh"
-
 class lnav_config_listener {
 public:
     lnav_config_listener() {

src/log_format.cc

@@ -1725,10 +1725,42 @@ void external_log_format::build(std::vector<std::string> &errors) {
         }
     }
 
-    for (int lpc = 0; lpc < this->elf_highlighter_patterns.size(); lpc++) {
-        const string &pattern = this->elf_highlighter_patterns[lpc];
+    for (auto &hd_pair : this->elf_highlighter_patterns) {
+        external_log_format::highlighter_def &hd = hd_pair.second;
+        const std::string &pattern = hd.hd_pattern;
+        std::string errmsg;
         const char *errptr;
-        int eoff;
+        rgb_color fg, bg;
+        int eoff, attrs = 0;
+
+        if (!hd.hd_color.empty()) {
+            if (!rgb_color::from_str(hd.hd_color, fg, errmsg)) {
+                errors.push_back("error:"
+                                 + this->elf_name.to_string()
+                                 + ":highlighters/"
+                                 + hd_pair.first.to_string()
+                                 + "/color:"
+                                 + errmsg);
+            }
+        }
+
+        if (!hd.hd_background_color.empty()) {
+            if (!rgb_color::from_str(hd.hd_background_color, bg, errmsg)) {
+                errors.push_back("error:"
+                                 + this->elf_name.to_string()
+                                 + ":highlighters/"
+                                 + hd_pair.first.to_string()
+                                 + "/color:"
+                                 + errmsg);
+            }
+        }
+
+        if (hd.hd_underline) {
+            attrs |= A_UNDERLINE;
+        }
+        if (hd.hd_blink) {
+            attrs |= A_BLINK;
+        }
 
         pcre *code = pcre_compile(pattern.c_str(),
                                   0,
@@ -1739,29 +1771,32 @@ void external_log_format::build(std::vector<std::string> &errors) {
         if (code == nullptr) {
             errors.push_back("error:"
                              + this->elf_name.to_string()
-                             + ":highlighters["
-                             + to_string(lpc)
-                             + "]:"
+                             + ":highlighters/"
+                             + hd_pair.first.to_string()
+                             + ":"
                              + string(errptr));
             errors.push_back("error:"
                              + this->elf_name.to_string()
-                             + ":highlighters["
-                             + to_string(lpc)
-                             + "]:"
+                             + ":highlighters/"
+                             + hd_pair.first.to_string()
+                             + ":"
                              + pattern);
             errors.push_back("error:"
                              + this->elf_name.to_string()
-                             + ":highlighters["
-                             + to_string(lpc)
-                             + "]:"
+                             + ":highlighters/"
+                             + hd_pair.first.to_string()
+                             + ":"
                              + string(eoff, ' ')
                              + "^");
         } else {
             this->lf_highlighters.emplace_back(code);
             this->lf_highlighters.back()
                 .with_pattern(pattern)
-                .with_format_name(this->elf_name);
+                .with_format_name(this->elf_name)
+                .with_color(fg, bg)
+                .with_attrs(attrs);
         }
+
     }
 }
 

src/log_format.hh

@@ -1145,6 +1145,17 @@ public:
         const intern_string_t jfc_field_name;
     };
 
+    struct highlighter_def {
+        highlighter_def() : hd_underline(false), hd_blink(false) {
+        }
+
+        std::string hd_pattern;
+        std::string hd_color;
+        std::string hd_background_color;
+        bool hd_underline;
+        bool hd_blink;
+    };
+
     long value_line_count(const intern_string_t ist,
                           bool top_level,
                           const unsigned char *str = NULL,
@@ -1242,7 +1253,7 @@ public:
     bool elf_has_module_format;
     bool elf_builtin_format;
     std::vector<std::pair<intern_string_t, std::string> > elf_search_tables;
-    std::vector<std::string> elf_highlighter_patterns;
+    std::map<const intern_string_t, highlighter_def> elf_highlighter_patterns;
 
     enum elf_type_t {
         ELF_TYPE_TEXT,

src/log_format_loader.cc

@@ -498,6 +498,35 @@ static struct json_path_handler value_def_handlers[] = {
     json_path_handler()
 };
 
+static struct json_path_handler highlighter_def_handlers[] = {
+    json_path_handler("pattern")
+        .with_synopsis("<regex>")
+        .with_description("A regular expression to highlight in logs of this format.")
+        .for_field(&nullobj<external_log_format::highlighter_def>()->hd_pattern),
+
+    json_path_handler("color")
+        .with_synopsis("#<hex>|<name>")
+        .with_description("The color to use when highlighting this pattern.")
+        .for_field(&nullobj<external_log_format::highlighter_def>()->hd_color),
+
+    json_path_handler("background-color")
+        .with_synopsis("#<hex>|<name>")
+        .with_description("The background color to use when highlighting this pattern.")
+        .for_field(&nullobj<external_log_format::highlighter_def>()->hd_background_color),
+
+    json_path_handler("underline")
+        .with_synopsis("<enabled>")
+        .with_description("Highlight this pattern with an underline.")
+        .for_field(&nullobj<external_log_format::highlighter_def>()->hd_underline),
+
+    json_path_handler("blink")
+        .with_synopsis("<enabled>")
+        .with_description("Highlight this pattern by blinking.")
+        .for_field(&nullobj<external_log_format::highlighter_def>()->hd_blink),
+
+    json_path_handler()
+};
+
 static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = {
     make_pair(logline::level_names[logline::LEVEL_TRACE], logline::LEVEL_TRACE),
     make_pair(logline::level_names[logline::LEVEL_DEBUG5], logline::LEVEL_DEBUG5),
@@ -585,10 +614,11 @@ struct json_path_handler format_handlers[] = {
         .with_synopsis("<regex>")
         .with_description("The regular expression for this search table."),
 
-    json_path_handler("highlights#")
-        .with_synopsis("<regex>")
-        .with_description("A regular expression to highlight in logs of this format.")
-        .for_field(&nullobj<external_log_format>()->elf_highlighter_patterns),
+    json_path_handler("highlights/(?<highlight_name>[^/]+)/")
+        .with_obj_provider<external_log_format::highlighter_def, external_log_format>([](const yajlpp_provider_context &ypc, external_log_format *root) {
+            return &(root->elf_highlighter_patterns[ypc.get_substr_i(0)]);
+        })
+        .with_children(highlighter_def_handlers),
 
     json_path_handler("file-type")
         .with_synopsis("The type of file that contains the log messages")

src/view_curses.cc

@@ -32,17 +32,132 @@
 #include "config.h"
 
 #include <string>
-#include <algorithm>
 
 #include "auto_mem.hh"
 #include "lnav_log.hh"
 #include "view_curses.hh"
 #include "ansi_scrubber.hh"
 #include "lnav_config.hh"
-#include "attr_line.hh"
+#include "yajlpp.hh"
+#include "xterm-palette.hh"
 
 using namespace std;
 
+struct xterm_color {
+    short xc_id;
+    string xc_name;
+    rgb_color xc_color;
+    lab_color xc_lab_color;
+};
+
+static struct json_path_handler xterm_color_handler[] = {
+    json_path_handler("colorId")
+        .for_field(&nullobj<xterm_color>()->xc_id),
+    json_path_handler("name")
+        .for_field(&nullobj<xterm_color>()->xc_name),
+    json_path_handler("rgb/r")
+        .for_field(&nullobj<xterm_color>()->xc_color.rc_r),
+    json_path_handler("rgb/g")
+        .for_field(&nullobj<xterm_color>()->xc_color.rc_g),
+    json_path_handler("rgb/b")
+        .for_field(&nullobj<xterm_color>()->xc_color.rc_b),
+
+    json_path_handler()
+};
+
+static struct json_path_handler root_color_handler[] = {
+    json_path_handler("#/")
+        .with_obj_provider<xterm_color, vector<xterm_color>>(
+            [](const yajlpp_provider_context &ypc, vector<xterm_color> *palette) {
+                palette->resize(ypc.ypc_index + 1);
+                return &((*palette)[ypc.ypc_index]);
+            })
+        .with_children(xterm_color_handler),
+
+    json_path_handler()
+};
+
+static struct _xterm_colors {
+    _xterm_colors() {
+        yajlpp_parse_context ypc_xterm("xterm-palette.json", root_color_handler);
+        yajl_handle handle;
+
+        handle = yajl_alloc(&ypc_xterm.ypc_callbacks, NULL, &ypc_xterm);
+        ypc_xterm
+            .with_ignore_unused(true)
+            .with_obj(this->xc_palette)
+            .with_handle(handle);
+        ypc_xterm.parse((const unsigned char *) xterm_palette_json,
+                        strlen(xterm_palette_json));
+        ypc_xterm.complete_parse();
+        yajl_free(handle);
+
+        for (auto &xc : this->xc_palette) {
+            xc.xc_lab_color = lab_color(xc.xc_color);
+        }
+    };
+
+    short match_color(const lab_color &to_match) {
+        double lowest = 1000.0;
+        short lowest_id = -1;
+
+        for (auto &xc : this->xc_palette) {
+            double xc_delta = xc.xc_lab_color.deltaE(to_match);
+
+            if (lowest_id == -1) {
+                lowest = xc_delta;
+                lowest_id = xc.xc_id;
+                continue;
+            }
+
+            if (xc_delta < lowest) {
+                lowest = xc_delta;
+                lowest_id = xc.xc_id;
+            }
+        }
+
+        return lowest_id;
+    };
+
+    vector<xterm_color> xc_palette;
+} xterm_colors;
+
+bool rgb_color::from_str(const string_fragment &color,
+                         rgb_color &rgb_out,
+                         std::string &errmsg)
+{
+    if (color[0] == '#') {
+        switch (color.length()) {
+            case 4:
+                if (sscanf(color.data(), "#%1hx%1hx%1hx",
+                           &rgb_out.rc_r, &rgb_out.rc_g, &rgb_out.rc_b) == 3) {
+                    rgb_out.rc_r |= rgb_out.rc_r << 4;
+                    rgb_out.rc_g |= rgb_out.rc_g << 4;
+                    rgb_out.rc_b |= rgb_out.rc_b << 4;
+                    return true;
+                }
+                break;
+            case 7:
+                if (sscanf(color.data(), "#%2hx%2hx%2hx",
+                           &rgb_out.rc_r, &rgb_out.rc_g, &rgb_out.rc_b) == 3) {
+                    return true;
+                }
+                break;
+        }
+        errmsg = "Could not parse color: " + color.to_string();
+        return false;
+    }
+
+    for (const auto &xc : xterm_colors.xc_palette) {
+        if (color == xc.xc_name) {
+            rgb_out = xc.xc_color;
+            return true;
+        }
+    }
+
+    return false;
+}
+
 string_attr_type view_curses::VC_STYLE("style");
 string_attr_type view_curses::VC_GRAPHIC("graphic");
 
@@ -440,7 +555,7 @@ view_colors &view_colors::singleton(void)
     return s_vc;
 }
 
-view_colors::view_colors()
+view_colors::view_colors() : vc_color_pair_end(0)
 {
 }
 
@@ -566,4 +681,14 @@ void view_colors::init_roles(int color_pair_base)
     this->vc_role_colors[VCR_LOW_THRESHOLD] = attr_for_colors(color_pair_base, COLOR_BLACK, COLOR_GREEN);
     this->vc_role_colors[VCR_MED_THRESHOLD] = attr_for_colors(color_pair_base, COLOR_BLACK, COLOR_YELLOW);
     this->vc_role_colors[VCR_HIGH_THRESHOLD] = attr_for_colors(color_pair_base, COLOR_BLACK, COLOR_RED);
+
+    this->vc_color_pair_end = color_pair_base + 1;
+}
+
+int view_colors::ensure_color_pair(const rgb_color &rgb_fg, const rgb_color &rgb_bg)
+{
+    return attr_for_colors(
+        this->vc_color_pair_end,
+        xterm_colors.match_color(rgb_fg),
+        rgb_bg.empty() ? (short) COLOR_BLACK : xterm_colors.match_color(rgb_bg));
 }

src/view_curses.hh

@@ -35,6 +35,7 @@
 #include "config.h"
 
 #include <zlib.h>
+#include <math.h>
 #include <stdint.h>
 #include <limits.h>
 #include <signal.h>
@@ -286,6 +287,80 @@ private:
     void (*va_invoker)(void *functor, _Sender *sender);
 };
 
+struct rgb_color {
+    static bool from_str(const string_fragment &color,
+                         rgb_color &rgb_out,
+                         std::string &errmsg);
+
+    rgb_color(short r = -1, short g = -1, short b = -1)
+        : rc_r(r), rc_g(g), rc_b(b) {
+    }
+
+    bool empty() const {
+        return this->rc_r == -1 && this->rc_g == -1 && this->rc_b == -1;
+    }
+
+    short rc_r;
+    short rc_g;
+    short rc_b;
+};
+
+struct lab_color {
+    lab_color() : lc_l(0), lc_a(0), lc_b(0) {
+    };
+
+    lab_color(const rgb_color &rgb) {
+        double r = rgb.rc_r / 255.0,
+            g = rgb.rc_g / 255.0,
+            b = rgb.rc_b / 255.0,
+            x, y, z;
+
+        r = (r > 0.04045) ? pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
+        g = (g > 0.04045) ? pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
+        b = (b > 0.04045) ? pow((b + 0.055) / 1.055, 2.4) : b / 12.92;
+
+        x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
+        y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
+        z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;
+
+        x = (x > 0.008856) ? pow(x, 1.0/3.0) : (7.787 * x) + 16.0/116.0;
+        y = (y > 0.008856) ? pow(y, 1.0/3.0) : (7.787 * y) + 16.0/116.0;
+        z = (z > 0.008856) ? pow(z, 1.0/3.0) : (7.787 * z) + 16.0/116.0;
+
+        this->lc_l = (116.0 * y) - 16;
+        this->lc_a = 500.0 * (x - y);
+        this->lc_b = 200.0 * (y - z);
+    }
+
+    double deltaE(const lab_color &other) const {
+        double deltaL = this->lc_l - other.lc_l;
+        double deltaA = this->lc_a - other.lc_a;
+        double deltaB = this->lc_b - other.lc_b;
+        double c1 = sqrt(this->lc_a * this->lc_a + this->lc_b * this->lc_b);
+        double c2 = sqrt(other.lc_a * other.lc_a + other.lc_b * other.lc_b);
+        double deltaC = c1 - c2;
+        double deltaH = deltaA * deltaA + deltaB * deltaB - deltaC * deltaC;
+        deltaH = deltaH < 0.0 ? 0.0 : sqrt(deltaH);
+        double sc = 1.0 + 0.045 * c1;
+        double sh = 1.0 + 0.015 * c1;
+        double deltaLKlsl = deltaL / (1.0);
+        double deltaCkcsc = deltaC / (sc);
+        double deltaHkhsh = deltaH / (sh);
+        double i = deltaLKlsl * deltaLKlsl + deltaCkcsc * deltaCkcsc + deltaHkhsh * deltaHkhsh;
+        return i < 0.0 ? 0.0 : sqrt(i);
+    }
+
+    void operator=(const lab_color &other) {
+        this->lc_l = other.lc_l;
+        this->lc_a = other.lc_a;
+        this->lc_b = other.lc_b;
+    };
+
+    double lc_l;
+    double lc_a;
+    double lc_b;
+};
+
 /**
  * Singleton used to manage the colorspace.
  */
@@ -389,6 +464,8 @@ public:
         return this->attrs_for_ident(str.c_str(), str.length());
     };
 
+    int ensure_color_pair(const rgb_color &fg, const rgb_color &bg);
+
     static inline int ansi_color_pair_index(int fg, int bg)
     {
         return VC_ANSI_START + ((fg * 8) + bg);
@@ -415,6 +492,8 @@ private:
     int vc_role_colors[VCR__MAX];
     /** Map of role IDs to reverse-video attribute values. */
     int vc_role_reverse_colors[VCR__MAX];
+    int vc_color_pair_end;
+
 };
 
 enum mouse_button_t {

src/xterm-palette.hh

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2017, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @file xterm-palette.hh
+ */
+
+#ifndef __xterm_palette_hh
+#define __xterm_palette_hh
+
+extern "C" {
+/**
+ * The help message text.  The value for this comes from the "init.sql" file,
+ * which gets linked into the executable by the Makefile.
+ */
+extern const char xterm_palette_json[];
+}
+#endif

src/yajlpp.cc

@@ -46,6 +46,14 @@ static T &resolve_root(yajlpp_parse_context *ypc)
 
     ptrdiff_t offset = (char *) jph->jph_simple_offset - (char *) NULL;
     char *retval = (char *) ypc->ypc_obj_stack.top();
+    char *ptr = retval + offset;
+
+    if (jph->jph_optional_wrapper) {
+        nonstd::optional<T> &wrapper = *(nonstd::optional<T> *) ptr;
+
+        wrapper = nonstd::make_optional(T());
+        return wrapper.value();
+    }
 
     return *((T *) (retval + offset));
 }
@@ -307,15 +315,21 @@ void yajlpp_validator_for_double(yajlpp_parse_context &ypc,
     }
 }
 
+template<typename T>
 int yajlpp_static_number(yajlpp_parse_context *ypc, long long num)
 {
-    long long &field_ptr = resolve_root<long long>(ypc);
+    T &field_ptr = resolve_root<T>(ypc);
 
     field_ptr = num;
 
     return 1;
 }
 
+template
+int yajlpp_static_number<long long>(yajlpp_parse_context *ypc, long long num);
+template
+int yajlpp_static_number<short>(yajlpp_parse_context *ypc, long long num);
+
 int yajlpp_static_decimal(yajlpp_parse_context *ypc, double num)
 {
     double &field_ptr = resolve_root<double>(ypc);

src/yajlpp.hh

@@ -43,6 +43,7 @@
 #include <string>
 #include <algorithm>
 
+#include "optional.hpp"
 #include "pcrepp.hh"
 #include "json_ptr.hh"
 #include "intern_string.hh"
@@ -121,7 +122,8 @@ struct json_path_handler_base {
               jph_min_length(0),
               jph_max_length(INT_MAX),
               jph_enum_values(NULL),
-              jph_min_value(LLONG_MIN)
+              jph_min_value(LLONG_MIN),
+              jph_optional_wrapper(false)
     {
         memset(&this->jph_callbacks, 0, sizeof(this->jph_callbacks));
     };
@@ -149,6 +151,7 @@ struct json_path_handler_base {
     size_t         jph_max_length;
     const enum_value_t  *jph_enum_values;
     long long      jph_min_value;
+    bool           jph_optional_wrapper;
 };
 
 int yajlpp_static_string(yajlpp_parse_context *, const unsigned char *, size_t);
@@ -172,7 +175,8 @@ void yajlpp_validator_for_int(yajlpp_parse_context &ypc,
 void yajlpp_validator_for_double(yajlpp_parse_context &ypc,
                                  const json_path_handler_base &jph);
 
-int yajlpp_static_number(yajlpp_parse_context *, long long);
+template<typename T>
+int yajlpp_static_number(yajlpp_parse_context *ypc, long long num);
 int yajlpp_static_decimal(yajlpp_parse_context *, double);
 
 int yajlpp_static_bool(yajlpp_parse_context *, int);
@@ -340,8 +344,9 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     };
 
-    json_path_handler &for_field(long long *field) {
-        this->add_cb(yajlpp_static_number);
+    template<typename T, typename T2 = typename std::enable_if<std::is_arithmetic<T>::value>::type>
+    json_path_handler &for_field(T *field) {
+        this->add_cb(yajlpp_static_number<T>);
         this->jph_simple_offset = field;
         this->jph_validator = yajlpp_validator_for_int;
         return *this;
@@ -361,6 +366,16 @@ struct json_path_handler : public json_path_handler_base {
         return *this;
     };
 
+    template<typename T>
+    json_path_handler &for_field(nonstd::optional<T> *field) {
+        T dummy;
+
+        this->for_field(&dummy);
+        this->jph_simple_offset = field;
+        this->jph_optional_wrapper = true;
+        return *this;
+    };
+
     json_path_handler &with_children(json_path_handler *children) {
         require(this->jph_path[strlen(this->jph_path) - 1] == '/');
 
@@ -508,6 +523,11 @@ public:
         return *this;
     }
 
+    yajlpp_parse_context &with_ignore_unused(bool ignore) {
+        this->ypc_ignore_unused = ignore;
+        return *this;
+    }
+
     yajl_status parse(const unsigned char *jsonText, size_t jsonTextLen) {
         this->ypc_json_text = jsonText;
 

test/bad-config/formats/invalid-regex/format.json

@@ -17,8 +17,10 @@
                 "line": "1428634687123; foo"
             }
         ],
-        "highlights": [
-            "abc("
-        ]
+        "highlights": {
+            "foobar": {
+                "pattern": "abc("
+            }
+        }
     }
 }

test/test_format_loader.sh

@@ -42,9 +42,9 @@ error:bad_regex_log.regex[std]:^(?<timestamp>\d+: (?<body>.*)$
 error:bad_regex_log.regex[std]:                               ^
 error:bad_regex_log.level:missing )
 error:bad_regex_log:invalid sample -- 1428634687123; foo
-error:bad_regex_log:highlighters[0]:missing )
-error:bad_regex_log:highlighters[0]:abc(
-error:bad_regex_log:highlighters[0]:    ^
+error:bad_regex_log:highlighters/foobar:missing )
+error:bad_regex_log:highlighters/foobar:abc(
+error:bad_regex_log:highlighters/foobar:    ^
 error:bad_sample_log:invalid sample -- 1428634687123; foo bar
 error:bad_sample_log:partial sample matched -- 1428634687123; foo
 error:  against pattern -- ^(?<timestamp>\d+); (?<body>\w+)$