[pt] rough draft of tail for papertrail

pull/254/head
Timothy Stack 9 years ago
parent 4ceddbe82e
commit 879bbd581e

@ -82,6 +82,7 @@ AC_SEARCH_LIBS(BZ2_bzopen, bz2,
AC_SUBST(BZIP2_SUPPORT)
AC_SEARCH_LIBS(dlopen, dl)
AC_SEARCH_LIBS(backtrace, execinfo)
LIBCURL_CHECK_CONFIG()
# Sometimes, curses depends on these libraries being linked in...
AC_ARG_ENABLE([tinfo],
@ -223,6 +224,7 @@ AS_IF([test $? -eq 0],
AC_DEFINE_UNQUOTED([VCS_PACKAGE_STRING], ["$PACKAGE_STRING"], [VCS package string]))
AM_CONDITIONAL(USE_INCLUDED_YAJL, test $HAVE_LOCAL_YAJL -eq 0)
AM_CONDITIONAL(HAVE_LIBCURL, test x"$LIBCURL" != x"")
AC_CONFIG_HEADERS([src/config.h])
AC_CONFIG_FILES([Makefile])

@ -0,0 +1,272 @@
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) 2006, David Shaw <dshaw@jabberwocky.com>
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://curl.haxx.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
###########################################################################
# LIBCURL_CHECK_CONFIG ([DEFAULT-ACTION], [MINIMUM-VERSION],
# [ACTION-IF-YES], [ACTION-IF-NO])
# ----------------------------------------------------------
# David Shaw <dshaw@jabberwocky.com> May-09-2006
#
# Checks for libcurl. DEFAULT-ACTION is the string yes or no to
# specify whether to default to --with-libcurl or --without-libcurl.
# If not supplied, DEFAULT-ACTION is yes. MINIMUM-VERSION is the
# minimum version of libcurl to accept. Pass the version as a regular
# version number like 7.10.1. If not supplied, any version is
# accepted. ACTION-IF-YES is a list of shell commands to run if
# libcurl was successfully found and passed the various tests.
# ACTION-IF-NO is a list of shell commands that are run otherwise.
# Note that using --without-libcurl does run ACTION-IF-NO.
#
# This macro #defines HAVE_LIBCURL if a working libcurl setup is
# found, and sets @LIBCURL@ and @LIBCURL_CPPFLAGS@ to the necessary
# values. Other useful defines are LIBCURL_FEATURE_xxx where xxx are
# the various features supported by libcurl, and LIBCURL_PROTOCOL_yyy
# where yyy are the various protocols supported by libcurl. Both xxx
# and yyy are capitalized. See the list of AH_TEMPLATEs at the top of
# the macro for the complete list of possible defines. Shell
# variables $libcurl_feature_xxx and $libcurl_protocol_yyy are also
# defined to 'yes' for those features and protocols that were found.
# Note that xxx and yyy keep the same capitalization as in the
# curl-config list (e.g. it's "HTTP" and not "http").
#
# Users may override the detected values by doing something like:
# LIBCURL="-lcurl" LIBCURL_CPPFLAGS="-I/usr/myinclude" ./configure
#
# For the sake of sanity, this macro assumes that any libcurl that is
# found is after version 7.7.2, the first version that included the
# curl-config script. Note that it is very important for people
# packaging binary versions of libcurl to include this script!
# Without curl-config, we can only guess what protocols are available,
# or use curl_version_info to figure it out at runtime.
AC_DEFUN([LIBCURL_CHECK_CONFIG],
[
AH_TEMPLATE([LIBCURL_FEATURE_SSL],[Defined if libcurl supports SSL])
AH_TEMPLATE([LIBCURL_FEATURE_KRB4],[Defined if libcurl supports KRB4])
AH_TEMPLATE([LIBCURL_FEATURE_IPV6],[Defined if libcurl supports IPv6])
AH_TEMPLATE([LIBCURL_FEATURE_LIBZ],[Defined if libcurl supports libz])
AH_TEMPLATE([LIBCURL_FEATURE_ASYNCHDNS],[Defined if libcurl supports AsynchDNS])
AH_TEMPLATE([LIBCURL_FEATURE_IDN],[Defined if libcurl supports IDN])
AH_TEMPLATE([LIBCURL_FEATURE_SSPI],[Defined if libcurl supports SSPI])
AH_TEMPLATE([LIBCURL_FEATURE_NTLM],[Defined if libcurl supports NTLM])
AH_TEMPLATE([LIBCURL_PROTOCOL_HTTP],[Defined if libcurl supports HTTP])
AH_TEMPLATE([LIBCURL_PROTOCOL_HTTPS],[Defined if libcurl supports HTTPS])
AH_TEMPLATE([LIBCURL_PROTOCOL_FTP],[Defined if libcurl supports FTP])
AH_TEMPLATE([LIBCURL_PROTOCOL_FTPS],[Defined if libcurl supports FTPS])
AH_TEMPLATE([LIBCURL_PROTOCOL_FILE],[Defined if libcurl supports FILE])
AH_TEMPLATE([LIBCURL_PROTOCOL_TELNET],[Defined if libcurl supports TELNET])
AH_TEMPLATE([LIBCURL_PROTOCOL_LDAP],[Defined if libcurl supports LDAP])
AH_TEMPLATE([LIBCURL_PROTOCOL_DICT],[Defined if libcurl supports DICT])
AH_TEMPLATE([LIBCURL_PROTOCOL_TFTP],[Defined if libcurl supports TFTP])
AH_TEMPLATE([LIBCURL_PROTOCOL_RTSP],[Defined if libcurl supports RTSP])
AH_TEMPLATE([LIBCURL_PROTOCOL_POP3],[Defined if libcurl supports POP3])
AH_TEMPLATE([LIBCURL_PROTOCOL_IMAP],[Defined if libcurl supports IMAP])
AH_TEMPLATE([LIBCURL_PROTOCOL_SMTP],[Defined if libcurl supports SMTP])
AC_ARG_WITH(libcurl,
AC_HELP_STRING([--with-libcurl=PREFIX],[look for the curl library in PREFIX/lib and headers in PREFIX/include]),
[_libcurl_with=$withval],[_libcurl_with=ifelse([$1],,[yes],[$1])])
if test "$_libcurl_with" != "no" ; then
AC_PROG_AWK
_libcurl_version_parse="eval $AWK '{split(\$NF,A,\".\"); X=256*256*A[[1]]+256*A[[2]]+A[[3]]; print X;}'"
_libcurl_try_link=yes
if test -d "$_libcurl_with" ; then
LIBCURL_CPPFLAGS="-I$withval/include"
_libcurl_ldflags="-L$withval/lib"
AC_PATH_PROG([_libcurl_config],[curl-config],[],
["$withval/bin"])
else
AC_PATH_PROG([_libcurl_config],[curl-config],[],[$PATH])
fi
if test x$_libcurl_config != "x" ; then
AC_CACHE_CHECK([for the version of libcurl],
[libcurl_cv_lib_curl_version],
[libcurl_cv_lib_curl_version=`$_libcurl_config --version | $AWK '{print $[]2}'`])
_libcurl_version=`echo $libcurl_cv_lib_curl_version | $_libcurl_version_parse`
_libcurl_wanted=`echo ifelse([$2],,[0],[$2]) | $_libcurl_version_parse`
if test $_libcurl_wanted -gt 0 ; then
AC_CACHE_CHECK([for libcurl >= version $2],
[libcurl_cv_lib_version_ok],
[
if test $_libcurl_version -ge $_libcurl_wanted ; then
libcurl_cv_lib_version_ok=yes
else
libcurl_cv_lib_version_ok=no
fi
])
fi
if test $_libcurl_wanted -eq 0 || test x$libcurl_cv_lib_version_ok = xyes ; then
if test x"$LIBCURL_CPPFLAGS" = "x" ; then
LIBCURL_CPPFLAGS=`$_libcurl_config --cflags`
fi
if test x"$LIBCURL" = "x" ; then
LIBCURL=`$_libcurl_config --libs`
# This is so silly, but Apple actually has a bug in their
# curl-config script. Fixed in Tiger, but there are still
# lots of Panther installs around.
case "${host}" in
powerpc-apple-darwin7*)
LIBCURL=`echo $LIBCURL | sed -e 's|-arch i386||g'`
;;
esac
fi
# All curl-config scripts support --feature
_libcurl_features=`$_libcurl_config --feature`
# Is it modern enough to have --protocols? (7.12.4)
if test $_libcurl_version -ge 461828 ; then
_libcurl_protocols=`$_libcurl_config --protocols`
fi
else
_libcurl_try_link=no
fi
unset _libcurl_wanted
fi
if test $_libcurl_try_link = yes ; then
# we didn't find curl-config, so let's see if the user-supplied
# link line (or failing that, "-lcurl") is enough.
LIBCURL=${LIBCURL-"$_libcurl_ldflags -lcurl"}
AC_CACHE_CHECK([whether libcurl is usable],
[libcurl_cv_lib_curl_usable],
[
_libcurl_save_cppflags=$CPPFLAGS
CPPFLAGS="$LIBCURL_CPPFLAGS $CPPFLAGS"
_libcurl_save_libs=$LIBS
LIBS="$LIBCURL $LIBS"
AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <curl/curl.h>]],[[
/* Try and use a few common options to force a failure if we are
missing symbols or can't link. */
int x;
curl_easy_setopt(NULL,CURLOPT_URL,NULL);
x=CURL_ERROR_SIZE;
x=CURLOPT_WRITEFUNCTION;
x=CURLOPT_WRITEDATA;
x=CURLOPT_ERRORBUFFER;
x=CURLOPT_STDERR;
x=CURLOPT_VERBOSE;
if (x) ;
]])],libcurl_cv_lib_curl_usable=yes,libcurl_cv_lib_curl_usable=no)
CPPFLAGS=$_libcurl_save_cppflags
LIBS=$_libcurl_save_libs
unset _libcurl_save_cppflags
unset _libcurl_save_libs
])
if test $libcurl_cv_lib_curl_usable = yes ; then
# Does curl_free() exist in this version of libcurl?
# If not, fake it with free()
_libcurl_save_cppflags=$CPPFLAGS
CPPFLAGS="$CPPFLAGS $LIBCURL_CPPFLAGS"
_libcurl_save_libs=$LIBS
LIBS="$LIBS $LIBCURL"
AC_CHECK_FUNC(curl_free,,
AC_DEFINE(curl_free,free,
[Define curl_free() as free() if our version of curl lacks curl_free.]))
CPPFLAGS=$_libcurl_save_cppflags
LIBS=$_libcurl_save_libs
unset _libcurl_save_cppflags
unset _libcurl_save_libs
AC_DEFINE(HAVE_LIBCURL,1,
[Define to 1 if you have a functional curl library.])
AC_SUBST(LIBCURL_CPPFLAGS)
AC_SUBST(LIBCURL)
for _libcurl_feature in $_libcurl_features ; do
AC_DEFINE_UNQUOTED(AS_TR_CPP(libcurl_feature_$_libcurl_feature),[1])
eval AS_TR_SH(libcurl_feature_$_libcurl_feature)=yes
done
if test "x$_libcurl_protocols" = "x" ; then
# We don't have --protocols, so just assume that all
# protocols are available
_libcurl_protocols="HTTP FTP FILE TELNET LDAP DICT TFTP"
if test x$libcurl_feature_SSL = xyes ; then
_libcurl_protocols="$_libcurl_protocols HTTPS"
# FTPS wasn't standards-compliant until version
# 7.11.0 (0x070b00 == 461568)
if test $_libcurl_version -ge 461568; then
_libcurl_protocols="$_libcurl_protocols FTPS"
fi
fi
# RTSP, IMAP, POP3 and SMTP were added in
# 7.20.0 (0x071400 == 463872)
if test $_libcurl_version -ge 463872; then
_libcurl_protocols="$_libcurl_protocols RTSP IMAP POP3 SMTP"
fi
fi
for _libcurl_protocol in $_libcurl_protocols ; do
AC_DEFINE_UNQUOTED(AS_TR_CPP(libcurl_protocol_$_libcurl_protocol),[1])
eval AS_TR_SH(libcurl_protocol_$_libcurl_protocol)=yes
done
else
unset LIBCURL
unset LIBCURL_CPPFLAGS
fi
fi
unset _libcurl_try_link
unset _libcurl_version_parse
unset _libcurl_config
unset _libcurl_feature
unset _libcurl_features
unset _libcurl_protocol
unset _libcurl_protocols
unset _libcurl_version
unset _libcurl_ldflags
fi
if test x$_libcurl_with = xno || test x$libcurl_cv_lib_curl_usable != xyes ; then
# This is the IF-NO path
ifelse([$4],,:,[$4])
else
# This is the IF-YES path
ifelse([$3],,:,[$3])
fi
unset _libcurl_with
])dnl

@ -74,13 +74,15 @@ AM_CPPFLAGS = \
-DSYSCONFDIR='"$(sysconfdir)"' \
-Wall \
$(READLINE_CFLAGS) \
$(SQLITE3_CFLAGS)
$(SQLITE3_CFLAGS) \
$(LIBCURL_CPPFLAGS)
LDADD = \
libdiag.a \
$(READLINE_LIBS) \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
$(LIBCURL) \
-lpcrecpp
dist_noinst_DATA = \
@ -133,6 +135,7 @@ noinst_HEADERS = \
log_format_loader.hh \
logfile.hh \
logfile_sub_source.hh \
papertrail_proc.hh \
pcrepp.hh \
piper_proc.hh \
pretty_printer.hh \
@ -253,6 +256,10 @@ libdiag_a_SOURCES += yajl/yajl.c \
yajl/yajl_version.c
endif
if HAVE_LIBCURL
libdiag_a_SOURCES += \
papertrail_proc.cc
endif
TEXT2C_FILES = \
dump-pid-sh.o \

@ -626,6 +626,44 @@
}
]
},
"papertrail_log": {
"title" : "Papertrail Service",
"url" : "https://papertrailapp.com/",
"description" : "Log format for the papertrail log management service",
"json" : true,
"hide-extra" : true,
"file-pattern" : "papertrailapp.com",
"line-format" : [
{ "field" : "display_received_at" },
" ",
{ "field" : "hostname" },
" ",
{ "field" : "program" },
": ",
{ "field" : "message" }
],
"level-field" : "severity",
"level" : {
"error" : "Error",
"debug" : "Debug",
"warning" : "Warning",
"info" : "Info(?:rmational)?|Notice",
"critical" : "Crit(?:ical)?",
"fatal" : "Emerg(?:ency)?|Alert"
},
"timestamp-field" : "generated_at",
"body-field" : "message",
"value" : {
"program" : {
"kind" : "string",
"identifier" : true
},
"hostname" : {
"kind" : "string",
"identifier" : true
}
}
},
"snaplogic_log" : {
"title" : "SnapLogic Server Log",
"description" : "The SnapLogic server log format.",

@ -118,6 +118,11 @@
#include "pretty_printer.hh"
#include "all_logs_vtab.hh"
#ifdef HAVE_LIBCURL
#include <curl/curl.h>
#include "papertrail_proc.hh"
#endif
#include "yajlpp.hh"
using namespace std;
@ -4247,6 +4252,10 @@ int main(int argc, char *argv[])
log_install_handlers();
sql_install_logger();
#ifdef HAVE_LIBCURL
curl_global_init(CURL_GLOBAL_DEFAULT);
#endif
lnav_data.ld_debug_log_name = "/dev/null";
while ((c = getopt(argc, argv, "hHarsCc:I:if:d:nqtw:VW")) != -1) {
switch (c) {
@ -4634,6 +4643,17 @@ int main(int argc, char *argv[])
lnav_data.ld_pipers.push_back(stdin_reader.release());
}
#ifdef HAVE_LIBCURL
char *api_key = getenv("PAPERTRAIL_API_TOKEN");
auto_ptr<papertrail_proc> ptp;
if (api_key != NULL) {
ptp.reset(new papertrail_proc(api_key));
ptp->start();
lnav_data.ld_file_names.insert(make_pair("papertrailapp.com", ptp->ptp_fd.release()));
}
#endif
if (lnav_data.ld_file_names.empty() && !(lnav_data.ld_flags & LNF_HELP)) {
fprintf(stderr, "error: no log files given/found.\n");
retval = EXIT_FAILURE;

@ -370,7 +370,8 @@ static int read_json_null(yajlpp_parse_context *ypc)
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
return 1;
}
if (find_if(line_format.begin(), line_format.end(),
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
@ -389,7 +390,8 @@ static int read_json_bool(yajlpp_parse_context *ypc, int val)
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
return 1;
}
if (find_if(line_format.begin(), line_format.end(),
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
@ -416,7 +418,8 @@ static int read_json_int(yajlpp_parse_context *ypc, long long val)
tv.tv_usec = (val % divisor) * (1000000.0 / divisor);
jlu->jlu_base_line->set_time(tv);
}
else if (find_if(line_format.begin(), line_format.end(),
else if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
@ -443,7 +446,8 @@ static int read_json_double(yajlpp_parse_context *ypc, double val)
tv.tv_usec = fmod(val, divisor) * (1000000.0 / divisor);
jlu->jlu_base_line->set_time(tv);
}
else if (find_if(line_format.begin(), line_format.end(),
else if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
@ -462,7 +466,8 @@ static int json_array_start(void *ctx)
if (ypc->ypc_path_index_stack.size() == 2) {
const intern_string_t field_name = ypc->get_path_fragment_i(0);
if (find_if(line_format.begin(), line_format.end(),
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
@ -793,7 +798,8 @@ static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str,
}
else if (ypc->is_level(1) || jlu->jlu_format->elf_value_defs.find(field_name) !=
jlu->jlu_format->elf_value_defs.end()) {
if (find_if(line_format.begin(), line_format.end(),
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
@ -973,40 +979,44 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
}
}
this->json_append_to_cache("\n", 1);
for (size_t lpc = 0; lpc < this->jlf_line_values.size(); lpc++) {
static const intern_string_t body_name = intern_string::lookup("body", -1);
logline_value &lv = this->jlf_line_values[lpc];
if (used_values[lpc] ||
lv.lv_name == this->lf_timestamp_field ||
lv.lv_name == body_name ||
lv.lv_name == this->elf_level_field) {
continue;
}
if (!this->jlf_hide_extra) {
for (size_t lpc = 0;
lpc < this->jlf_line_values.size(); lpc++) {
static const intern_string_t body_name = intern_string::lookup(
"body", -1);
logline_value &lv = this->jlf_line_values[lpc];
if (used_values[lpc] ||
lv.lv_name == this->lf_timestamp_field ||
lv.lv_name == body_name ||
lv.lv_name == this->elf_level_field) {
continue;
}
const std::string str = lv.to_string();
size_t curr_pos = 0, nl_pos, line_len = -1;
const std::string str = lv.to_string();
size_t curr_pos = 0, nl_pos, line_len = -1;
lv.lv_origin.lr_start = this->jlf_cached_line.size();
do {
nl_pos = str.find('\n', curr_pos);
if (nl_pos != std::string::npos) {
line_len = nl_pos - curr_pos;
}
else {
line_len = str.size() - curr_pos;
}
this->json_append_to_cache(" ", 2);
this->json_append_to_cache(lv.lv_name.get(),
lv.lv_name.size());
this->json_append_to_cache(": ", 2);
this->json_append_to_cache(
&str.c_str()[curr_pos], line_len);
this->json_append_to_cache("\n", 1);
curr_pos = nl_pos + 1;
} while (nl_pos != std::string::npos &&
nl_pos < str.size());
lv.lv_origin.lr_end = this->jlf_cached_line.size();
lv.lv_origin.lr_start = this->jlf_cached_line.size();
do {
nl_pos = str.find('\n', curr_pos);
if (nl_pos != std::string::npos) {
line_len = nl_pos - curr_pos;
}
else {
line_len = str.size() - curr_pos;
}
this->json_append_to_cache(" ", 2);
this->json_append_to_cache(lv.lv_name.get(),
lv.lv_name.size());
this->json_append_to_cache(": ", 2);
this->json_append_to_cache(
&str.c_str()[curr_pos], line_len);
this->json_append_to_cache("\n", 1);
curr_pos = nl_pos + 1;
} while (nl_pos != std::string::npos &&
nl_pos < str.size());
lv.lv_origin.lr_end = this->jlf_cached_line.size();
}
}
this->jlf_line_offsets.push_back(0);

@ -699,6 +699,7 @@ public:
elf_timestamp_divisor(1.0),
elf_body_field(intern_string::lookup("body", -1)),
jlf_json(false),
jlf_hide_extra(false),
jlf_cached_offset(-1),
jlf_yajl_handle(yajl_free),
elf_name(name) {
@ -829,6 +830,7 @@ public:
};
bool jlf_json;
bool jlf_hide_extra;
std::vector<json_format_element> jlf_line_format;
std::vector<logline_value> jlf_line_values;

@ -94,6 +94,8 @@ static int read_format_bool(yajlpp_parse_context *ypc, int val)
elf->lf_date_time.dts_local_time = val;
else if (field_name == "json")
elf->jlf_json = val;
else if (field_name == "hide-extra")
elf->jlf_hide_extra = val;
return 1;
}
@ -373,7 +375,7 @@ static int read_json_variable_num(yajlpp_parse_context *ypc, long long val)
static struct json_path_handler format_handlers[] = {
json_path_handler("^/\\w+/regex/[^/]+/pattern$", read_format_regex),
json_path_handler("^/\\w+/(json|convert-to-local-time|epoch-timestamp)$", read_format_bool),
json_path_handler("^/\\w+/(json|convert-to-local-time|epoch-timestamp|hide-extra)$", read_format_bool),
json_path_handler("^/\\w+/timestamp-divisor$", read_format_double)
.add_cb(read_format_int),
json_path_handler("^/\\w+/(file-pattern|level-field|timestamp-field|"

@ -0,0 +1,222 @@
/**
* Copyright (c) 2015, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file papertrail_proc.cc
*/
#include "config.h"
#include "papertrail_proc.hh"
#include "yajl/api/yajl_parse.h"
static const int POLL_DELAY = 2;
static const char *PT_SEARCH_URL = "https://papertrailapp.com/api/v1/events/search.json?min_id=%s";
static int read_max_id(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
papertrail_proc *ptp = (papertrail_proc *) ypc->ypc_userdata;
ptp->ptp_last_max_id = std::string((const char *) str, len);
return 1;
}
static int ignore_bool(yajlpp_parse_context *ypc, int val)
{
return 1;
}
static int ignore_str(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
return 1;
}
static int read_event_int(yajlpp_parse_context *ypc, long long val)
{
papertrail_proc *ptp = (papertrail_proc *) ypc->ypc_userdata;
yajl_gen_string(ptp->ptp_gen, ypc->get_path_fragment(2));
yajl_gen_integer(ptp->ptp_gen, val);
return 1;
}
static int read_event_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
papertrail_proc *ptp = (papertrail_proc *) ypc->ypc_userdata;
yajl_gen_string(ptp->ptp_gen, ypc->get_path_fragment(2));
yajl_gen_string(ptp->ptp_gen, str, len);
return 1;
}
static int json_map_start(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
papertrail_proc *ptp = (papertrail_proc *) ypc->ypc_userdata;
if (ypc->ypc_path_index_stack.size() == 3) {
yajl_gen_map_open(ptp->ptp_gen);
}
return 1;
}
static int json_map_end(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
papertrail_proc *ptp = (papertrail_proc *) ypc->ypc_userdata;
if (ypc->ypc_path_index_stack.size() == 2) {
yajl_gen_map_close(ptp->ptp_gen);
yajl_gen_reset(ptp->ptp_gen, "\n");
}
return 1;
}
struct json_path_handler papertrail_proc::FORMAT_HANDLERS[] = {
json_path_handler("^/max_id", read_max_id),
json_path_handler("/(min_id|min_time_at|reached_beginning|reached_record_limit|tail|partial_results)")
.add_cb(ignore_bool)
.add_cb(ignore_str),
json_path_handler("/events#/\\w+")
.add_cb(read_event_field)
.add_cb(read_event_int),
json_path_handler()
};
size_t papertrail_proc::write_cb(void *contents, size_t size, size_t nmemb, void *userp)
{
yajl_handle handle = (yajl_handle) userp;
size_t realsize = size * nmemb;
if (yajl_parse(handle, (const unsigned char *)contents, realsize) != yajl_status_ok) {
return -1;
}
return realsize;
}
void papertrail_proc::yajl_writer(void *context, const char *str, size_t len)
{
papertrail_proc *ptp = (papertrail_proc *) context;
write(ptp->ptp_fd, str, len);
}
void papertrail_proc::start(void)
{
char piper_tmpname[PATH_MAX];
const char *tmpdir;
if ((tmpdir = getenv("TMPDIR")) == NULL) {
tmpdir = _PATH_VARTMP;
}
snprintf(piper_tmpname, sizeof(piper_tmpname),
"%s/lnav.papertrail.XXXXXX",
tmpdir);
if ((this->ptp_fd = mkstemp(piper_tmpname)) == -1) {
perror("Unable to make temporary file for papertrail");
return;
}
unlink(piper_tmpname);
fcntl(this->ptp_fd.get(), F_SETFD, FD_CLOEXEC);
if ((this->ptp_child = fork()) < 0) {
perror("Unable to fork papertrail child");
return;
}
if (this->ptp_child != 0) {
return;
}
try {
this->child_body();
} catch (...) {
fprintf(stderr, "papertrail child failed");
}
_exit(0);
};
void papertrail_proc::child_body()
{
int nullfd;
nullfd = open("/dev/null", O_RDWR);
dup2(nullfd, STDIN_FILENO);
dup2(nullfd, STDOUT_FILENO);
auto_mem<CURL> handle(curl_easy_cleanup);
yajlpp_parse_context ypc("papertrailapp.com", FORMAT_HANDLERS);
ypc.ypc_alt_callbacks.yajl_start_map = json_map_start;
ypc.ypc_alt_callbacks.yajl_end_map = json_map_end;
yajl_handle jhandle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc);
auto_mem<yajl_gen_t> gen(yajl_gen_free);
this->ptp_gen = gen = yajl_gen_alloc(NULL);
ypc.ypc_userdata = this;
yajl_gen_config(gen, yajl_gen_print_callback, yajl_writer, this);
bool looping = true;
while (looping) {
auto_mem<char> url;
handle = curl_easy_init();
asprintf(url.out(), PT_SEARCH_URL, this->ptp_last_max_id.c_str());
if (!url.in()) {
break;
}
curl_easy_setopt(handle, CURLOPT_URL, url.in());
curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt(handle, CURLOPT_WRITEDATA, jhandle);
auto_mem<struct curl_slist> chunk(curl_slist_free_all);
auto_mem<char> token_header;
asprintf(token_header.out(), "X-Papertrail-Token: %s", this->ptp_api_key);
if (!token_header.in()) {
break;
}
chunk = curl_slist_append(chunk, token_header.in());
curl_easy_setopt(handle, CURLOPT_HTTPHEADER, chunk.in());
curl_easy_perform(handle);
yajl_reset(jhandle);
sleep(POLL_DELAY);
}
}

@ -0,0 +1,87 @@
/**
* Copyright (c) 2015, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file papertrail_proc.hh
*/
#ifndef LNAV_PAPERTRAIL_PROC_HH
#define LNAV_PAPERTRAIL_PROC_HH
#include <fcntl.h>
#include <paths.h>
#include <signal.h>
#include <unistd.h>
#include <sys/wait.h>
#include <curl/curl.h>
#include <memory>
#include <string>
#include "auto_fd.hh"
#include "auto_mem.hh"
#include "yajlpp.hh"
#include "line_buffer.hh"
class papertrail_proc {
public:
papertrail_proc(const char *api_key)
: ptp_api_key(api_key), ptp_child(-1) {
};
~papertrail_proc() {
// TODO: refactor this with piper_proc
if (this->ptp_child > 0) {
int status;
kill(this->ptp_child, SIGTERM);
while (waitpid(this->ptp_child, &status, 0) < 0 && (errno == EINTR)) {
;
}
this->ptp_child = -1;
}
}
void start(void);
void child_body(void);
static size_t write_cb(void *contents, size_t size, size_t nmemb, void *userp);
static void yajl_writer(void *context, const char *str, size_t len);
static struct json_path_handler FORMAT_HANDLERS[];
const char *ptp_api_key;
auto_fd ptp_fd;
pid_t ptp_child;
line_buffer ptp_line_buffer;
yajl_gen ptp_gen;
std::string ptp_last_max_id;
};
#endif //LNAV_PAPERTRAIL_PROC_HH

@ -261,6 +261,11 @@ public:
yajl_gen_string(this->yg_handle, (const unsigned char *)str, strlen(str));
};
void operator()(const char *str, size_t len)
{
yajl_gen_string(this->yg_handle, (const unsigned char *)str, len);
};
void operator()(long long value)
{
yajl_gen_integer(this->yg_handle, value);
@ -275,7 +280,6 @@ public:
{
yajl_gen_null(this->yg_handle);
};
private:
yajl_gen yg_handle;
};

Loading…
Cancel
Save