[formats] add cloudflare format

Related to #1081

NEWS.md

@@ -12,6 +12,7 @@ Features:
   field should automatically be determined by the observed
   values.
 * Added bunyan log format from Tobias Gruetzmacher.
+* Added cloudlare log format from @minusf.
 * Number fields used in a JSON log format `line-format`
   array now default to being right-aligned.  Also, added
   `prefix` and `suffix` to `line-format` elements so a

src/formats/cloudflare_log.json

@@ -0,0 +1,236 @@
+{
+    "$schema": "https://lnav.org/schemas/format-v1.schema.json",
+    "cloudflare_json_log": {
+        "title": "Cloudflare Access Log",
+        "description": "Cloudflare Enterprise detailed logs of metadata",
+        "url": "https://developers.cloudflare.com/logs/",
+        "json": true,
+        "hide-extra": true,
+        "ordered-by-time": false,
+        "level-field": "CacheCacheStatus",
+        "level": {
+            "info": "hit",
+            "warning": "miss"
+        },
+        "line-format": [
+            {
+                "field": "ClientIP",
+                "auto-width": true
+            },
+            " ",
+            {
+                "prefix": "[",
+                "field": "__timestamp__",
+                "suffix": "]"
+            },
+            " \"",
+            {
+                "field": "ClientRequestMethod"
+            },
+            " ",
+            {
+                "field": "ClientRequestURI"
+            },
+            " ",
+            {
+                "field": "ClientRequestProtocol"
+            },
+            "\" ",
+            {
+                "field": "EdgeResponseStatus"
+            },
+            " ",
+            {
+                "field": "EdgeResponseBytes"
+            },
+            " ",
+            {
+                "field": "ClientRequestReferer"
+            },
+            " ",
+            {
+                "field": "ClientRequestUserAgent"
+            }
+        ],
+        "timestamp-field": "EdgeStartTimestamp",
+        "opid-field": "ClientIP",
+        "value": {
+            "ClientIP": {
+                "kind": "string",
+                "identifier": true
+            },
+            "ClientRequestMethod": {
+                "kind": "string"
+            },
+            "ClientRequestURI": {
+                "kind": "string",
+                "identifier": true
+            },
+            "EdgeEndTimestamp": {
+                "kind": "string",
+                "hidden": true
+            },
+            "EdgeResponseBytes": {
+                "kind": "integer"
+            },
+            "EdgeResponseStatus": {
+                "kind": "integer"
+            },
+            "EdgeStartTimestamp": {
+                "kind": "string"
+            },
+            "RayID": {
+                "kind": "string",
+                "identifier": true,
+                "foreign-key": true,
+                "hidden": true
+            },
+            "CacheCacheStatus": {
+                "kind": "string",
+                "hidden": true
+            },
+            "CacheTieredFill": {
+                "kind": "json",
+                "hidden": true
+            },
+            "CacheResponseBytes": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "CacheResponseStatus": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "FirewallMatchesActions": {
+                "kind": "json",
+                "hidden": true
+            },
+            "FirewallMatchesRuleIDs": {
+                "kind": "json",
+                "hidden": true
+            },
+            "FirewallMatchesSources": {
+                "kind": "json",
+                "hidden": true
+            },
+            "OriginResponseBytes": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginResponseDurationMs": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginResponseHTTPExpires": {
+                "kind": "string",
+                "hidden": true
+            },
+            "OriginResponseHTTPLastModified": {
+                "kind": "string",
+                "hidden": true
+            },
+            "OriginResponseHeaderReceiveDurationMs": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginResponseStatus": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginResponseTime": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginDNSResponseTimeMs": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginIP": {
+                "kind": "string",
+                "hidden": true
+            },
+            "OriginRequestHeaderSendDurationMs": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginSSLProtocol": {
+                "kind": "string",
+                "hidden": true
+            },
+            "OriginTCPHandshakeDurationMs": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "OriginTLSHandshakeDurationMs": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "WAFAction": {
+                "kind": "string",
+                "hidden": true
+            },
+            "WAFFlags": {
+                "kind": "string",
+                "hidden": true
+            },
+            "WAFMatchedVar": {
+                "kind": "string",
+                "hidden": true
+            },
+            "WAFProfile": {
+                "kind": "string",
+                "hidden": true
+            },
+            "WAFRuleID": {
+                "kind": "string",
+                "hidden": true
+            },
+            "WAFRuleMessage": {
+                "kind": "string",
+                "hidden": true
+            },
+            "ClientASN": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "ClientCountry": {
+                "kind": "string",
+                "hidden": true
+            },
+            "ClientDeviceType": {
+                "kind": "string",
+                "hidden": true
+            },
+            "ClientIPClass": {
+                "kind": "string",
+                "hidden": true
+            },
+            "ClientRequestBytes": {
+                "kind": "integer",
+                "hidden": true
+            },
+            "ClientRequestPath": {
+                "kind": "integer",
+                "identifier": true,
+                "hidden": true
+            },
+            "ClientRequestProtocol": {
+                "kind": "string"
+            },
+            "ClientRequestUserAgent": {
+                "kind": "string"
+            },
+            "ClientRequestReferer": {
+                "kind": "string"
+            },
+            "ClientRequestScheme": {
+                "kind": "string",
+                "hidden": true
+            },
+            "ClientRequestSource": {
+                "kind": "string",
+                "hidden": true
+            }
+        }
+    }
+}

src/formats/formats.am

@@ -6,6 +6,7 @@ FORMAT_FILES = \
     $(srcdir)/%reldir%/bunyan_log.json \
     $(srcdir)/%reldir%/candlepin_log.json \
     $(srcdir)/%reldir%/choose_repo_log.json \
+    $(srcdir)/%reldir%/cloudflare_log.json \
     $(srcdir)/%reldir%/cups_log.json \
     $(srcdir)/%reldir%/dpkg_log.json \
     $(srcdir)/%reldir%/elb_log.json \

test/Makefile.am

@@ -298,6 +298,7 @@ dist_noinst_DATA = \
 	logfile_bro_http.log.0 \
 	logfile_bunyan.0 \
 	logfile_crlf.0 \
+	logfile_cloudflare.json \
 	logfile_cxx.0 \
 	logfile_empty.0 \
 	logfile_epoch.0 \

test/expected/expected.am

@@ -284,6 +284,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_json_format.sh_989e52d167582648b73c5d025cc0e814c642b3c8.out \
     $(srcdir)/%reldir%/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.err \
     $(srcdir)/%reldir%/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.out \
+    $(srcdir)/%reldir%/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.err \
+    $(srcdir)/%reldir%/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.out \
     $(srcdir)/%reldir%/test_json_format.sh_c1a23804c39b0f74642286d69865ee9d0961a58a.err \
     $(srcdir)/%reldir%/test_json_format.sh_c1a23804c39b0f74642286d69865ee9d0961a58a.out \
     $(srcdir)/%reldir%/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.err \

test/expected/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.out

@@ -0,0 +1 @@
+87.226.160.250 2022-10-30T00:00:02.000 "HEAD / HTTP/1.1" 301 570  

test/logfile_cloudflare.json

@@ -0,0 +1 @@
+{"ClientIP":"87.226.160.250","ClientRequestHost":"www.ripe.net","ClientRequestMethod":"HEAD","ClientRequestURI":"/","EdgeEndTimestamp":"2022-10-30T00:00:02Z","EdgeResponseBytes":570,"EdgeResponseStatus":301,"EdgeStartTimestamp":"2022-10-30T00:00:02Z","RayID":"761fde4e984e5ab2","CacheCacheStatus":"unknown","CacheTieredFill":false,"CacheResponseBytes":0,"CacheResponseStatus":0,"FirewallMatchesActions":[],"FirewallMatchesRuleIDs":[],"FirewallMatchesSources":[],"OriginResponseBytes":0,"OriginResponseDurationMs":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseHeaderReceiveDurationMs":0,"OriginResponseStatus":0,"OriginResponseTime":0,"OriginDNSResponseTimeMs":0,"OriginIP":"","OriginRequestHeaderSendDurationMs":0,"OriginSSLProtocol":"unknown","OriginTCPHandshakeDurationMs":0,"OriginTLSHandshakeDurationMs":0,"WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","ClientASN":12389,"ClientCountry":"ru","ClientDeviceType":"desktop","ClientIPClass":"noRecord","ClientRequestBytes":567,"ClientRequestPath":"/","ClientRequestProtocol":"HTTP/1.1","ClientRequestUserAgent":"","ClientRequestReferer":"","ClientRequestScheme":"http","ClientRequestSource":"eyeball"}

test/test_json_format.sh

@@ -147,3 +147,6 @@ run_cap_test ${lnav_test} -n \
 
 run_cap_test ${lnav_test} -n \
     ${test_dir}/logfile_bunyan.0
+
+run_cap_test ${lnav_test} -n \
+    ${test_dir}/logfile_cloudflare.json