diff --git a/NEWS.md b/NEWS.md index 7f7cdae0..fcd0aacd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,7 @@ Features: field should automatically be determined by the observed values. * Added bunyan log format from Tobias Gruetzmacher. +* Added cloudlare log format from @minusf. * Number fields used in a JSON log format `line-format` array now default to being right-aligned. Also, added `prefix` and `suffix` to `line-format` elements so a diff --git a/src/formats/cloudflare_log.json b/src/formats/cloudflare_log.json new file mode 100644 index 00000000..01fa9569 --- /dev/null +++ b/src/formats/cloudflare_log.json @@ -0,0 +1,236 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "cloudflare_json_log": { + "title": "Cloudflare Access Log", + "description": "Cloudflare Enterprise detailed logs of metadata", + "url": "https://developers.cloudflare.com/logs/", + "json": true, + "hide-extra": true, + "ordered-by-time": false, + "level-field": "CacheCacheStatus", + "level": { + "info": "hit", + "warning": "miss" + }, + "line-format": [ + { + "field": "ClientIP", + "auto-width": true + }, + " ", + { + "prefix": "[", + "field": "__timestamp__", + "suffix": "]" + }, + " \"", + { + "field": "ClientRequestMethod" + }, + " ", + { + "field": "ClientRequestURI" + }, + " ", + { + "field": "ClientRequestProtocol" + }, + "\" ", + { + "field": "EdgeResponseStatus" + }, + " ", + { + "field": "EdgeResponseBytes" + }, + " ", + { + "field": "ClientRequestReferer" + }, + " ", + { + "field": "ClientRequestUserAgent" + } + ], + "timestamp-field": "EdgeStartTimestamp", + "opid-field": "ClientIP", + "value": { + "ClientIP": { + "kind": "string", + "identifier": true + }, + "ClientRequestMethod": { + "kind": "string" + }, + "ClientRequestURI": { + "kind": "string", + "identifier": true + }, + "EdgeEndTimestamp": { + "kind": "string", + "hidden": true + }, + "EdgeResponseBytes": { + "kind": "integer" + }, + "EdgeResponseStatus": { + "kind": "integer" + }, + "EdgeStartTimestamp": { + "kind": "string" + }, + "RayID": { + "kind": "string", + "identifier": true, + "foreign-key": true, + "hidden": true + }, + "CacheCacheStatus": { + "kind": "string", + "hidden": true + }, + "CacheTieredFill": { + "kind": "json", + "hidden": true + }, + "CacheResponseBytes": { + "kind": "integer", + "hidden": true + }, + "CacheResponseStatus": { + "kind": "integer", + "hidden": true + }, + "FirewallMatchesActions": { + "kind": "json", + "hidden": true + }, + "FirewallMatchesRuleIDs": { + "kind": "json", + "hidden": true + }, + "FirewallMatchesSources": { + "kind": "json", + "hidden": true + }, + "OriginResponseBytes": { + "kind": "integer", + "hidden": true + }, + "OriginResponseDurationMs": { + "kind": "integer", + "hidden": true + }, + "OriginResponseHTTPExpires": { + "kind": "string", + "hidden": true + }, + "OriginResponseHTTPLastModified": { + "kind": "string", + "hidden": true + }, + "OriginResponseHeaderReceiveDurationMs": { + "kind": "integer", + "hidden": true + }, + "OriginResponseStatus": { + "kind": "integer", + "hidden": true + }, + "OriginResponseTime": { + "kind": "integer", + "hidden": true + }, + "OriginDNSResponseTimeMs": { + "kind": "integer", + "hidden": true + }, + "OriginIP": { + "kind": "string", + "hidden": true + }, + "OriginRequestHeaderSendDurationMs": { + "kind": "integer", + "hidden": true + }, + "OriginSSLProtocol": { + "kind": "string", + "hidden": true + }, + "OriginTCPHandshakeDurationMs": { + "kind": "integer", + "hidden": true + }, + "OriginTLSHandshakeDurationMs": { + "kind": "integer", + "hidden": true + }, + "WAFAction": { + "kind": "string", + "hidden": true + }, + "WAFFlags": { + "kind": "string", + "hidden": true + }, + "WAFMatchedVar": { + "kind": "string", + "hidden": true + }, + "WAFProfile": { + "kind": "string", + "hidden": true + }, + "WAFRuleID": { + "kind": "string", + "hidden": true + }, + "WAFRuleMessage": { + "kind": "string", + "hidden": true + }, + "ClientASN": { + "kind": "integer", + "hidden": true + }, + "ClientCountry": { + "kind": "string", + "hidden": true + }, + "ClientDeviceType": { + "kind": "string", + "hidden": true + }, + "ClientIPClass": { + "kind": "string", + "hidden": true + }, + "ClientRequestBytes": { + "kind": "integer", + "hidden": true + }, + "ClientRequestPath": { + "kind": "integer", + "identifier": true, + "hidden": true + }, + "ClientRequestProtocol": { + "kind": "string" + }, + "ClientRequestUserAgent": { + "kind": "string" + }, + "ClientRequestReferer": { + "kind": "string" + }, + "ClientRequestScheme": { + "kind": "string", + "hidden": true + }, + "ClientRequestSource": { + "kind": "string", + "hidden": true + } + } + } +} \ No newline at end of file diff --git a/src/formats/formats.am b/src/formats/formats.am index dffc74e8..d37a9765 100644 --- a/src/formats/formats.am +++ b/src/formats/formats.am @@ -6,6 +6,7 @@ FORMAT_FILES = \ $(srcdir)/%reldir%/bunyan_log.json \ $(srcdir)/%reldir%/candlepin_log.json \ $(srcdir)/%reldir%/choose_repo_log.json \ + $(srcdir)/%reldir%/cloudflare_log.json \ $(srcdir)/%reldir%/cups_log.json \ $(srcdir)/%reldir%/dpkg_log.json \ $(srcdir)/%reldir%/elb_log.json \ diff --git a/test/Makefile.am b/test/Makefile.am index b2900322..a92ceb89 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -298,6 +298,7 @@ dist_noinst_DATA = \ logfile_bro_http.log.0 \ logfile_bunyan.0 \ logfile_crlf.0 \ + logfile_cloudflare.json \ logfile_cxx.0 \ logfile_empty.0 \ logfile_epoch.0 \ diff --git a/test/expected/expected.am b/test/expected/expected.am index 152350b4..e359d3f2 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -284,6 +284,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_json_format.sh_989e52d167582648b73c5d025cc0e814c642b3c8.out \ $(srcdir)/%reldir%/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.err \ $(srcdir)/%reldir%/test_json_format.sh_a06b3cdd46b387e72d6faa4cce648b8b11ae870b.out \ + $(srcdir)/%reldir%/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.err \ + $(srcdir)/%reldir%/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.out \ $(srcdir)/%reldir%/test_json_format.sh_c1a23804c39b0f74642286d69865ee9d0961a58a.err \ $(srcdir)/%reldir%/test_json_format.sh_c1a23804c39b0f74642286d69865ee9d0961a58a.out \ $(srcdir)/%reldir%/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.err \ diff --git a/test/expected/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.err b/test/expected/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.out b/test/expected/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.out new file mode 100644 index 00000000..db5fbef7 --- /dev/null +++ b/test/expected/test_json_format.sh_ad3a238d03493de305544f9b30a0c69d4f474d3a.out @@ -0,0 +1 @@ +87.226.160.250 2022-10-30T00:00:02.000 "HEAD / HTTP/1.1" 301 570 diff --git a/test/logfile_cloudflare.json b/test/logfile_cloudflare.json new file mode 100644 index 00000000..980948f0 --- /dev/null +++ b/test/logfile_cloudflare.json @@ -0,0 +1 @@ +{"ClientIP":"87.226.160.250","ClientRequestHost":"www.ripe.net","ClientRequestMethod":"HEAD","ClientRequestURI":"/","EdgeEndTimestamp":"2022-10-30T00:00:02Z","EdgeResponseBytes":570,"EdgeResponseStatus":301,"EdgeStartTimestamp":"2022-10-30T00:00:02Z","RayID":"761fde4e984e5ab2","CacheCacheStatus":"unknown","CacheTieredFill":false,"CacheResponseBytes":0,"CacheResponseStatus":0,"FirewallMatchesActions":[],"FirewallMatchesRuleIDs":[],"FirewallMatchesSources":[],"OriginResponseBytes":0,"OriginResponseDurationMs":0,"OriginResponseHTTPExpires":"","OriginResponseHTTPLastModified":"","OriginResponseHeaderReceiveDurationMs":0,"OriginResponseStatus":0,"OriginResponseTime":0,"OriginDNSResponseTimeMs":0,"OriginIP":"","OriginRequestHeaderSendDurationMs":0,"OriginSSLProtocol":"unknown","OriginTCPHandshakeDurationMs":0,"OriginTLSHandshakeDurationMs":0,"WAFAction":"unknown","WAFFlags":"0","WAFMatchedVar":"","WAFProfile":"unknown","WAFRuleID":"","WAFRuleMessage":"","ClientASN":12389,"ClientCountry":"ru","ClientDeviceType":"desktop","ClientIPClass":"noRecord","ClientRequestBytes":567,"ClientRequestPath":"/","ClientRequestProtocol":"HTTP/1.1","ClientRequestUserAgent":"","ClientRequestReferer":"","ClientRequestScheme":"http","ClientRequestSource":"eyeball"} diff --git a/test/test_json_format.sh b/test/test_json_format.sh index 496f4d78..8a50ad29 100644 --- a/test/test_json_format.sh +++ b/test/test_json_format.sh @@ -147,3 +147,6 @@ run_cap_test ${lnav_test} -n \ run_cap_test ${lnav_test} -n \ ${test_dir}/logfile_bunyan.0 + +run_cap_test ${lnav_test} -n \ + ${test_dir}/logfile_cloudflare.json