Archives
/
bathyscaphe
mirror of https://github.com/creekorful/bathyscaphe
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrate to zerolog

Browse Source 
Closes: #20
pull/24/head
Aloïs Micard 4 years ago
parent 589f3e2b07
commit 325c6ef175
No known key found for this signature in database
GPG Key ID: 1A0EB82F071F5EFE
11 changed files with 136 additions and 113 deletions
Show Stats Download Patch File Download Diff File

7
go.mod
Unescape Escape View File

@ -6,10 +6,11 @@ require (
github.com/PuerkitoBio/purell v1.1.1
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/elastic/go-elasticsearch/v7 v7.6.0
github.com/golang/protobuf v1.4.2 // indirect
github.com/labstack/echo/v4 v4.1.16
github.com/labstack/gommon v0.3.0
github.com/nats-io/nats.go v1.9.2
github.com/sirupsen/logrus v1.5.0
github.com/nats-io/nats-server/v2 v2.1.8 // indirect
github.com/nats-io/nats.go v1.10.0
github.com/rs/zerolog v1.20.0
github.com/urfave/cli/v2 v2.2.0
github.com/valyala/fasthttp v1.9.0
mvdan.cc/xurls/v2 v2.1.0

53
go.sum
Unescape Escape View File

@ -3,22 +3,29 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA=
github.com/elastic/go-elasticsearch/v7 v7.6.0 h1:sYpGLpEFHgLUKLsZUBfuaVI9QgHjS3JdH9fX4/z8QI8=
github.com/elastic/go-elasticsearch/v7 v7.6.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs=
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI=
github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o=
github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
@ -32,23 +39,27 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
github.com/nats-io/nats.go v1.9.2 h1:oDeERm3NcZVrPpdR/JpGdWHMv3oJ8yY30YwxKq+DU2s=
github.com/nats-io/nats.go v1.9.2/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE=
github.com/nats-io/nats-server/v2 v2.1.8 h1:d5GoJA6W7vQkmt99Nfdeie3pEFFUEjIwt1YZp50DkIQ=
github.com/nats-io/nats-server/v2 v2.1.8/go.mod h1:rbRrRE/Iv93O/rUvZ9dh4NfT0Cm9HWjW/BqOWLGgYiE=
github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY=
github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA=
github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
@ -66,15 +77,16 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
@ -84,8 +96,21 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
mvdan.cc/xurls v1.1.0 h1:kj0j2lonKseISJCiq1Tfk+iTv65dDGCl0rTbanXJGGc=
mvdan.cc/xurls/v2 v2.1.0 h1:KaMb5GLhlcSX+e+qhbRJODnUUBvlw01jt4yrjFIHAuA=
mvdan.cc/xurls/v2 v2.1.0/go.mod h1:5GrSd9rOnKOpZaji1OZLYL/yeAAtGDlo/cFe+8K5n8E=

48
internal/api/api.go
Unescape Escape View File

@ -5,14 +5,14 @@ import (
"context"
"encoding/base64"
"encoding/json"
"github.com/creekorful/trandoshan/internal/util/log"
"github.com/creekorful/trandoshan/internal/util/logging"
natsutil "github.com/creekorful/trandoshan/internal/util/nats"
"github.com/creekorful/trandoshan/pkg/proto"
"github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esapi"
"github.com/labstack/echo/v4"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"net/http"
"regexp"
@ -39,7 +39,7 @@ func GetApp() *cli.App {
Version: "0.2.0",
Usage: "Trandoshan API process",
Flags: []cli.Flag{
log.GetLogFlag(),
logging.GetLogFlag(),
&cli.StringFlag{
Name: "nats-uri",
Usage: "URI to the NATS server",
@ -56,20 +56,20 @@ func GetApp() *cli.App {
}
func execute(ctx *cli.Context) error {
log.ConfigureLogger(ctx)
logging.ConfigureLogger(ctx)
e := echo.New()
e.HideBanner = true
logrus.Infof("Starting tdsh-api v%s", ctx.App.Version)
log.Info().Str("ver", ctx.App.Version).Msg("Starting tdsh-api")
logrus.Debugf("Using elasticsearch server at: %s", ctx.String("elasticsearch-uri"))
logrus.Debugf("Using NATS server at: %s", ctx.String("nats-uri"))
log.Debug().Str("uri", ctx.String("elasticsearch-uri")).Msg("Using Elasticsearch server")
log.Debug().Str("uri", ctx.String("nats-uri")).Msg("Using NATS server")
// Connect to the NATS server
nc, err := nats.Connect(ctx.String("nats-uri"))
if err != nil {
logrus.Errorf("Error while connecting to NATS server %s: %s", ctx.String("nats-uri"), err)
log.Err(err).Str("uri", ctx.String("nats-uri")).Msg("Error while connecting to NATS server")
return err
}
defer nc.Close()
@ -77,7 +77,7 @@ func execute(ctx *cli.Context) error {
// Create Elasticsearch client
es, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{ctx.String("elasticsearch-uri")}})
if err != nil {
logrus.Errorf("Error while creating elasticsearch client: %s", err)
log.Err(err).Msg("Error while creating ES client")
return err
}
@ -86,7 +86,7 @@ func execute(ctx *cli.Context) error {
e.POST("/v1/resources", addResource(es))
e.POST("/v1/urls", addURL(nc))
logrus.Info("Successfully initialized tdsh-api. Waiting for requests")
log.Info().Msg("Successfully initialized tdsh-api. Waiting for requests")
return e.Start(":8080")
}
@ -96,7 +96,7 @@ func searchResources(es *elasticsearch.Client) echo.HandlerFunc {
b64URL := c.QueryParam("url")
b, err := base64.URLEncoding.DecodeString(b64URL)
if err != nil {
logrus.Errorf("Error while decoding URL: %s", err)
log.Err(err).Msg("Error while decoding URL")
return c.NoContent(http.StatusInternalServerError)
}
@ -109,7 +109,7 @@ func searchResources(es *elasticsearch.Client) echo.HandlerFunc {
},
}
if err := json.NewEncoder(&buf).Encode(query); err != nil {
logrus.Errorf("Error encoding query: %s", err)
log.Err(err).Msg("Error encoding query")
return c.NoContent(http.StatusInternalServerError)
}
@ -120,10 +120,12 @@ func searchResources(es *elasticsearch.Client) echo.HandlerFunc {
es.Search.WithBody(&buf),
)
if err != nil || (res.IsError() && res.StatusCode != http.StatusNotFound) {
logrus.Errorf("Error getting response from ES: %s", err)
evt := log.Err(err)
if res != nil {
logrus.Errorf("Received status code: %d", res.StatusCode)
evt.Int("status", res.StatusCode)
}
evt.Msg("Error getting response from ES")
return c.NoContent(http.StatusInternalServerError)
}
@ -135,7 +137,7 @@ func searchResources(es *elasticsearch.Client) echo.HandlerFunc {
var resp map[string]interface{}
if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
logrus.Errorf("Error parsing the response body: %s", err)
log.Err(err).Msg("Error parsing the response body from ES")
return c.NoContent(http.StatusInternalServerError)
}
@ -164,11 +166,11 @@ func addResource(es *elasticsearch.Client) echo.HandlerFunc {
return func(c echo.Context) error {
var resourceDto proto.ResourceDto
if err := json.NewDecoder(c.Request().Body).Decode(&resourceDto); err != nil {
logrus.Errorf("Error while un-marshaling resource: %s", err)
log.Err(err).Msg("Error while un-marshaling resource")
return c.NoContent(http.StatusUnprocessableEntity)
}
logrus.Debugf("Saving resource %s", resourceDto.URL)
log.Debug().Str("url", resourceDto.URL).Msg("Saving resource")
// TODO store on file system
@ -183,7 +185,7 @@ func addResource(es *elasticsearch.Client) echo.HandlerFunc {
// Serialize document into json
docBytes, err := json.Marshal(&doc)
if err != nil {
logrus.Errorf("Error while serializing document into json: %s", err)
log.Err(err).Msg("Error while serializing document into json")
return c.NoContent(http.StatusInternalServerError)
}
@ -195,12 +197,12 @@ func addResource(es *elasticsearch.Client) echo.HandlerFunc {
}
res, err := req.Do(context.Background(), es)
if err != nil {
logrus.Errorf("Error while creating elasticsearch index: %s", err)
log.Err(err).Msg("Error while creating elasticsearch index")
return c.NoContent(http.StatusInternalServerError)
}
defer res.Body.Close()
logrus.Debugf("Successfully saved resource %s", resourceDto.URL)
log.Debug().Str("url", resourceDto.URL).Msg("Successfully saved resource")
return c.NoContent(http.StatusCreated)
}
@ -210,17 +212,17 @@ func addURL(nc *nats.Conn) echo.HandlerFunc {
return func(c echo.Context) error {
var url string
if err := json.NewDecoder(c.Request().Body).Decode(&url); err != nil {
logrus.Errorf("Error while un-marshaling url: %s", err)
log.Err(err).Msg("Error while un-marshaling URL")
return c.NoContent(http.StatusUnprocessableEntity)
}
// Publish the URL
if err := natsutil.PublishJSON(nc, proto.URLFoundSubject, &proto.URLFoundMsg{URL: url}); err != nil {
logrus.Errorf("Unable to publish URL: %s", err)
log.Err(err).Msg("Unable to publish URL")
return c.NoContent(http.StatusInternalServerError)
}
logrus.Debugf("Successfully published URL: %s", url)
log.Debug().Str("url", url).Msg("Successfully published URL")
return nil
}

30
internal/crawler/crawler.go
Unescape Escape View File

@ -2,11 +2,11 @@ package crawler
import (
"crypto/tls"
"github.com/creekorful/trandoshan/internal/util/log"
"github.com/creekorful/trandoshan/internal/util/logging"
natsutil "github.com/creekorful/trandoshan/internal/util/nats"
"github.com/creekorful/trandoshan/pkg/proto"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpproxy"
@ -24,7 +24,7 @@ func GetApp() *cli.App {
Version: "0.2.0",
Usage: "Trandoshan crawler process",
Flags: []cli.Flag{
log.GetLogFlag(),
logging.GetLogFlag(),
&cli.StringFlag{
Name: "nats-uri",
Usage: "URI to the NATS server",
@ -51,13 +51,13 @@ func GetApp() *cli.App {
}
func execute(ctx *cli.Context) error {
log.ConfigureLogger(ctx)
logging.ConfigureLogger(ctx)
logrus.Infof("Starting tdsh-crawler v%s", ctx.App.Version)
log.Info().Str("ver", ctx.App.Version).Msg("Starting tdsh-crawler")
logrus.Debugf("Using NATS server at: %s", ctx.String("nats-uri"))
logrus.Debugf("Using TOR proxy at: %s", ctx.String("tor-uri"))
logrus.Debugf("Allowed content types: %s", ctx.StringSlice("allowed-ct"))
log.Debug().Str("uri", ctx.String("nats-uri")).Msg("Using NATS server")
log.Debug().Str("uri", ctx.String("tor-uri")).Msg("Using TOR proxy")
log.Debug().Strs("content-types", ctx.StringSlice("allowed-ct")).Msg("Allowed content types")
// Create the HTTP client
httpClient := &fasthttp.Client{
@ -77,7 +77,7 @@ func execute(ctx *cli.Context) error {
}
defer sub.Close()
logrus.Info("Successfully initialized tdsh-crawler. Waiting for URLs")
log.Info().Msg("Successfully initialized tdsh-crawler. Waiting for URLs")
if err := sub.QueueSubscribe(proto.URLTodoSubject, "crawlers", handleMessage(httpClient, ctx.StringSlice("allowed-ct"))); err != nil {
return err
@ -93,7 +93,7 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) na
return err
}
logrus.Debugf("Processing URL: %s", urlMsg.URL)
log.Debug().Str("url", urlMsg.URL).Msg("Processing URL")
// Query the website
req := fasthttp.AcquireRequest()
@ -104,7 +104,7 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) na
req.SetRequestURI(urlMsg.URL)
if err := httpClient.Do(req, resp); err != nil {
logrus.Errorf("Error while querying website: %s", err)
log.Err(err).Msg("Error while crawling website")
return err
}
@ -119,7 +119,7 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) na
}
if !allowed {
logrus.Debugf("Discarding forbidden content type: %s", contentType)
log.Debug().Str("content-type", contentType).Msg("Discarding forbidden content type")
return nil
}
@ -131,7 +131,7 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) na
Body: body,
}
if err := natsutil.PublishJSON(nc, proto.ResourceSubject, &res); err != nil {
logrus.Errorf("Error while publishing resource body: %s", err)
log.Err(err).Msg("Error while publishing resource body")
}
// Extract URLs
@ -140,10 +140,10 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) na
// Publish found URLs
for _, url := range urls {
logrus.Tracef("Found URL: %s", url)
log.Trace().Str("url", url).Msg("Found URL")
if err := natsutil.PublishJSON(nc, proto.URLFoundSubject, &proto.URLFoundMsg{URL: url}); err != nil {
logrus.Errorf("Error while publishing URL: %s", err)
log.Err(err).Msg("Error while publishing URL")
}
}

21
internal/feeder/feeder.go
Unescape Escape View File

@ -3,8 +3,8 @@ package feeder
import (
"fmt"
"github.com/creekorful/trandoshan/internal/util/http"
"github.com/creekorful/trandoshan/internal/util/log"
"github.com/sirupsen/logrus"
"github.com/creekorful/trandoshan/internal/util/logging"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)
@ -15,7 +15,7 @@ func GetApp() *cli.App {
Version: "0.2.0",
Usage: "Trandoshan feeder process",
Flags: []cli.Flag{
log.GetLogFlag(),
logging.GetLogFlag(),
&cli.StringFlag{
Name: "api-uri",
Usage: "URI to the API server",
@ -32,25 +32,22 @@ func GetApp() *cli.App {
}
func execute(ctx *cli.Context) error {
log.ConfigureLogger(ctx)
logging.ConfigureLogger(ctx)
logrus.Infof("Starting tdsh-feeder v%s", ctx.App.Version)
log.Info().Str("ver", ctx.App.Version).Msg("Starting tdsh-feeder")
logrus.Debugf("Using API server at: %s", ctx.String("api-uri"))
log.Debug().Str("uri", ctx.String("api-uri")).Msg("Using API server")
apiURL := fmt.Sprintf("%s/v1/urls", ctx.String("api-uri"))
c := http.Client{}
res, err := c.JSONPost(apiURL, ctx.String("url"), nil)
_, err := c.JSONPost(apiURL, ctx.String("url"), nil)
if err != nil {
logrus.Errorf("Unable to publish URL: %s", err)
if res != nil {
logrus.Errorf("Received status code: %d", res.StatusCode)
}
log.Err(err).Msg("Unable to publish URL")
return err
}
logrus.Infof("URL %s successfully sent to the crawler", ctx.String("url"))
log.Info().Str("url", ctx.String("url")).Msg("URL successfully sent to the crawler")
return nil
}

25
internal/persister/persister.go
Unescape Escape View File

@ -3,11 +3,11 @@ package persister
import (
"fmt"
"github.com/creekorful/trandoshan/internal/util/http"
"github.com/creekorful/trandoshan/internal/util/log"
"github.com/creekorful/trandoshan/internal/util/logging"
natsutil "github.com/creekorful/trandoshan/internal/util/nats"
"github.com/creekorful/trandoshan/pkg/proto"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)
@ -18,7 +18,7 @@ func GetApp() *cli.App {
Version: "0.2.0",
Usage: "Trandoshan persister process",
Flags: []cli.Flag{
log.GetLogFlag(),
logging.GetLogFlag(),
&cli.StringFlag{
Name: "nats-uri",
Usage: "URI to the NATS server",
@ -35,12 +35,12 @@ func GetApp() *cli.App {
}
func execute(ctx *cli.Context) error {
log.ConfigureLogger(ctx)
logging.ConfigureLogger(ctx)
logrus.Infof("Starting tdsh-persister v%s", ctx.App.Version)
log.Info().Str("ver", ctx.App.Version).Msg("Starting tdsh-persister")
logrus.Debugf("Using NATS server at: %s", ctx.String("nats-uri"))
logrus.Debugf("Using API server at: %s", ctx.String("api-uri"))
log.Debug().Str("uri", ctx.String("nats-uri")).Msg("Using NATS server")
log.Debug().Str("uri", ctx.String("api-uri")).Msg("Using API server")
// Create the HTTP client
httpClient := &http.Client{}
@ -52,7 +52,7 @@ func execute(ctx *cli.Context) error {
}
defer sub.Close()
logrus.Info("Successfully initialized tdsh-persister. Waiting for resources")
log.Info().Msg("Successfully initialized tdsh-persister. Waiting for resources")
if err := sub.QueueSubscribe(proto.ResourceSubject, "persisters", handleMessage(httpClient, ctx.String("api-uri"))); err != nil {
return err
@ -68,7 +68,7 @@ func handleMessage(httpClient *http.Client, apiURI string) natsutil.MsgHandler {
return err
}
logrus.Debugf("Processing resource: %s", resMsg.URL)
log.Debug().Str("url", resMsg.URL).Msg("Processing resource")
url := fmt.Sprintf("%s/v1/resources", apiURI)
r, err := httpClient.JSONPost(url, &proto.ResourceDto{
@ -77,14 +77,11 @@ func handleMessage(httpClient *http.Client, apiURI string) natsutil.MsgHandler {
}, nil)
if err != nil || r.StatusCode != http.StatusCreated {
logrus.Errorf("Error while sending resource to the API: %s", err)
if r != nil {
logrus.Errorf("Received status code: %d", r.StatusCode)
}
log.Err(err).Msg("Error while sending resource to the API")
return err
}
logrus.Debugf("Successfully processed resource: %s", resMsg.URL)
log.Debug().Str("url", resMsg.URL).Msg("Successfully processed resource")
return nil
}

33
internal/scheduler/scheduler.go
Unescape Escape View File

@ -5,11 +5,11 @@ import (
"fmt"
"github.com/PuerkitoBio/purell"
"github.com/creekorful/trandoshan/internal/util/http"
"github.com/creekorful/trandoshan/internal/util/log"
"github.com/creekorful/trandoshan/internal/util/logging"
natsutil "github.com/creekorful/trandoshan/internal/util/nats"
"github.com/creekorful/trandoshan/pkg/proto"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"net/url"
"strings"
@ -22,7 +22,7 @@ func GetApp() *cli.App {
Version: "0.2.0",
Usage: "Trandoshan scheduler process",
Flags: []cli.Flag{
log.GetLogFlag(),
logging.GetLogFlag(),
&cli.StringFlag{
Name: "nats-uri",
Usage: "URI to the NATS server",
@ -39,12 +39,12 @@ func GetApp() *cli.App {
}
func execute(ctx *cli.Context) error {
log.ConfigureLogger(ctx)
logging.ConfigureLogger(ctx)
logrus.Infof("Starting tdsh-scheduler v%s", ctx.App.Version)
log.Info().Str("ver", ctx.App.Version).Msg("Starting tdsh-scheduler")
logrus.Debugf("Using NATS server at: %s", ctx.String("nats-uri"))
logrus.Debugf("Using API server at: %s", ctx.String("api-uri"))
log.Debug().Str("uri", ctx.String("nats-uri")).Msg("Using NATS server")
log.Debug().Str("uri", ctx.String("api-uri")).Msg("Using API server")
// Create the HTTP client
httpClient := &http.Client{}
@ -56,7 +56,7 @@ func execute(ctx *cli.Context) error {
}
defer sub.Close()
logrus.Info("Successfully initialized tdsh-scheduler. Waiting for URLs")
log.Info().Msg("Successfully initialized tdsh-scheduler. Waiting for URLs")
if err := sub.QueueSubscribe(proto.URLFoundSubject, "schedulers", handleMessage(httpClient, ctx.String("api-uri"))); err != nil {
return err
@ -72,16 +72,16 @@ func handleMessage(httpClient *http.Client, apiURI string) natsutil.MsgHandler {
return err
}
logrus.Debugf("Processing URL: %s", urlMsg.URL)
log.Debug().Str("url", urlMsg.URL).Msg("Processing URL: %s")
normalizedURL, err := normalizeURL(urlMsg.URL)
if err != nil {
logrus.Errorf("Error while normalizing URL: %s", err)
log.Err(err).Msg("Error while normalizing URL")
return err
}
// Make sure URL is valid .onion
if !strings.Contains(normalizedURL.Host, ".onion") {
logrus.Debugf("Url %s is not a valid hidden service", normalizedURL)
log.Debug().Stringer("url", normalizedURL).Msg("URL is not a valid hidden service")
return err
}
@ -89,23 +89,20 @@ func handleMessage(httpClient *http.Client, apiURI string) natsutil.MsgHandler {
apiURL := fmt.Sprintf("%s/v1/resources?url=%s", apiURI, b64URI)
var urls []proto.ResourceDto
r, err := httpClient.JSONGet(apiURL, &urls)
_, err = httpClient.JSONGet(apiURL, &urls)
if err != nil {
logrus.Errorf("Error while searching URL: %s", err)
if r != nil {
logrus.Errorf("Received status code: %d", r.StatusCode)
}
log.Err(err).Msg("Error while searching URL")
return err
}
// No matches: schedule!
if len(urls) == 0 {
logrus.Debugf("%s should be scheduled", normalizedURL)
log.Debug().Stringer("url", normalizedURL).Msg("URL should be scheduled")
if err := natsutil.PublishJSON(nc, proto.URLTodoSubject, &proto.URLTodoMsg{URL: urlMsg.URL}); err != nil {
return fmt.Errorf("error while publishing URL: %s", err)
}
} else {
logrus.Tracef("%s should not scheduled", normalizedURL)
log.Trace().Stringer("url", normalizedURL).Msg("URL should not be scheduled")
}
return nil

6
internal/util/http/client.go
Unescape Escape View File

@ -3,7 +3,7 @@ package http
import (
"bytes"
"encoding/json"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"net/http"
)
@ -19,7 +19,7 @@ type Client struct {
// JSONGet perform a GET request and serialize response body into given interface if any
func (c *Client) JSONGet(url string, response interface{}) (*http.Response, error) {
logrus.Tracef("GET %s", url)
log.Trace().Str("verb", "GET").Str("url", url).Msg("")
r, err := c.client.Get(url)
if err != nil {
@ -35,7 +35,7 @@ func (c *Client) JSONGet(url string, response interface{}) (*http.Response, erro
// JSONPost perform a POST request and serialize request/response body into given interface if any
func (c *Client) JSONPost(url string, request, response interface{}) (*http.Response, error) {
logrus.Tracef("POST %s", url)
log.Trace().Str("verb", "POST").Str("url", url).Msg("")
var err error
var b []byte

14
internal/util/log/log.go → internal/util/logging/log.go
Unescape Escape View File

@ -1,8 +1,10 @@
package log
package logging
import (
"github.com/sirupsen/logrus"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"os"
)
// GetLogFlag return the CLI flag parameter used to setup application log level
@ -16,10 +18,12 @@ func GetLogFlag() *cli.StringFlag {
// ConfigureLogger configure the logger using given log level (read from cli context)
func ConfigureLogger(ctx *cli.Context) {
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
// Set application log level
if lvl, err := logrus.ParseLevel(ctx.String("log-level")); err == nil {
logrus.SetLevel(lvl)
if lvl, err := zerolog.ParseLevel(ctx.String("log-level")); err == nil {
zerolog.SetGlobalLevel(lvl)
} else {
logrus.SetLevel(logrus.InfoLevel)
zerolog.SetGlobalLevel(zerolog.InfoLevel)
}
}

2
internal/util/log/log_test.go → internal/util/logging/log_test.go
Unescape Escape View File

@ -1,4 +1,4 @@
package log
package logging
import (
"testing"

10
internal/util/nats/subscriber.go
Unescape Escape View File

@ -3,7 +3,7 @@ package nats
import (
"context"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
)
// MsgHandler represent an handler for a NATS subscriber
@ -18,7 +18,7 @@ type Subscriber struct {
func NewSubscriber(address string) (*Subscriber, error) {
nc, err := nats.Connect(address)
if err != nil {
logrus.Errorf("Error while connecting to NATS server %s: %s", address, err)
log.Err(err).Str("server-uri", address).Msg("Error while connecting to NATS server")
return nil, err
}
@ -32,7 +32,7 @@ func (qs *Subscriber) QueueSubscribe(subject, queue string, handler MsgHandler)
// Create the subscriber
sub, err := qs.nc.QueueSubscribeSync(subject, queue)
if err != nil {
logrus.Errorf("Error while reading message from NATS server: %s", err)
log.Err(err).Msg("Error while reading message from NATS server")
return err
}
@ -40,13 +40,13 @@ func (qs *Subscriber) QueueSubscribe(subject, queue string, handler MsgHandler)
// Read incoming message
msg, err := sub.NextMsgWithContext(context.Background())
if err != nil {
logrus.Warnf("Skipping current message because of error: %s", err)
log.Warn().Str("err", err.Error()).Msg("Skipping current message because of error")
continue
}
// ... And process it
if err := handler(qs.nc, msg); err != nil {
logrus.Warnf("Skipping current message because of error: %s", err)
log.Warn().Str("error", err.Error()).Msg("Skipping current message because of error")
continue
}
}

Write Preview
Loading…
Cancel
Save