Fix nonexistent error handling crawlrss

.gitignore

@@ -8,7 +8,7 @@ why.html
 generate-top-torrents
 import-magnetico-db/import-magnetico-db
 crawl-rss/crawl-rss
-seedleech-daemon/seedleech-daemon
+seedleech/seedleech
 spider/spider
 .vscode/
 api/api

README.md

@@ -11,13 +11,15 @@ Maybe, open an issue. Be sure to demonstrate an effort that you tried to solve t
 # Setup
 
 Here's what the setup looks like rn:
-- VPS, Debian Buster, 4 GB RAM
+- VPS, Debian Bullseye, 8 GB RAM
   - user with username nextgen on the server
 - my laptop w/ Linux
   - Go toolchain installed
-  - node v10.15 & npm
+  - node & npm
   - Python 3 (required only for index-generator/fix-metajson.py)
 
+Read the server-setup.md file for more precise info.
+
 The programs create their own tables in the DB that they need. Database name is "nextgen". You need to create the materialized views (fresh and search). You can find some useful SQL code in snippets.sql.
 
 Each of the daemons (api, crawl-rss, seedleech-daemon) is its own standalone Go package and resulting binary. You have to compile the binaries yourself. There are systemd .service files available for each of the daemons.

api/main.go

@@ -133,5 +133,8 @@ func main() {
 		w.Write(marshaledResults)
 	})
 
-	http.ListenAndServe(":8000", nil)
+	err := http.ListenAndServe(":8000", nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 }

crawl-rss/tpb.go

@@ -24,9 +24,19 @@ func parseApibayJSON(url string) []Torrent {
 	defer httpresp.Body.Close()
 	body, err := ioutil.ReadAll(httpresp.Body)
 
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+
 	var resp []ApibayTorrent
 	err = json.Unmarshal(body, &resp)
 
+	if err != nil {
+		log.Println(err)
+		return nil
+	}
+
 	var torrents []Torrent
 	for _, apibayTorr := range resp {
 		torrents = append(torrents, Torrent{apibayTorr.Info_hash, apibayTorr.Name, apibayTorr.Size})

seedleech/config.toml

@@ -1,4 +1,4 @@
-trackers = ["udp://tracker.coppersurfer.tk:6969", "udp://tracker.leechers-paradise.org:6969/announce", "udp://tracker.internetwarriors.net:1337/announce","udp://tracker.opentrackr.org:1337/announce","udp://tracker.cyberia.is:6969/announce","udp://9.rarbg.to:2920/announce"]
+trackers = ["udp://tracker.coppersurfer.tk:6969", "udp://tracker.leechers-paradise.org:6969/announce", "udp://tracker.opentrackr.org:1337/announce","udp://tracker.cyberia.is:6969/announce","udp://tracker.torrent.eu.org:451"]
 waitTime = "250ms"
 logInterval = "1h"
 

seedleech/seedleech.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
 	"os/signal"
 	"strconv"
@@ -104,7 +105,7 @@ func runWorkFetcher(trackerRequests chan []string, tracker string, minseed int,
 		if minseed != 0 {
 			rows, err = db.Query("SELECT infohash FROM trackerdata WHERE tracker = $1 AND seeders > $2 AND scraped < $3 LIMIT 740", tracker, minseed, freshlimit)
 		} else {
-			//time.Sleep(time.Duration(int64(rand.Intn(12000)) * int64(time.Second))) //sleep for random time between 0 mins and 200 mins
+			time.Sleep(time.Duration(int64(rand.Intn(12000)) * int64(time.Second))) //sleep for random time between 0 mins and 200 mins
 			rows, err = db.Query("SELECT infohash FROM torrent WHERE NOT EXISTS (SELECT from trackerdata WHERE infohash = torrent.infohash AND tracker = $1 AND scraped > $2)", tracker, freshlimit)
 		}
 		if err != nil {
@@ -129,7 +130,7 @@ func runWorkFetcher(trackerRequests chan []string, tracker string, minseed int,
 			}
 		}
 		trackerRequests <- infohashes
-		time.Sleep(time.Minute)
+		time.Sleep(10 * time.Minute)
 	}
 }
 

server-setup.md

@@ -0,0 +1,51 @@
+
+
+# Run as root
+
+Assumming Debian 11 Bullseye
+
+```apt update
+apt upgrade
+apt install -y htop kitty-terminfo screenfetch postgresql-13 mosh nload
+apt install -y debian-keyring debian-archive-keyring apt-transport-https
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | tee /etc/apt/trusted.gpg.d/caddy-stable.asc
+curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
+apt update
+apt install caddy
+
+mkdir /www
+adduser nextgen
+```
+
+### /etc/caddy/Caddyfile
+
+```
+http://torrentparadise.ml, http://torrent-paradise.ml {
+        reverse_proxy /api/* http://localhost:8000
+        root * /www/torrent-paradise.ml/
+        file_server
+}
+```
+
+### /etc/postgresql/13/main/postgresql.conf
+
+Use https://pgtune.leopard.in.ua/
+
+Just get the optimal settings and paste them at the end of the file, they override the defaults.
+
+### Set up nextgen user and database in Postgres
+
+```
+postgres $ createuser -d nextgen
+nextgen $ createdb nextgen
+```
+
+### Ship compiled static executables to server
+
+You can either build it on the server or just ship the binaries to the server via scp. In the end, you need binaries in /home/nextgen and .service files in /etc/systemd/system/.
+
+Might come in handy: a way to build go binaries truly statically (incl glibc) `CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' .`
+
+### Ship contents of static/ to /www/torrent-paradise.ml
+
+Use scp.

static/index.html

@@ -38,11 +38,14 @@
             <br><br>
             <ul>
                 <li>designed for humans, doesn't make you angry</li>
-                <li>over 10 million torrents (up from 2 million)</li>
+                <li>over 10 million torrents</li>
                 <li>new releases added from RSS feeds</li>
                 <li>freshest seed/leech counts</li>
             </ul>
         </div>
+        <div class="notification is-info">
+            Now faster than ever.
+        </div>
         <div class="field has-addons">
             <div class="control is-expanded">
                 <input class="input" placeholder='' type='text' id="searchbox" />