|
|
@ -9,7 +9,6 @@ import (
|
|
|
|
"time"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
|
|
"github.com/lib/pq"
|
|
|
|
"github.com/lib/pq"
|
|
|
|
_ "github.com/lib/pq"
|
|
|
|
|
|
|
|
"github.com/mmcdole/gofeed"
|
|
|
|
"github.com/mmcdole/gofeed"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
@ -25,9 +24,13 @@ func main() {
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
torrents := CrawlYts()
|
|
|
|
torrents := CrawlYts()
|
|
|
|
for _, torrent := range torrents {
|
|
|
|
for _, torrent := range torrents {
|
|
|
|
addTorrent(db, torrent, &crawled)
|
|
|
|
addTorrent(db, torrent, crawled)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
time.Sleep(time.Minute * 45)
|
|
|
|
torrents = CrawlEztv()
|
|
|
|
|
|
|
|
for _, torrent := range torrents {
|
|
|
|
|
|
|
|
addTorrent(db, torrent, crawled)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
time.Sleep(time.Minute * 60)
|
|
|
|
go refresh(db)
|
|
|
|
go refresh(db)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -37,15 +40,15 @@ func refresh(db *sql.DB) {
|
|
|
|
db.Exec("REFRESH MATERIALIZED VIEW CONCURRENTLY search")
|
|
|
|
db.Exec("REFRESH MATERIALIZED VIEW CONCURRENTLY search")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func addTorrent(db *sql.DB, torr Torrent, crawled *map[string]bool) {
|
|
|
|
func addTorrent(db *sql.DB, torr Torrent, crawled map[string]bool) {
|
|
|
|
if !(*crawled)[string(torr.Infohash)] {
|
|
|
|
if !(crawled[string(torr.Infohash)]) {
|
|
|
|
_, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", torr.Infohash, torr.Name, torr.Length)
|
|
|
|
_, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", torr.Infohash, torr.Name, torr.Length)
|
|
|
|
if err, ok := err.(*pq.Error); ok { //dark magic
|
|
|
|
if err, ok := err.(*pq.Error); ok { //dark magic
|
|
|
|
if err.Code != "23505" {
|
|
|
|
if err.Code != "23505" {
|
|
|
|
log.Fatal(err)
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
(*crawled)[torr.Infohash] = true
|
|
|
|
crawled[torr.Infohash] = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -69,6 +72,24 @@ func CrawlYts() []Torrent {
|
|
|
|
return torrents
|
|
|
|
return torrents
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func CrawlEztv() []Torrent { //maybe is there some kind of interface that this can share with CrawlYts? This function has the same signature and purpose.
|
|
|
|
|
|
|
|
fp := gofeed.NewParser()
|
|
|
|
|
|
|
|
feed, err := fp.ParseURL("https://eztv.io/ezrss.xml")
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Fatal(err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
var torrents []Torrent
|
|
|
|
|
|
|
|
for _, item := range feed.Items {
|
|
|
|
|
|
|
|
size, err := strconv.Atoi(item.Extensions["torrent"]["contentLength"][0].Value)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Print(err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
torrents = append(torrents, Torrent{item.Extensions["torrent"]["infoHash"][0].Value, item.Extensions["torrent"]["fileName"][0].Value, size})
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return torrents
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Parses torrent length from YTS description
|
|
|
|
// Parses torrent length from YTS description
|
|
|
|
func parseSizeYts(description string) (int, error) {
|
|
|
|
func parseSizeYts(description string) (int, error) {
|
|
|
|
s := strings.Split(description, "<br />Size: ")
|
|
|
|
s := strings.Split(description, "<br />Size: ")
|
|
|
|