Update shoutwiki.com spider and list

listsofwikis/mediawiki/shoutwiki-spider.py

@@ -1,55 +1,72 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright (C) 2014 WikiTeam developers
+# Copyright (C) 2021 Simon Liu
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
-import random
-import re
-import requests
 import time
+import requests
+from tqdm import tqdm
 
 def main():
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:24.0) Gecko/20100101 Firefox/24.0',
+    ids, wikis = [], []
+    gcont = 'tmp'
+    url = 'http://www.shoutwiki.com/w/api.php'
+
+    # grab wiki pages
+    params = {
+        'action': 'query',
+        'format': 'json',
+        'prop': 'info',
+        'generator': 'categorymembers',
+        'inprop': 'url',
+        'gcmtitle': 'Category:Flat_list_of_all_wikis',
+        'gcmlimit': 'max'
+    }
+    while gcont:
+        if gcont != 'tmp':
+            params['gcmcontinue'] = gcont
+        json = requests.get(url, params=params).json()
+        gcont = json['continue']['gcmcontinue'] if 'continue' in json else ''
+        query = json['query']['pages']
+        for wiki in query:
+            ids.append(wiki)
+
+    # grab wiki API
+    params = {
+        'action': 'query',
+        'format': 'json',
+        'prop': 'revisions',
+        'formatversion': '2',
+        'rvprop': 'content',
+        'rvslots': '*'
     }
-    swfrom = 1
-    swlimit = 500
-    while swfrom:
-        params = {
-            'action': 'listwikis',
-            'swfrom': swfrom,
-            'swlimit': swlimit,
-            'format': 'json',
-        }
-        url = 'http://www.shoutwiki.com/w/api.php'
-        r = requests.get(url, params=params, headers=headers)
-        jsonsites = json.loads(r.text)
-        
-        for site in jsonsites['query']['listwikis']:
-            siteid = int(site['id'])
-            siteurl = site['url']
-            print siteurl
-        
-        if len(jsonsites['query']['listwikis']) == int(swlimit):
-            #there are more
-            swfrom = siteid + 1
-        else:
-            swfrom = ''
-        
-        time.sleep(random.randint(3,10))
-    
+    for n in tqdm(range(0, len(ids), 50)):
+        params['pageids'] = '|'.join(ids[n:n+50])
+        json = requests.get(url, params=params).json()
+
+        for wiki in json['query']['pages']:
+            for val in wiki['revisions'][0]['slots']['main']['content'].split('\n|'):
+                if 'subdomain' in val:
+                    wikis.append('http://%s.shoutwiki.com/w/api.php' % val.split('subdomain =')[-1].strip())
+                    break
+
+        time.sleep(0.3)
+    wikis.sort()
+
+    with open('shoutwiki.com', 'w') as f:
+        f.write('\n'.join(wikis))
+
 if __name__ == '__main__':
     main()

listsofwikis/mediawiki/shoutwiki.com

@@ -471,8 +471,8 @@ http://eflarchives.shoutwiki.com/w/api.php
 http://efrabase.shoutwiki.com/w/api.php
 http://ejemploter.shoutwiki.com/w/api.php
 http://ekklesia.shoutwiki.com/w/api.php
-http://el.shoutwiki.com/w/api.php
 http://el.historic.shoutwiki.com/w/api.php
+http://el.shoutwiki.com/w/api.php
 http://elhuerto.shoutwiki.com/w/api.php
 http://elmwood.shoutwiki.com/w/api.php
 http://elpuerto.shoutwiki.com/w/api.php
@@ -550,16 +550,16 @@ http://en.writing.shoutwiki.com/w/api.php
 http://en.zootycoon.shoutwiki.com/w/api.php
 http://enciclo.shoutwiki.com/w/api.php
 http://encyclopaedia.shoutwiki.com/w/api.php
-http://encyclopedia.shoutwiki.com/w/api.php
 http://encyclopedia-of-war.shoutwiki.com/w/api.php
+http://encyclopedia.shoutwiki.com/w/api.php
 http://encyclopediaprogramsitrk.shoutwiki.com/w/api.php
 http://encycloshpik.shoutwiki.com/w/api.php
 http://endymion.shoutwiki.com/w/api.php
 http://enfermeria.shoutwiki.com/w/api.php
 http://enfrateam1.shoutwiki.com/w/api.php
 http://engineeringsciencewiki.shoutwiki.com/w/api.php
-http://english.shoutwiki.com/w/api.php
 http://english-learning.shoutwiki.com/w/api.php
+http://english.shoutwiki.com/w/api.php
 http://englishmoserschule.shoutwiki.com/w/api.php
 http://englishteaching.shoutwiki.com/w/api.php
 http://enquan.shoutwiki.com/w/api.php
@@ -573,8 +573,8 @@ http://er.shoutwiki.com/w/api.php
 http://eredivisiealternate.shoutwiki.com/w/api.php
 http://ereticopedia.shoutwiki.com/w/api.php
 http://erfrpg.shoutwiki.com/w/api.php
-http://es.amigos.shoutwiki.com/w/api.php
 http://es.amigos-imaginarios.shoutwiki.com/w/api.php
+http://es.amigos.shoutwiki.com/w/api.php
 http://es.aprender-a-aprender.shoutwiki.com/w/api.php
 http://es.audelina90cordova.shoutwiki.com/w/api.php
 http://es.aux-inf111-umsa.shoutwiki.com/w/api.php
@@ -677,8 +677,8 @@ http://fictionworld.shoutwiki.com/w/api.php
 http://fierybaytwio.shoutwiki.com/w/api.php
 http://figjamsbc.shoutwiki.com/w/api.php
 http://filmnoir.shoutwiki.com/w/api.php
-http://filosofia.shoutwiki.com/w/api.php
 http://filosofia-frases.shoutwiki.com/w/api.php
+http://filosofia.shoutwiki.com/w/api.php
 http://filosofiahistopoli.shoutwiki.com/w/api.php
 http://fim-the-comic.shoutwiki.com/w/api.php
 http://finalfantasy.shoutwiki.com/w/api.php
@@ -1332,8 +1332,8 @@ http://oazo.shoutwiki.com/w/api.php
 http://obiriku.shoutwiki.com/w/api.php
 http://ocastaneda.shoutwiki.com/w/api.php
 http://ocastaneda1.shoutwiki.com/w/api.php
-http://occitania.shoutwiki.com/w/api.php
 http://occitania-micronation.shoutwiki.com/w/api.php
+http://occitania.shoutwiki.com/w/api.php
 http://oceanologia.shoutwiki.com/w/api.php
 http://ocemporium.shoutwiki.com/w/api.php
 http://odireitopesquisa.shoutwiki.com/w/api.php
@@ -1720,16 +1720,16 @@ http://slwiki.shoutwiki.com/w/api.php
 http://smart-latam.shoutwiki.com/w/api.php
 http://smartphones.shoutwiki.com/w/api.php
 http://smashburros.shoutwiki.com/w/api.php
-http://sml.shoutwiki.com/w/api.php
 http://sml-heroes.shoutwiki.com/w/api.php
+http://sml.shoutwiki.com/w/api.php
 http://sn2ro.shoutwiki.com/w/api.php
 http://sniper02.shoutwiki.com/w/api.php
 http://so-ups.shoutwiki.com/w/api.php
 http://soapt.shoutwiki.com/w/api.php
 http://socialmediamanagers.shoutwiki.com/w/api.php
 http://sociologiaordinaria.shoutwiki.com/w/api.php
-http://software.shoutwiki.com/w/api.php
 http://software-engineering-api.shoutwiki.com/w/api.php
+http://software.shoutwiki.com/w/api.php
 http://softwareandgames.shoutwiki.com/w/api.php
 http://softwiki.shoutwiki.com/w/api.php
 http://solaris-es.shoutwiki.com/w/api.php
@@ -1957,8 +1957,8 @@ http://txroads.shoutwiki.com/w/api.php
 http://uavlaws.shoutwiki.com/w/api.php
 http://uba.shoutwiki.com/w/api.php
 http://ubuntudesdecero.shoutwiki.com/w/api.php
-http://ufo.shoutwiki.com/w/api.php
 http://ufo-wiki.shoutwiki.com/w/api.php
+http://ufo.shoutwiki.com/w/api.php
 http://ugi.shoutwiki.com/w/api.php
 http://uk.astrology.shoutwiki.com/w/api.php
 http://uk.entsiklopedia.shoutwiki.com/w/api.php