|
|
|
@ -1,7 +1,7 @@
|
|
|
|
|
#!/usr/bin/env python2
|
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
|
|
|
|
# Copyright (C) 2011-2014 WikiTeam developers
|
|
|
|
|
# Copyright (C) 2011-2015 WikiTeam developers
|
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
|
|
@ -26,7 +26,7 @@ import time
|
|
|
|
|
import unittest
|
|
|
|
|
import urllib
|
|
|
|
|
import urllib2
|
|
|
|
|
from dumpgenerator import delay, getImageNames, getPageTitles, getUserAgent, getWikiEngine, mwGetAPIAndIndex
|
|
|
|
|
from dumpgenerator import delay, getImageNames, getPageTitles, getUserAgent, getWikiEngine, mwGetAPIAndIndex, domain2prefix
|
|
|
|
|
|
|
|
|
|
class TestDumpgenerator(unittest.TestCase):
|
|
|
|
|
# Documentation
|
|
|
|
@ -91,7 +91,12 @@ class TestDumpgenerator(unittest.TestCase):
|
|
|
|
|
for index, api, filetocheck in tests:
|
|
|
|
|
# Testing with API
|
|
|
|
|
print '\nTesting', api
|
|
|
|
|
config_api = {'api': api, 'delay': 0}
|
|
|
|
|
config_api = {'api': api,
|
|
|
|
|
'delay': 0,
|
|
|
|
|
'retries': 5,
|
|
|
|
|
'date': "20150807",
|
|
|
|
|
}
|
|
|
|
|
config_api['path'] = './%s-%s-wikidump' % (domain2prefix(config=config_api, session=session), config_api['date'])
|
|
|
|
|
req = urllib2.Request(url=api, data=urllib.urlencode({'action': 'query', 'meta': 'siteinfo', 'siprop': 'statistics', 'format': 'json'}), headers={'User-Agent': getUserAgent()})
|
|
|
|
|
f = urllib2.urlopen(req)
|
|
|
|
|
imagecount = int(json.loads(f.read())['query']['statistics']['images'])
|
|
|
|
@ -104,7 +109,12 @@ class TestDumpgenerator(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
# Testing with index
|
|
|
|
|
print '\nTesting', index
|
|
|
|
|
config_index = {'index': index, 'delay': 0}
|
|
|
|
|
config_index = {'index': index,
|
|
|
|
|
'delay': 0,
|
|
|
|
|
'retries': 5,
|
|
|
|
|
'date': "20150807",
|
|
|
|
|
}
|
|
|
|
|
config_api['path'] = './%s-%s-wikidump' % (domain2prefix(config=config_api, session=session), config_api['date'])
|
|
|
|
|
req = urllib2.Request(url=api, data=urllib.urlencode({'action': 'query', 'meta': 'siteinfo', 'siprop': 'statistics', 'format': 'json'}), headers={'User-Agent': getUserAgent()})
|
|
|
|
|
f = urllib2.urlopen(req)
|
|
|
|
|
imagecount = int(json.loads(f.read())['query']['statistics']['images'])
|
|
|
|
@ -149,14 +159,28 @@ class TestDumpgenerator(unittest.TestCase):
|
|
|
|
|
# Testing with API
|
|
|
|
|
print '\nTesting', api
|
|
|
|
|
print 'Trying to parse', pagetocheck, 'with API'
|
|
|
|
|
config_api = {'api': api, 'delay': 0, 'namespaces': ['all'], 'exnamespaces': []}
|
|
|
|
|
config_api = {'api': api,
|
|
|
|
|
'delay': 0,
|
|
|
|
|
'namespaces': ['all'],
|
|
|
|
|
'exnamespaces': [],
|
|
|
|
|
'retries': 5,
|
|
|
|
|
'date': "20150807",
|
|
|
|
|
}
|
|
|
|
|
config_api['path'] = './%s-%s-wikidump' % (domain2prefix(config=config_api, session=session), config_api['date'])
|
|
|
|
|
result_api = getPageTitles(config=config_api, session=session)
|
|
|
|
|
self.assertTrue(pagetocheck in result_api)
|
|
|
|
|
|
|
|
|
|
# Testing with index
|
|
|
|
|
print 'Testing', index
|
|
|
|
|
print 'Trying to parse', pagetocheck, 'with index'
|
|
|
|
|
config_index = {'index': index, 'delay': 0, 'namespaces': ['all'], 'exnamespaces': []}
|
|
|
|
|
config_index = {'index': index,
|
|
|
|
|
'delay': 0,
|
|
|
|
|
'namespaces': ['all'],
|
|
|
|
|
'exnamespaces': [],
|
|
|
|
|
'retries': 5,
|
|
|
|
|
'date': "20150807",
|
|
|
|
|
}
|
|
|
|
|
config_api['path'] = './%s-%s-wikidump' % (domain2prefix(config=config_api, session=session), config_api['date'])
|
|
|
|
|
result_index = getPageTitles(config=config_index, session=session)
|
|
|
|
|
self.assertTrue(pagetocheck in result_index)
|
|
|
|
|
self.assertEqual(len(result_api), len(result_index))
|
|
|
|
|