rnhmjoj
/
pirate-get
mirror of https://github.com/vikstrous/pirate-get
1
0
Fork 0
Code Issues Releases Wiki Activity

reimplement fetching multiple pages

api
Michele Guerini Rocco 2020-05-25 10:55:26 +02:00
parent 3eb240f316
commit 35ecb3041f
Signed by: rnhmjoj
GPG Key ID: BFBAF4C975F76450
3 changed files with 41 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pirate/pirate.py
View File

@ -134,14 +134,18 @@ def parse_args(args_in):
help='specify a sort option', default='SeedersDsc')
parser.add_argument('-R', '--recent',
action='store_true',
help='torrents uploaded in the last 48hours.'
help='torrents uploaded in the last 48hours. '
'*ignored in searches*')
parser.add_argument('-l', '--list-categories',
action='store_true',
help='list categories')
parser.add_argument('--list-sorts', '--list_sorts',
action='store_true',
help='list Sortable Types')
help='list types by which results can be sorted')
parser.add_argument('-p', '--pages',
default=1, type=int,
help='the number of pages to fetch. '
'(only used with --recent)')
parser.add_argument('-L', '--local', dest='database',
help='a csv file containing the Pirate Bay database '
'downloaded from '
@ -271,6 +275,7 @@ def connect_mirror(mirror, printer, args):
url = pirate.torrent.find_api(mirror, args.timeout)
results = pirate.torrent.remote(
printer=printer,
pages=args.pages,
category=pirate.torrent.parse_category(printer, args.category),
sort=pirate.torrent.parse_sort(printer, args.sort),
mode=args.action,

41
pirate/torrent.py
View File

@ -93,14 +93,14 @@ def build_magnet(name, info_hash):
info_hash, parse.quote(name, ''))
def build_request_path(mode, category, terms):
def build_request_path(mode, page, category, terms):
if mode == 'search':
query = '/q.php?q={}&cat={}'.format(' '.join(terms), category)
elif mode == 'top':
cat = 'all' if category == 0 else category
query = '/precompiled/data_top100_{}.json'.format(cat)
elif mode == 'recent':
query = '/precompiled/data_top100_recent.json'
query = '/precompiled/data_top100_recent_{}.json'.format(page)
elif mode == 'browse':
if category == 0:
raise Exception('You must specify a category')
@ -111,25 +111,30 @@ def build_request_path(mode, category, terms):
return parse.quote(query, '?=&/')
def remote(printer, category, sort, mode, terms, mirror, timeout):
query = build_request_path(mode, category, terms)
# Catch the Ctrl-C exception and exit cleanly
try:
req = request.Request(
mirror + query,
headers=pirate.data.default_headers)
def remote(printer, pages, category, sort, mode, terms, mirror, timeout):
results = []
for i in range(1, pages + 1):
query = build_request_path(mode, i, category, terms)
# Catch the Ctrl-C exception and exit cleanly
try:
f = request.urlopen(req, timeout=timeout)
except urllib.error.URLError as e:
raise e
req = request.Request(
mirror + query,
headers=pirate.data.default_headers)
try:
f = request.urlopen(req, timeout=timeout)
except urllib.error.URLError as e:
raise e
if f.info().get('Content-Encoding') == 'gzip':
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
except KeyboardInterrupt:
printer.print('\nCancelled.')
sys.exit(0)
if f.info().get('Content-Encoding') == 'gzip':
f = gzip.GzipFile(fileobj=BytesIO(f.read()))
except KeyboardInterrupt:
printer.print('\nCancelled.')
sys.exit(0)
return sort_results(sort, parse_page(f))
results.extend(parse_page(f))
return sort_results(sort, results)
def find_api(mirror, timeout):

22
tests/test_torrent.py
View File

@ -70,18 +70,18 @@ class TestTorrent(unittest.TestCase):
def test_request_path(self):
# the args are (mode, category, terms)
succeed = [
(('recent', 0, []), '/precompiled/data_top100_recent.json'),
(('recent', 100, []), '/precompiled/data_top100_recent.json'),
(('top', 0, []), '/precompiled/data_top100_all.json'),
(('top', 100, []), '/precompiled/data_top100_100.json'),
(('search', 100, ['abc']), '/q.php?q=abc&cat=100'),
(('search', 100, ['abc', 'def']), '/q.php?q=abc%20def&cat=100'),
(('search', 100, ['\u1234']), '/q.php?q=%E1%88%B4&cat=100'),
(('browse', 100, []), '/q.php?q=category%3A100'),
(('recent', 1, 0, []), '/precompiled/data_top100_recent_1.json'),
(('recent', 2, 100, []), '/precompiled/data_top100_recent_2.json'),
(('top', 1, 0, []), '/precompiled/data_top100_all.json'),
(('top', 1, 100, []), '/precompiled/data_top100_100.json'),
(('search', 1, 100, ['abc']), '/q.php?q=abc&cat=100'),
(('search', 1, 100, ['abc', 'def']), '/q.php?q=abc%20def&cat=100'),
(('search', 1, 100, ['\u1234']), '/q.php?q=%E1%88%B4&cat=100'),
(('browse', 1, 100, []), '/q.php?q=category%3A100'),
]
fail = [
(('browse', 0, []), Exception),
(('asdf', 100, []), Exception)
(('browse', 1, 0, []), Exception),
(('asdf', 1, 100, []), Exception)
]
for inp, out in succeed:
path = pirate.torrent.build_request_path(*inp)
@ -154,7 +154,7 @@ class TestTorrent(unittest.TestCase):
with patch('urllib.request.Request', return_value=req_obj) as req:
with patch('urllib.request.urlopen', return_value=res_obj) as res:
results = pirate.torrent.remote(
MagicMock(Printer), 100, sort, 'top',
MagicMock(Printer), 1, 100, sort, 'top',
[], 'http://example.com', 9)
req.assert_called_once_with(
'http://example.com/precompiled/data_top100_100.json',