diff options
Diffstat (limited to 'tsp/scripts')
| -rwxr-xr-x | tsp/scripts/download_all.py | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/tsp/scripts/download_all.py b/tsp/scripts/download_all.py index feb0656..52399b9 100755 --- a/tsp/scripts/download_all.py +++ b/tsp/scripts/download_all.py @@ -88,7 +88,7 @@ class Crawler(object): @classmethod def get_links(self, session, url): - main = session.get(url) + main = session.get(url, timeout=120) soup = bs4.BeautifulSoup(main.text, 'html.parser') links = set() for link in soup.find_all('a'): @@ -173,8 +173,8 @@ class Downloader(threading.Thread): snap_url = re.sub("\." + self.img_ver.get_submission(), '', snap_url) logging.info("snap: %s" % snap_url) - snap = self.session.get(snap_url) - pre = self.session.get(pre_url) + snap = self.session.get(snap_url, timeout=120) + pre = self.session.get(pre_url, timeout=120) if self.check_diff(pre.text, snap.text): return @@ -238,7 +238,7 @@ class Downloader(threading.Thread): class ImageDownloader(object): - def __init__(self, url, dry, mapping=None): + def __init__(self, url, dry, mapping): self.url = url self.dry = dry self.mapping = mapping @@ -331,7 +331,7 @@ def main(): raise ValueError('Invalid log level: %s' % args.loglevel) logging.basicConfig(format='%(asctime)s %(message)s',level=numeric_level) logging.debug("Begin") - ImageDownloader(args.url, args.dry, mapping=args.map) + ImageDownloader(args.url, args.dry, args.map) logging.debug("End") if __name__ == '__main__': |