diff options
author | Aleksander Mistewicz <a.mistewicz@samsung.com> | 2016-11-04 11:20:57 +0100 |
---|---|---|
committer | Aleksander Mistewicz <a.mistewicz@samsung.com> | 2017-01-16 11:07:48 +0100 |
commit | 98d24d50cb90d802754df0fcba9ac7ba3e210ec2 (patch) | |
tree | 05cd3a4aafe87aa23cfde2654741adfab3255fb7 /tsp/scripts | |
parent | e54b9b8fc5197c32d02cfab7a17e30effe3b20df (diff) | |
download | major-98d24d50cb90d802754df0fcba9ac7ba3e210ec2.tar.gz major-98d24d50cb90d802754df0fcba9ac7ba3e210ec2.tar.bz2 major-98d24d50cb90d802754df0fcba9ac7ba3e210ec2.zip |
Set request timeout in download_all.py
Change-Id: I95d135eeac2e8a10f2352f6d626b250aa2ddd75e
Signed-off-by: Aleksander Mistewicz <a.mistewicz@samsung.com>
Diffstat (limited to 'tsp/scripts')
-rwxr-xr-x | tsp/scripts/download_all.py | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/tsp/scripts/download_all.py b/tsp/scripts/download_all.py index feb0656..52399b9 100755 --- a/tsp/scripts/download_all.py +++ b/tsp/scripts/download_all.py @@ -88,7 +88,7 @@ class Crawler(object): @classmethod def get_links(self, session, url): - main = session.get(url) + main = session.get(url, timeout=120) soup = bs4.BeautifulSoup(main.text, 'html.parser') links = set() for link in soup.find_all('a'): @@ -173,8 +173,8 @@ class Downloader(threading.Thread): snap_url = re.sub("\." + self.img_ver.get_submission(), '', snap_url) logging.info("snap: %s" % snap_url) - snap = self.session.get(snap_url) - pre = self.session.get(pre_url) + snap = self.session.get(snap_url, timeout=120) + pre = self.session.get(pre_url, timeout=120) if self.check_diff(pre.text, snap.text): return @@ -238,7 +238,7 @@ class Downloader(threading.Thread): class ImageDownloader(object): - def __init__(self, url, dry, mapping=None): + def __init__(self, url, dry, mapping): self.url = url self.dry = dry self.mapping = mapping @@ -331,7 +331,7 @@ def main(): raise ValueError('Invalid log level: %s' % args.loglevel) logging.basicConfig(format='%(asctime)s %(message)s',level=numeric_level) logging.debug("Begin") - ImageDownloader(args.url, args.dry, mapping=args.map) + ImageDownloader(args.url, args.dry, args.map) logging.debug("End") if __name__ == '__main__': |