diff options
author | Aleksander Mistewicz <a.mistewicz@samsung.com> | 2016-07-13 12:05:20 +0200 |
---|---|---|
committer | Aleksander Mistewicz <a.mistewicz@samsung.com> | 2016-10-07 17:29:30 +0200 |
commit | df86143ffedec31ecc5287f627a670dc2a14c204 (patch) | |
tree | a335dc9121e9dcc4f7eef047b9b38331709107e1 | |
parent | e201c2f2eacf602e9091c8eed00ff60a3f5e3008 (diff) | |
download | major-df86143ffedec31ecc5287f627a670dc2a14c204.tar.gz major-df86143ffedec31ecc5287f627a670dc2a14c204.tar.bz2 major-df86143ffedec31ecc5287f627a670dc2a14c204.zip |
Update md5sums logic in tsp/scripts/crawler.py
If any of the md5sums_url is missing (http error code 404 was returned)
from md5sums_urls list then it will be skipped and not dispatched.
It saves http requests so that it won't timeout if the server is busy.
For url to be added to dispatched_urls it must:
* have every md5sums_url from md5sums_urls list available
(no 404 error returned)
* have any of the md5sums_urls modified since last timestamp
Change-Id: Ie790f1039e91003b0da85c19d7db11ac160007da
Signed-off-by: Aleksander Mistewicz <a.mistewicz@samsung.com>
-rwxr-xr-x | tsp/scripts/crawler.py | 20 |
1 files changed, 13 insertions, 7 deletions
diff --git a/tsp/scripts/crawler.py b/tsp/scripts/crawler.py index 4cc1669..bbc6704 100755 --- a/tsp/scripts/crawler.py +++ b/tsp/scripts/crawler.py @@ -70,17 +70,23 @@ def get_modified_paths(discovered, timestamp): logging.info("Previous timestamp: %s", stamp) for url in discovered: logging.debug("Check for MD5SUMS change: %s", url) - for md5sums_url in [url + 'images/arm-wayland/common-wayland-3parts-armv7l-odroidu3/MD5SUMS',\ - url + 'images/x86_64-wayland/common-wayland-efi-x86_64/MD5SUMS',\ - url + 'images/ia32-wayland/common-wayland-efi-i586/MD5SUMS']: + md5sums_urls = [url + 'images/arm-wayland/common-wayland-3parts-armv7l-odroidu3/MD5SUMS',\ + url + 'images/x86_64-wayland/common-wayland-efi-x86_64/MD5SUMS',\ + url + 'images/ia32-wayland/common-wayland-efi-i586/MD5SUMS'] + change = False + for md5sums_url in md5sums_urls: try: u = urllib2.urlopen(urllib2.Request(md5sums_url, headers={"If-Modified-Since": stamp})) except urllib2.HTTPError as e: - pass + if e.code == 404: + logging.debug("MD5SUMS missing: %s", md5sums_url) + break else: - logging.info("MD5SUMS changed: %s", url) - ret.add(url) - break + logging.debug("MD5SUMS changed: %s", md5sums_url) + change = True + if change: + logging.info("Add to dispatch: %s", url) + ret.add(url) with open(timestamp, 'w') as f: f.write(str_time) return ret |