summaryrefslogtreecommitdiff
path: root/tsp/scripts
diff options
context:
space:
mode:
authorAleksander Mistewicz <a.mistewicz@samsung.com>2016-07-13 10:05:20 (GMT)
committerAleksander Mistewicz <a.mistewicz@samsung.com>2016-10-07 15:29:30 (GMT)
commitdf86143ffedec31ecc5287f627a670dc2a14c204 (patch)
treea335dc9121e9dcc4f7eef047b9b38331709107e1 /tsp/scripts
parente201c2f2eacf602e9091c8eed00ff60a3f5e3008 (diff)
downloadmajor-df86143ffedec31ecc5287f627a670dc2a14c204.zip
major-df86143ffedec31ecc5287f627a670dc2a14c204.tar.gz
major-df86143ffedec31ecc5287f627a670dc2a14c204.tar.bz2
Update md5sums logic in tsp/scripts/crawler.pyrefs/changes/15/79915/5
If any of the md5sums_url is missing (http error code 404 was returned) from md5sums_urls list then it will be skipped and not dispatched. It saves http requests so that it won't timeout if the server is busy. For url to be added to dispatched_urls it must: * have every md5sums_url from md5sums_urls list available (no 404 error returned) * have any of the md5sums_urls modified since last timestamp Change-Id: Ie790f1039e91003b0da85c19d7db11ac160007da Signed-off-by: Aleksander Mistewicz <a.mistewicz@samsung.com>
Diffstat (limited to 'tsp/scripts')
-rwxr-xr-xtsp/scripts/crawler.py20
1 files changed, 13 insertions, 7 deletions
diff --git a/tsp/scripts/crawler.py b/tsp/scripts/crawler.py
index 4cc1669..bbc6704 100755
--- a/tsp/scripts/crawler.py
+++ b/tsp/scripts/crawler.py
@@ -70,17 +70,23 @@ def get_modified_paths(discovered, timestamp):
logging.info("Previous timestamp: %s", stamp)
for url in discovered:
logging.debug("Check for MD5SUMS change: %s", url)
- for md5sums_url in [url + 'images/arm-wayland/common-wayland-3parts-armv7l-odroidu3/MD5SUMS',\
- url + 'images/x86_64-wayland/common-wayland-efi-x86_64/MD5SUMS',\
- url + 'images/ia32-wayland/common-wayland-efi-i586/MD5SUMS']:
+ md5sums_urls = [url + 'images/arm-wayland/common-wayland-3parts-armv7l-odroidu3/MD5SUMS',\
+ url + 'images/x86_64-wayland/common-wayland-efi-x86_64/MD5SUMS',\
+ url + 'images/ia32-wayland/common-wayland-efi-i586/MD5SUMS']
+ change = False
+ for md5sums_url in md5sums_urls:
try:
u = urllib2.urlopen(urllib2.Request(md5sums_url, headers={"If-Modified-Since": stamp}))
except urllib2.HTTPError as e:
- pass
+ if e.code == 404:
+ logging.debug("MD5SUMS missing: %s", md5sums_url)
+ break
else:
- logging.info("MD5SUMS changed: %s", url)
- ret.add(url)
- break
+ logging.debug("MD5SUMS changed: %s", md5sums_url)
+ change = True
+ if change:
+ logging.info("Add to dispatch: %s", url)
+ ret.add(url)
with open(timestamp, 'w') as f:
f.write(str_time)
return ret