#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ## # @author Aleksander Mistewicz import sys import os import subprocess import time import re import urllib2 import argparse import logging __version__ = "0.2.2" __license__ = "APACHE-2.0" __author__ = "Aleksander Mistewicz" __author_email__ = "a.mistewicz@samsung.com" USAGE = "%prog " AGENT = "%s/%s" % (__name__, __version__) class Fetcher(object): def __init__(self, url): self.url = url def _addHeaders(self, request): request.add_header("User-Agent", AGENT) def open(self): url = self.url try: request = urllib2.Request(url) handle = urllib2.build_opener() except IOError: return None return (request, handle) def fetch(self): request, handle = self.open() self._addHeaders(request) if handle: try: content = unicode(handle.open(request).read(), "utf-8", errors="replace") return content except (urllib2.HTTPError, urllib2.URLError, urllib2.socket.error) as error: logging.error("ERROR: %s", error) class ImageVersion(object): def __init__(self, url): versions = re.findall(r'\d{8}\.\d+', url) if len(versions) == 3: if versions[0] != versions[1]: raise ValueError self.snapshot = versions[0] self.submission = versions[2] elif len(versions) == 1: self.snapshot = versions[0] self.submission = None else: raise ValueError def get_version(self): if self.submission: return '.'.join([self.snapshot, self.submission]) else: return self.snapshot def get_snapshot(self): return self.snapshot def is_prerelease(self): if self.submission: return True else: return False class ImageDownloader: BASE_ARM_WAYLAND = "images/arm-wayland/" BOOT_NAME = "common-boot-armv7l-odroidu3" USR_NAME = "common-wayland-3parts-armv7l-odroidu3" def __init__(self, url, arch): if not arch or "i586" in arch: self.BASE_MINNOW_WAYLAND = "images/ia32-wayland/" self.MINNOW_NAME = "common-wayland-efi-i586" else: self.BASE_MINNOW_WAYLAND = "images/x86_64-wayland/" self.MINNOW_NAME = "common-wayland-efi-x86_64" self.url = url img_ver = ImageVersion(url) self.snap_nr = img_ver.get_snapshot() self.ver_nr = img_ver.get_version() self.is_prerelease = img_ver.is_prerelease() odroid_common_boot_prefix = self.url + self.BASE_ARM_WAYLAND + self.BOOT_NAME self.odroid_boot_path = odroid_common_boot_prefix \ + "/tizen-common_" + self.ver_nr + "_" + self.BOOT_NAME self.odroid_boot_path_md5 = odroid_common_boot_prefix + "/MD5SUMS" odroid_common_usr_prefix = self.url + self.BASE_ARM_WAYLAND + self.USR_NAME self.odroid_usr_path = odroid_common_usr_prefix \ + "/tizen-common_" + self.ver_nr + "_" + self.USR_NAME self.odroid_usr_path_md5 = odroid_common_usr_prefix + "/MD5SUMS" self.odroid_snap_pkgs = "http://download.tizen.org/snapshots/tizen/common/tizen-common_" \ + self.snap_nr + "/" + self.BASE_ARM_WAYLAND + self.USR_NAME \ + "/tizen-common_" + self.snap_nr + "_" + self.USR_NAME + ".packages" minnow_common_minnow_prefix = self.url + self.BASE_MINNOW_WAYLAND + self.MINNOW_NAME self.minnow_usr_path = minnow_common_minnow_prefix \ + "/tizen-common_" + self.ver_nr + "_" + self.MINNOW_NAME self.minnow_usr_path_md5 = minnow_common_minnow_prefix + "/MD5SUMS" self.minnow_snap_pkgs = "http://download.tizen.org/snapshots/tizen/common/tizen-common_" \ + self.snap_nr + "/" + self.BASE_MINNOW_WAYLAND + self.MINNOW_NAME \ + "/tizen-common_" + self.snap_nr + "_" + self.MINNOW_NAME + ".packages" logging.debug('snapshot number: %s', self.snap_nr) logging.debug('version number: %s', self.ver_nr) logging.debug('odroid (boot path): %s', self.odroid_boot_path) logging.debug('odroid (usr path): %s', self.odroid_usr_path) logging.debug('minnow (usr path): %s', self.minnow_usr_path) def create_projectconf(self, filename, arch, target): logging.debug("Create project.conf file for: %s %s", arch, target) prjconf = [ "tizen-common_" + self.ver_nr, arch, target ] with open(filename, 'w') as f: f.write('\n'.join(prjconf) + '\n') def fetch_url(self, url): logging.debug("Fetching: %s", url) target = Fetcher(url) f = target.fetch() while f == None: logging.info("Retrying...") time.sleep(60) f = target.fetch() return f def check_diff(self, filename, snap_pkgs, pre_pkgs): logging.debug("Checking diff") set_snap_pkgs = set(snap_pkgs.splitlines()) set_pre_pkgs = set(pre_pkgs.splitlines()) diff = set_pre_pkgs - set_snap_pkgs with open(filename, 'w') as f: ret = (len(diff) == 0) if ret: s = 'Images are identical' else: s = '\n'.join(diff) logging.info(s) f.write(s) return ret def write_diff_for_snapshot(self, filename): logging.debug("Write diff for snapshot image") with open(filename, 'w') as f: f.write('Snapshot') def check_md5(self, url): logging.debug("Checking md5sum") md5_file = "md5sums" subprocess.call(["wget", url, "-O", md5_file]) subprocess.call(["sed", "-e", "/\(ks\|json\|log\|xml\|-default\|packages\)/d", "-i", md5_file]) ret = subprocess.call(["md5sum", "-c", md5_file]) if not ret: logging.info("Checksum OK") else: logging.warn("Checksum FAILED\nRemoving files mentioned in md5sums file") with open(md5_file, 'r') as f: for i in f: os.remove(re.findall(r'tizen-common_.+', i)[0]) os.remove(md5_file) return ret def get(self, urls, md5sum): while True: subprocess.call(["wget", "-cq"] + urls) if not self.check_md5(md5sum): break time.sleep(60) def odroid(self): logging.debug("Downloading images for: odroid") self.create_projectconf("project-odroid.conf", "armv7l", "Odroid U3") diff_report_filename="diff-odroid.report" if not self.is_prerelease: self.write_diff_for_snapshot(diff_report_filename) else: if self.check_diff(diff_report_filename, self.fetch_url(self.odroid_usr_path + ".packages"), \ self.fetch_url(self.odroid_snap_pkgs)): return logging.debug("Downloading prerelease images...") self.get([self.odroid_boot_path + ".tar.gz"], self.odroid_boot_path_md5) self.get([self.odroid_usr_path + ".tar.gz"], self.odroid_usr_path_md5) def minnow(self): logging.debug("Downloading images for: minnowboard") self.create_projectconf("project-minnow.conf", "i586/x86_64", "MinnowboardMax") diff_report_filename="diff-minnow.report" if not self.is_prerelease: self.write_diff_for_snapshot(diff_report_filename) else: if self.check_diff(diff_report_filename, self.fetch_url(self.minnow_usr_path + ".packages"), \ self.fetch_url(self.minnow_snap_pkgs)): return logging.debug("Downloading prerelease images...") self.get([self.minnow_usr_path + "-sda.raw.bz2", self.minnow_usr_path + "-sda.bmap"], \ self.minnow_usr_path_md5) def parse_arguments(): """parse_arguments() -> args Parse any command-line options given returning both the parsed options and arguments. """ parser = argparse.ArgumentParser(description="Image downloader for download.tizen.org") parser.add_argument("url", metavar='', type=str, help='URL of prerelease or snapshot to download images from.') parser.add_argument("-m", "--minnow", action="store_true", default=False, dest="minnow", help="Download images for minnowboard") parser.add_argument("--minnow32", action="store_true", default=False, dest="minnow32", help="Download images for minnowboard i586") parser.add_argument("--minnow64", action="store_true", default=False, dest="minnow64", help="Download images for minnowboard x86_64") parser.add_argument("-o", "--odroid", action="store_true", default=False, dest="odroid", help="Download images for odroid") parser.add_argument("-a", "--arch", action="store", dest="arch", help="Choose architecture of minnowboard images") parser.add_argument("-l", "--log", action="store", dest="loglevel", help="Verbosity level") args = parser.parse_args() return args def main(): args = parse_arguments() if args.loglevel: numeric_level = getattr(logging, args.loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % args.loglevel) logging.basicConfig(format='%(asctime)s %(message)s',level=numeric_level) logging.debug("Begin") if args.minnow64: downloader = ImageDownloader(args.url, 'x86_64') args.minnow = True elif args.minnow32: downloader = ImageDownloader(args.url, 'i586') args.minnow = True else: downloader = ImageDownloader(args.url, args.arch) if args.odroid: downloader.odroid() if args.minnow: downloader.minnow() logging.debug("End") if __name__ == '__main__': main()