Make TITS profile independent

Overall TITS adjustments

Change-Id: I8a9b1255a122efa50a8275a558c0bc01174ad4fa
Signed-off-by: Aleksander Mistewicz <a.mistewicz@samsung.com>

5 files changed, 100 insertions, 41 deletions

diff --git a/tsp/jobs/common_prep_flash_conf.sh b/tsp/jobs/common_prep_flash_conf.sh
index 9499195..fabec78 100755
--- a/tsp/jobs/common_prep_flash_conf.sh
+++ b/tsp/jobs/common_prep_flash_conf.sh
@@ -33,24 +33,37 @@ cd "${WORKDIR}"
 WORKDIR_REMOTE="${WS_TEST_REMOTE}/${BUILD_NR}"
 DWN_WORKDIR="${WS_DOWNLOAD}/${BUILD_NR}"
 
+SR="$(project2sr "${DWN_WORKDIR}/project.conf")"
+PROFILE="$(sr2profile "${SR}")"
+
 # Update status in database
-update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "Testing" "$BUILD_NR" "$TARGET"
+update_db "${SR}" "Testing" "$BUILD_NR" "$TARGET"
 
+USR_IMG_DIR="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")"
 case "$TARGET" in
     *odroid*|*artik*)
-        BOOT_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-boot-*.tar.gz)"
-        USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-3parts-*.tar.gz)"
+        BOOT_IMG="$(ls "$(get_boot_dir "${TARGET}" "${DWN_WORKDIR}")"/*.tar.gz)"
+        test -f "${BOOT_IMG}" || BOOT_IMG="$(ls "$(get_boot_dir "${TARGET}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}")"/*.tar.gz)"
+        USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.tar.gz)"
+        test -f "${USERSPACE_IMG}" || USERSPACE_IMG="$(ls "$(get_userspace_dir "${TARGET}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}")"/*.tar.gz)"
+        # TODO: Both images shouldn't be replaced by snapshot's at the same time
         ;;
     *minnow*)
-        USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-wayland-*-sda.raw.bz2)"
-        USERSPACE_IMG_BMAP="$(ls "${DWN_WORKDIR}"/tizen-common_*-wayland-*-sda.bmap)"
+        USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.raw.bz2)"
+        USERSPACE_IMG_BMAP="$(ls "${USR_IMG_DIR}"/*.bmap)"
         ;;
     *mbr64*)
-        USERSPACE_IMG="$(ls "${DWN_WORKDIR}"/tizen-common_*-minimal-mbr-*.tar.gz)"
-        KERNEL="$(ls "${DWN_WORKDIR}"/tizen-common_*-minimal-mbr-*-vmlinuz-*)"
+        USERSPACE_IMG="$(ls "${USR_IMG_DIR}"/*.tar.gz)"
+        KERNEL="$(ls "${USR_IMG_DIR}"/*-vmlinuz-*)"
+        ;;
+    *)
+        die "Unknown target: ${TARGET}"
         ;;
 esac
 
+test -z "${BOOT_IMG}" || test -f "${BOOT_IMG}" || die "No boot image found: ${BOOT_IMG}!"
+test -f "${USERSPACE_IMG}" || die "No userspace image found: ${USERSPACE_IMG}!"
+
 FAIL_CNT="$3"
 test -n "${FAIL_CNT}" || FAIL_CNT=0
 
@@ -66,7 +79,7 @@ retrigger() {
     tsrun_target "${TARGET#headless-}" -L "RETRIGGERED_$(echo "$TARGET" | tr '[:lower:]' '[:upper:]')_PREPARE_FLASH_CONF" \
         sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "$BUILD_NR" "${TARGET}" "$((FAIL_CNT + 1))"
     # Update status in database
-    update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "In test queue" "$BUILD_NR" "$TARGET"
+    update_db "${SR}" "In test queue" "$BUILD_NR" "$TARGET"
     exit 0
 }
 
diff --git a/tsp/jobs/img_test_common.sh b/tsp/jobs/img_test_common.sh
index 40b9ce0..304742f 100755
--- a/tsp/jobs/img_test_common.sh
+++ b/tsp/jobs/img_test_common.sh
@@ -20,21 +20,32 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.."
 
 . "${TSP_DIR}/common.sh"
 
-test -n "$1" || die "Missing argument: build nr"
-
-WORKDIR="${WS_TEST}/$1"
-DWN_WORKDIR="${WS_DOWNLOAD}/$1"
+BUILD_NR="$1"
+WORKDIR="${WS_TEST}/${BUILD_NR}"
+DWN_WORKDIR="${WS_DOWNLOAD}/${BUILD_NR}"
 TARGET="$2"
 
+test -n "$BUILD_NR" || die "Missing argument: build_nr"
+test -n "$TARGET" || die "Missing argument: target"
+
+# update symlink (all images are downloaded)
+SR="$(project2sr "${DWN_WORKDIR}/project.conf")"
+PROFILE="$(sr2profile "${SR}")"
+test -L "${SNAPSHOT_WS_DOWNLOAD_NEW}" && mv -T "${SNAPSHOT_WS_DOWNLOAD_NEW}-${PROFILE}" "${SNAPSHOT_WS_DOWNLOAD}-${PROFILE}"
+
+rm -rf "${WORKDIR}"
+mkdir -p "${WORKDIR}"
+
 check_diff() {
-    DIFF_FILE="$1"
-    JOB_NR="$2"
-    TARGET="$3"
-    test -n "$1" || die "Missing argument: diff file"
-    test -n "$2" || die "Missing argument: build nr"
-    test -n "$3" || die "Missing argument: target"
+    DIFF_FILE1="$1"
+    DIFF_FILE2="$2"
+    JOB_NR="$3"
+    TARGET="$4"
+    test -n "${DIFF_FILE1}" || die "Missing argument: diff_file1"
+    test -n "${JOB_NR}" || die "Missing argument: build_nr"
+    test -n "${TARGET}" || die "Missing argument: target"
     test -n "${TSP_DIR}" || die "Missing environment variable: TSP_DIR"
-    if grep -q "identical" "${DIFF_FILE}"
+    if ( grep -q "identical" "${DIFF_FILE1}" ) && ( [ -z "${DIFF_FILE2}" ] || grep -q "identical" "${DIFF_FILE2}" )
     then
         echo "Image is unchanged in respect to snapshot"
         tspwb -L PUBLISH sh "${TSP_DIR}/jobs/publish.sh" "${JOB_NR}" "${TARGET}"
@@ -42,16 +53,40 @@ check_diff() {
     fi
 }
 
+report_no_image() {
+    DWN_WORKDIR="$1"
+    JOB_NR="$2"
+    TARGET="$3"
+    test -n "${DWN_WORKDIR}" || die "Missing argument: dwn_workdir"
+    test -n "${JOB_NR}" || die "Missing argument: build_nr"
+    test -n "${TARGET}" || die "Missing argument: target"
+    test -n "${SR}" || die "Missing argument: sr"
+    echo "Report missing image"
+    update_db "$SR" "No image" "$JOB_NR" "$TARGET"
+    return 1
+}
+
 test -n "$TARGET" || die "Missing argument: target"
 
-DIFF_FILE="${DWN_WORKDIR}/diff-${TARGET}.report"
+USR_IMG_DIR="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")" \
+    || report_no_image "${DWN_WORKDIR}" "${BUILD_NR}" "${TARGET}" || die "No image"
 
-rm -rf "${WORKDIR}"
-if check_diff "${DIFF_FILE}" "$1" "${TARGET}";
+DIFF_FILE1="${USR_IMG_DIR}/diff.report"
+case "${TARGET}" in
+    *artik*|*odroid*)
+        DIFF_FILE2="$(get_boot_dir "${TARGET}" "${DWN_WORKDIR}")/diff.report" \
+            || report_no_image "${DWN_WORKDIR}" "${BUILD_NR}" "${TARGET}" || die "No image"
+        ;;
+    *)
+        DIFF_FILE2=""
+        ;;
+esac
+
+if check_diff "${DIFF_FILE1}" "${DIFF_FILE2}" "${BUILD_NR}" "${TARGET}"
 then
     echo "Add common_prep_flash_conf job to the $TARGET queue"
     tsrun_target "${TARGET#headless-}" -L "$(echo "$TARGET" | tr '[:lower:]' '[:upper:]')_PREPARE_FLASH_CONF" \
-        sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "$1" "${TARGET}"
+        sh "${TSP_DIR}/jobs/common_prep_flash_conf.sh" "${BUILD_NR}" "${TARGET}"
     # Update status in database
-    update_db "$(project2sr "${DWN_WORKDIR}/project-${TARGET}.conf")" "In test queue" "$1" "$TARGET"
+    update_db "$SR" "In test queue" "${BUILD_NR}" "$TARGET"
 fi
diff --git a/tsp/jobs/publish.sh b/tsp/jobs/publish.sh
index d684abd..bc90920 100755
--- a/tsp/jobs/publish.sh
+++ b/tsp/jobs/publish.sh
@@ -97,8 +97,8 @@ compare_with_snapshot() {
     echo "OK"
 }
 
-PROJECT_FILE="${DWN_WORKDIR}/project-${TARGET}.conf"
-DIFF_FILE="${DWN_WORKDIR}/diff-${TARGET}.report"
+PROJECT_FILE="${DWN_WORKDIR}/project.conf"
+DIFF_FILE="$(get_userspace_dir "${TARGET}" "${DWN_WORKDIR}")/diff.report"
 SYSCTL_FILE="${IMG_WORKDIR}/sysctl.result"
 SCREENSHOT_FILE="${IMG_WORKDIR}/screenshot.png"
 SNAPSHOT_SCREENSHOT_FILE="${WS_TEST}/snapshot_screenshot_${TARGET}.png"
diff --git a/tsp/jobs/trigger_downloads.sh b/tsp/jobs/trigger_downloads.sh
index 6db9001..53c12a2 100755
--- a/tsp/jobs/trigger_downloads.sh
+++ b/tsp/jobs/trigger_downloads.sh
@@ -33,7 +33,7 @@ touch timestamp
 touch timestamp_snapshot
 
 # Crawl given url
-timeout 120 "${TSP_DIR}/scripts/crawler.py" --log INFO "${URL}" || die "Crawl failed" "$?"
+timeout 240 "${TSP_DIR}/scripts/crawler.py" --log DEBUG "${URL}" || die "Crawl failed" "$?"
 
 # Read next buid_nr
 touch next_dwn
@@ -48,29 +48,40 @@ then
 else
     cat modified_urls >> unique_dispatch
 fi
+
+# Choose image_map
+IMAGE_MAP="${TSP_DIR}/image_map"
+test -f "${IMAGE_MAP}" || IMAGE_MAP="${TSP_DIR}/image_map.example"
+
 while read -r url
 do
     test -n "${url}" || continue
     echo "Processing: ${url}"
-    if [ "$url" = "*latest*" ]
-    then
-        echo "Skipping symlink url"
-        continue
-    fi
     # kill currently running jobs for this url
     for i in $(tsmaster | awk -v URL="$url" '$2 ~ "running" && $0 ~ URL {print $1}'); do kill "$(tsmaster -p "$i")"; done
+    next=$((next+1))
+    dwn_next="${next}"
+    # prepare download directory
+    rm -rf "${WS_DOWNLOAD}/$dwn_next"
+    mkdir -p "${WS_DOWNLOAD}/$dwn_next"
+    cd "${WS_DOWNLOAD}/$dwn_next"
+    # download images
+    dwn_nr=$(tsmaster -L "$dwn_next" python "${TSP_DIR}/scripts/download_all.py" --log=INFO -m "${IMAGE_MAP}" "$url")
+    case "${url}" in
+        *snapshots*)
+            echo "Url is a snapshot, create/update symlink"
+            ln -Tsf "${WS_DOWNLOAD}/${dwn_next}" "${SNAPSHOT_WS_DOWNLOAD_NEW}-$(url2profile "${url}")"
+            ;;
+    esac
     for target in "minnow32" "minnow64" "odroid" "mbr64" "headless-artik5" "headless-artik10" "artik5" "artik10"
     do
-        rm -rf "${WS_DOWNLOAD}/$next"
-        mkdir -p "${WS_DOWNLOAD}/$next"
-        cd "${WS_DOWNLOAD}/$next"
-        # Download image for $target
-        nr=$(tsmaster -L "$next" python "${TSP_DIR}/scripts/download_image.py" --log=INFO -t "${target}" "$url")
-        nr=$(tsmaster -D "$nr" -L "IMAGE_TEST" \
-            sh "${TSP_DIR}/jobs/img_test_common.sh" "$next" "$target")
+        next=$((next+1))
+        # prepare download directory
+        rm -rf "${WS_DOWNLOAD}/${next}"
+        ln -sf "${WS_DOWNLOAD}/${dwn_next}" "${WS_DOWNLOAD}/${next}"
+        tsmaster -D "$dwn_nr" -L "IMAGE_TEST" sh "${TSP_DIR}/jobs/img_test_common.sh" "$next" "$target"
         # Update status in database
         update_db "$(url2sr "$url")" "Downloading" "$next" "$target"
-        next=$((next+1))
     done
 done < unique_dispatch
 cd "${WS_WATCHER}"
diff --git a/tsp/jobs/watch_diskspace.sh b/tsp/jobs/watch_diskspace.sh
index 0ea233c..aac0baf 100755
--- a/tsp/jobs/watch_diskspace.sh
+++ b/tsp/jobs/watch_diskspace.sh
@@ -24,7 +24,7 @@ export TSP_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)/.."
 PER="$(df -P "${WS}" | grep -o "[0-9][0-9]%.*$")"
 # Leave only numerical value.
 PER="${PER%\%*}"
-if [ "$PER" -ge 70 ]
+if [ "$PER" -ge 30 ]
 then
     nr="$(tsmaster -L DISK_CLEANUP sh "${TSP_DIR}/scripts/free_diskspace.sh")"
     # make the above job urgent