diff options
Diffstat (limited to 'ci.sh')
| -rwxr-xr-x | ci.sh | 197 |
1 files changed, 84 insertions, 113 deletions
@@ -5,8 +5,8 @@ # license that can be found in the LICENSE file. # Continuous integration helper module. This module is meant to be called from -# the .gitlab-ci.yml file during the continuous integration build, as well as -# from the command line for developers. +# workflows during the continuous integration build, as well as from the +# command line for developers. set -eu @@ -21,7 +21,10 @@ CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-} CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-} CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-} CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-} +SKIP_BUILD="${SKIP_BUILD:-0}" SKIP_TEST="${SKIP_TEST:-0}" +FASTER_MSAN_BUILD="${FASTER_MSAN_BUILD:-0}" +TARGETS="${TARGETS:-all doc}" TEST_SELECTOR="${TEST_SELECTOR:-}" BUILD_TARGET="${BUILD_TARGET:-}" ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}" @@ -32,6 +35,8 @@ else fi # Whether we should post a message in the MR when the build fails. POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}" +# By default, do a lightweight debian HWY package build. +HWY_PKG_OPTIONS="${HWY_PKG_OPTIONS:---set-envvar=HWY_EXTRA_CONFIG=-DBUILD_TESTING=OFF -DHWY_ENABLE_EXAMPLES=OFF -DHWY_ENABLE_CONTRIB=OFF}" # Set default compilers to clang if not already set export CC=${CC:-clang} @@ -69,13 +74,18 @@ if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128" fi +if [[ "${ENABLE_WASM_SIMD}" -eq "2" ]]; then + CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_WANT_WASM2" + CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -DHWY_WANT_WASM2" +fi + if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}" fi # Version inferred from the CI variables. -CI_COMMIT_SHA=${CI_COMMIT_SHA:-${GITHUB_SHA:-}} -JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}} +CI_COMMIT_SHA=${GITHUB_SHA:-} +JPEGXL_VERSION=${JPEGXL_VERSION:-} # Benchmark parameters STORE_IMAGES=${STORE_IMAGES:-1} @@ -139,6 +149,7 @@ detect_clang_version() { fi local clang_version=$("${CC:-clang}" --version | head -n1) clang_version=${clang_version#"Debian "} + clang_version=${clang_version#"Ubuntu "} local llvm_tag case "${clang_version}" in "clang version 6."*) @@ -171,27 +182,6 @@ on_exit() { local retcode="$1" # Always cleanup the CLEANUP_FILES. cleanup - - # Post a message in the MR when requested with POST_MESSAGE_ON_ERROR but only - # if the run failed and we are not running from a MR pipeline. - if [[ ${retcode} -ne 0 && -n "${CI_BUILD_NAME:-}" && - -n "${POST_MESSAGE_ON_ERROR}" && -z "${CI_MERGE_REQUEST_ID:-}" && - "${CI_BUILD_REF_NAME}" = "master" ]]; then - load_mr_vars_from_commit - { set +xeu; } 2>/dev/null - local message="**Run ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} failed.** - -Check the output of the job at ${CI_JOB_URL:-} to see if this was your problem. -If it was, please rollback this change or fix the problem ASAP, broken builds -slow down development. Check if the error already existed in the previous build -as well. - -Pipeline: ${CI_PIPELINE_URL} - -Previous build commit: ${CI_COMMIT_BEFORE_SHA} -" - cmd_post_mr_comment "${message}" - fi } trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT @@ -203,7 +193,7 @@ trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT # running from a merge request pipeline). MR_HEAD_SHA="" # The common ancestor between the current commit and the tracked branch, such -# as master. This includes a list +# as main. This includes a list MR_ANCESTOR_SHA="" # Populate MR_HEAD_SHA and MR_ANCESTOR_SHA. @@ -216,30 +206,23 @@ merge_request_commits() { # changes on the Pull Request if needed. This fetches 10 more commits which # should be enough given that PR normally should have 1 commit. git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10 - MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null || + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null || echo "${GITHUB_SHA}")" + else + MR_HEAD_SHA="${GITHUB_SHA}" + fi else - # CI_BUILD_REF is the reference currently being build in the CI workflow. - MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "${CI_BUILD_REF:-HEAD}") + MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "HEAD") fi - if [[ -n "${CI_MERGE_REQUEST_IID:-}" ]]; then - # Merge request pipeline in CI. In this case the upstream is called "origin" - # but it refers to the forked project that's the source of the merge - # request. We need to get the target of the merge request, for which we need - # to query that repository using our CI_JOB_TOKEN. - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" \ - >> "${HOME}/.netrc" - git -C "${MYDIR}" fetch "${CI_MERGE_REQUEST_PROJECT_URL}" \ - "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" - MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD) - elif [[ -n "${GITHUB_BASE_REF:-}" ]]; then + if [[ -n "${GITHUB_BASE_REF:-}" ]]; then # Pull request workflow in GitHub Actions. GitHub checkout action uses # "origin" as the remote for the git checkout. git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}" MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD) else - # We are in a local branch, not a merge request. + # We are in a local branch, not a pull request workflow. MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true) fi @@ -258,40 +241,6 @@ merge_request_commits() { set -x } -# Load the MR iid from the landed commit message when running not from a -# merge request workflow. This is useful to post back results at the merge -# request when running pipelines from master. -load_mr_vars_from_commit() { - { set +x; } 2>/dev/null - if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then - local mr_iid=$(git rev-list --format=%B --max-count=1 HEAD | - grep -F "${CI_PROJECT_URL}" | grep -F "/merge_requests" | head -n 1) - # mr_iid contains a string like this if it matched: - # Part-of: <https://gitlab.com/wg1/jpeg-xlm/merge_requests/123456> - if [[ -n "${mr_iid}" ]]; then - mr_iid=$(echo "${mr_iid}" | - sed -E 's,^.*merge_requests/([0-9]+)>.*$,\1,') - CI_MERGE_REQUEST_IID="${mr_iid}" - CI_MERGE_REQUEST_PROJECT_ID=${CI_PROJECT_ID} - fi - fi - set -x -} - -# Posts a comment to the current merge request. -cmd_post_mr_comment() { - { set +x; } 2>/dev/null - local comment="$1" - if [[ -n "${BOT_TOKEN:-}" && -n "${CI_MERGE_REQUEST_IID:-}" ]]; then - local url="${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes" - curl -X POST -g \ - -H "PRIVATE-TOKEN: ${BOT_TOKEN}" \ - --data-urlencode "body=${comment}" \ - --output /dev/null \ - "${url}" - fi - set -x -} # Set up and export the environment variables needed by the child processes. export_env() { @@ -452,9 +401,12 @@ cmake_configure() { } cmake_build_and_test() { + if [[ "${SKIP_BUILD}" -eq "1" ]]; then + return 0 + fi # gtest_discover_tests() runs the test binaries to discover the list of tests # at build time, which fails under qemu. - ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- all doc + ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- $TARGETS # Pack test binaries if requested. if [[ "${PACK_TEST:-}" == "1" ]]; then (cd "${BUILD_DIR}" @@ -547,6 +499,7 @@ cmd_coverage_report() { # Only print coverage information for the libjxl directories. The rest # is not part of the code under test. --filter '.*jxl/.*' + --exclude '.*_gbench.cc' --exclude '.*_test.cc' --exclude '.*_testonly..*' --exclude '.*_debug.*' @@ -576,7 +529,7 @@ cmd_test() { (cd "${BUILD_DIR}" export UBSAN_OPTIONS=print_stacktrace=1 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" - ctest -j $(nproc --all || echo 1) --output-on-failure "$@") + ctest -j $(nproc --all || echo 1) ${TEST_SELECTOR} --output-on-failure "$@") } cmd_gbench() { @@ -652,7 +605,6 @@ cmd_msan() { local msan_c_flags=( -fsanitize=memory -fno-omit-frame-pointer - -fsanitize-memory-track-origins -DJXL_ENABLE_ASSERT=1 -g @@ -661,6 +613,13 @@ cmd_msan() { # Force gtest to not use the cxxbai. -DGTEST_HAS_CXXABI_H_=0 ) + if [[ "${FASTER_MSAN_BUILD}" -ne "1" ]]; then + msan_c_flags=( + "${msan_c_flags[@]}" + -fsanitize-memory-track-origins + ) + fi + local msan_cxx_flags=( "${msan_c_flags[@]}" @@ -724,6 +683,15 @@ cmd_msan_install() { local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" rm -rf "${msan_prefix}" + local TARGET_OPTS="" + if [[ -n "${BUILD_TARGET}" ]]; then + TARGET_OPTS=" \ + -DCMAKE_C_COMPILER_TARGET=\"${BUILD_TARGET}\" \ + -DCMAKE_CXX_COMPILER_TARGET=\"${BUILD_TARGET}\" \ + -DCMAKE_SYSTEM_PROCESSOR=\"${BUILD_TARGET%%-*}\" \ + " + fi + declare -A CMAKE_EXTRAS CMAKE_EXTRAS[libcxx]="\ -DLIBCXX_CXX_ABI=libstdc++ \ @@ -745,6 +713,7 @@ cmd_msan_install() { -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \ -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \ -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \ + ${TARGET_OPTS} \ ${CMAKE_EXTRAS[${project}]} cmake --build "${proj_build}" ninja -C "${proj_build}" install @@ -788,7 +757,7 @@ _cmd_ossfuzz() { -e MSAN_LIBS_PATH="/work/msan" \ -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \ -v "${MYDIR}":/src/libjxl \ - -v "${MYDIR}/tools/ossfuzz-build.sh":/src/build.sh \ + -v "${MYDIR}/tools/scripts/ossfuzz-build.sh":/src/build.sh \ -v "${real_build_dir}":/work \ gcr.io/oss-fuzz/libjxl } @@ -902,7 +871,7 @@ run_benchmark() { --input "${src_img_dir}/*.png" --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2 --output_dir "${output_dir}" - --noprofiler --show_progress + --show_progress --num_threads="${num_threads}" ) if [[ "${STORE_IMAGES}" == "1" ]]; then @@ -917,15 +886,7 @@ run_benchmark() { return ${PIPESTATUS[0]} ) - if [[ -n "${CI_BUILD_NAME:-}" ]]; then - { set +x; } 2>/dev/null - local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}): -$(cat "${output_dir}/results.txt") -" - cmd_post_mr_comment "${message}" - set -x - fi } # Helper function to wait for the CPU temperature to cool down on ARM. @@ -1153,18 +1114,6 @@ cmd_arm_benchmark() { cmd_cpuset "${RUNNER_CPU_ALL:-}" cat "${runs_file}" - if [[ -n "${CI_BUILD_NAME:-}" ]]; then - load_mr_vars_from_commit - { set +x; } 2>/dev/null - local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}): - -\`\`\` -$(column -t -s " " "${runs_file}") -\`\`\` -" - cmd_post_mr_comment "${message}" - set -x - fi } # Generate a corpus and run the fuzzer on that corpus. @@ -1184,22 +1133,38 @@ cmd_fuzz() { ) } -# Runs the linter (clang-format) on the pending CLs. +# Runs the linters (clang-format, build_cleaner, buildirier) on the pending CLs. cmd_lint() { merge_request_commits { set +x; } 2>/dev/null - local versions=(${1:-6.0 7 8 9 10 11}) + local versions=(${1:-16 15 14 13 12 11 10 9 8 7 6.0}) local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format) local tmpdir=$(mktemp -d) CLEANUP_FILES+=("${tmpdir}") local ret=0 local build_patch="${tmpdir}/build_cleaner.patch" - if ! "${MYDIR}/tools/build_cleaner.py" >"${build_patch}"; then + if ! "${MYDIR}/tools/scripts/build_cleaner.py" >"${build_patch}"; then ret=1 echo "build_cleaner.py findings:" >&2 "${COLORDIFF_BIN}" <"${build_patch}" - echo "Run \`tools/build_cleaner.py --update\` to apply them" >&2 + echo "Run \`tools/scripts/build_cleaner.py --update\` to apply them" >&2 + fi + + # It is ok, if buildifier is not installed. + if which buildifier >/dev/null; then + local buildifier_patch="${tmpdir}/buildifier.patch" + local bazel_files=`git -C ${MYDIR} ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"` + set -x + buildifier -d ${bazel_files} >"${buildifier_patch}"|| true + { set +x; } 2>/dev/null + if [ -s "${buildifier_patch}" ]; then + ret=1 + echo 'buildifier have found some problems in Bazel build files:' >&2 + "${COLORDIFF_BIN}" <"${buildifier_patch}" + echo 'To fix them run (from the base directory):' >&2 + echo ' buildifier `git ls-files | grep -E "/BUILD$|WORKSPACE|.bzl$"`' >&2 + fi fi local installed=() @@ -1218,7 +1183,7 @@ cmd_lint() { git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \ --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" || true { set +x; } 2>/dev/null - if grep -E '^--- ' "${tmppatch}">/dev/null; then + if grep -E '^--- ' "${tmppatch}" | grep -v 'a/third_party' >/dev/null; then if [[ -n "${LINT_OUTPUT:-}" ]]; then cp "${tmppatch}" "${LINT_OUTPUT}" fi @@ -1326,6 +1291,7 @@ cmd_debian_stats() { build_debian_pkg() { local srcdir="$1" local srcpkg="$2" + local options="${3:-}" local debsdir="${BUILD_DIR}/debs" local builddir="${debsdir}/${srcpkg}" @@ -1341,7 +1307,7 @@ build_debian_pkg() { done ( cd "${builddir}" - debuild -b -uc -us + debuild "${options}" -b -uc -us ) } @@ -1353,7 +1319,7 @@ cmd_debian_build() { build_debian_pkg "${MYDIR}" "jpeg-xl" ;; highway) - build_debian_pkg "${MYDIR}/third_party/highway" "highway" + build_debian_pkg "${MYDIR}/third_party/highway" "highway" "${HWY_PKG_OPTIONS}" ;; *) echo "ERROR: Must pass a valid source package name to build." >&2 @@ -1374,7 +1340,7 @@ cmd_bump_version() { local newver="${1:-}" if ! which dch >/dev/null; then - echo "Run:\n sudo apt install debhelper" + echo "Missing dch\nTo install it run:\n sudo apt install devscripts" exit 1 fi @@ -1402,9 +1368,13 @@ cmd_bump_version() { -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \ -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \ -i lib/CMakeLists.txt + sed -E \ + -e "s/(LIBJXL_VERSION: )[0-9\\.]+/\\1 ${major}.${minor}.${patch}/" \ + -e "s/(LIBJXL_ABI_VERSION: )[0-9\\.]+/\\1 ${major}.${minor}/" \ + -i .github/workflows/conformance.yml # Update lib.gni - tools/build_cleaner.py --update + tools/scripts/build_cleaner.py --update # Mark the previous version as "unstable". DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release '' @@ -1418,11 +1388,11 @@ cmd_authors() { merge_request_commits local emails local names - readarray -t emails < <(git log --format='%ae' "${MR_HEAD_SHA}...${MR_ANCESTOR_SHA}") - readarray -t names < <(git log --format='%an' "${MR_HEAD_SHA}...${MR_ANCESTOR_SHA}") + readarray -t emails < <(git log --format='%ae' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}") + readarray -t names < <(git log --format='%an' "${MR_ANCESTOR_SHA}..${MR_HEAD_SHA}") for i in "${!names[@]}"; do echo "Checking name '${names[$i]}' with email '${emails[$i]}' ..." - "${MYDIR}"/tools/check_author.py "${emails[$i]}" "${names[$i]}" + "${MYDIR}"/tools/scripts/check_author.py "${emails[$i]}" "${names[$i]}" done } @@ -1450,7 +1420,7 @@ Where cmd is one of: benchmark Run the benchmark over the default corpus. fast_benchmark Run the benchmark over the small corpus. - coverage Buils and run tests with coverage support. Runs coverage_report as + coverage Build and run tests with coverage support. Runs coverage_report as well. coverage_report Generate HTML, XML and text coverage report after a coverage run. @@ -1483,6 +1453,7 @@ You can pass some optional environment variables as well: - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds. - LINT_OUTPUT: Path to the output patch from the "lint" command. - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark. + - SKIP_BUILD=1: Skip the build stage, cmake configure only. - SKIP_TEST=1: Skip the test stage. - STORE_IMAGES=0: Makes the benchmark discard the computed images. - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB. |