Use ccache.tar file with --pkg-ccache option

1. first build. create _ccache.tar
$ gbs build -A armv7l --ccache

after build is done
_ccache.tar file will be created
BUILD-ROOTS/scratch.armv7l.0/home/abuild/rpmbuild/OTHER/_ccache.tar
backup _ccache.tar file to your storage

2. rebuild with _ccache.tar
$ gbs build -A armv7l --ccache --pkg-ccache /path/to/_ccache.tar

test
build chromium-efl
build env: laptop NT371B5M
  - Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz
  - SAMSUNG MZNLN256 256G SSD

first clean build
gbs build -A armv7l --ccache
build time: 29806s

rebuild
 gbs build -A armv7l --ccache --overwrite --pkg-ccache ~/_ccache.tar
build time: 3294s

rebuild (with qemu-accel (ccache, ninja)
gbs build -A armv7l --ccache --overwrite --pkg-ccache ~/_ccache.tar
build time: 2209s

related upstream patch
https://github.com/openSUSE/obs-build/pull/580
https://github.com/openSUSE/obs-build/pull/578
https://github.com/openSUSE/obs-build/pull/577
https://github.com/openSUSE/obs-build/pull/576
https://github.com/openSUSE/obs-build/pull/571
https://github.com/openSUSE/obs-build/pull/560
https://github.com/openSUSE/obs-build/pull/559

Change-Id: Ib5e6f60eddcca6f4ba67e209f9068f11a125d86e
Signed-off-by: Hyunggi Lee <hyunggi.lee@samsung.com>

1 files changed, 57 insertions, 5 deletions

diff --git a/build b/build
index f98e1b7..374ebc9 100755..100644
--- a/build
+++ b/build
@@ -67,6 +67,7 @@ DO_CHECKS=true
 SHORT_CIRCUIT=false
 NO_TOPDIR_CLEANUP=false
 CLEAN_BUILD=
+REMOVE_CCACHE=
 KEEP_PACKS=
 RECIPEFILES=()
 SRCDIR=
@@ -99,6 +100,8 @@ CCACHE=
 DLNOSIGNATURE=
 CACHE_DIR=/var/cache/build
 USEHIGHERDEPS=
+PKG_CCACHE=
+CCACHE_SETUP_START_TIME=
 
 # This is for insserv
 export YAST_IS_RUNNING=instsys
@@ -232,6 +235,10 @@ Known Parameters:
   --ccache
               Use ccache to speed up rebuilds
 
+  --pkg-ccache /path/to/ccache.tar.gz
+              path to archive of ccache directory content.
+              Its contents will be extracted to /.ccache
+
   --icecream N
               Use N parallel build jobs with icecream
 
@@ -425,9 +432,9 @@ toshellscript() {
 
 setupccache() {
     if test -n "$CCACHE" ; then
+	CCACHE_SETUP_START_TIME=`date +%s`
 	if mkdir -p "$BUILD_ROOT"/var/lib/build/ccache/bin; then
-	    for i in $(ls "$BUILD_ROOT"/usr/bin | grep -E '^(cc|gcc|[cg][+][+])([-]?[234][.]?[0-9])*$'); do
-#                ln -sf /usr/bin/ccache $BUILD_ROOT/var/lib/build/ccache/bin/$i
+		for i in $(ls $BUILD_ROOT/usr/bin | grep -E '^(cc|gcc|[cg][+][+]|clang|clang[+][+])([-]?[.0-9])*$'); do
 		rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/$i
 		test -e "$BUILD_ROOT"/usr/bin/$i || continue
 		echo '#! /bin/sh' > "$BUILD_ROOT"/var/lib/build/ccache/bin/$i
@@ -439,11 +446,33 @@ setupccache() {
 	    done
 	fi
 	mkdir -p "$BUILD_ROOT/.ccache"
-	chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.ccache"
-	echo "export CCACHE_DIR=/.ccache" > "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+	# overwrite /etc/profile.d/build_ccache.sh
+	echo 'export CCACHE_DIR=/.ccache' > "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
 	echo 'export PATH=/var/lib/build/ccache/bin:$PATH' >> "$BUILD_ROOT"/etc/profile.d/build_ccache.sh
+
+	# Note
+	# Disable compression setting temporary
+	# Until ccache is applied to qemu-accel, it is very slow when using the function, so it will be applied later.
+	#
+	# https://ccache.dev/releasenotes.html#_ccache_4_0
+	# Changed compression algorithm from Deflate (zlib) to Zstandard, enabled by default.
+	enable_compression=false
+
+	local cconf="$BUILD_ROOT"/etc/ccache.conf
+	if test -e $cconf; then
+		grep -q "^compression*" $cconf && sed -i "s:compression.*:compression = $enable_compression:g" $cconf || echo "compression = $enable_compression"   >> $cconf
+		grep -q "^cache_dir*"   $cconf && sed -i "s:cache_dir.*:cache_dir = /.ccache:g" $cconf || echo "cache_dir = /.ccache" >> $cconf
     else
-	rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/{gcc,g++,cc,c++}
+		echo "compression = $enable_compression" >> $cconf
+		echo "cache_dir = /.ccache" >> $cconf
+	fi
+		local ccachetar="$BUILD_ROOT/.build.oldpackages/_ccache.tar"
+		test -e $ccachetar && tar -xf $ccachetar -C "$BUILD_ROOT/.ccache/"
+		rm -f $ccachetar
+		chown -R "$ABUILD_UID:$ABUILD_GID" "$BUILD_ROOT/.ccache"
+	else
+		CCACHE_SETUP_START_TIME=
+		rm -f "$BUILD_ROOT"/var/lib/build/ccache/bin/{gcc,g++,cc,c++,clang,clang++}
     fi
 }
 
@@ -765,6 +794,7 @@ while test -n "$1"; do
       ;;
       -clean)
 	CLEAN_BUILD='--clean'
+        REMOVE_CCACHE="$CLEAN_BUILD"
       ;;
       -kill)
 	KILL=true
@@ -891,6 +921,12 @@ while test -n "$1"; do
       -ccache)
 	CCACHE=true
       ;;
+      -pkg-ccache)
+	needarg
+	PKG_CCACHE="$ARG"
+	shift
+	CCACHE=true
+      ;;
       -statistics)
 	DO_STATISTICS=1
       ;;
@@ -1204,6 +1240,11 @@ for RECIPEFILE in "${RECIPEFILES[@]}" ; do
 	mount -n -tdevpts -omode=0620,gid=5 none "$BUILD_ROOT"/dev/pts
 
 	copy_oldpackages
+	# chroot based builds
+	# rsync as source and dest could be same
+	test -n "$PKG_CCACHE" -a -e "$PKG_CCACHE" && \
+		mkdir -p "$BUILD_ROOT/.build.oldpackages/" && \
+		rsync -v "$PKG_CCACHE" "$BUILD_ROOT/.build.oldpackages/_ccache.tar"
     fi
 
     # hack to process preinstallimages early
@@ -1465,6 +1506,17 @@ if test -n "$RPMS" -a "$DO_CHECKS" != false ; then
     recipe_run_rpmlint
 fi
 
+if test -n "$CCACHE" ; then
+    if test -n "$REMOVE_CCACHE" ; then
+        echo "... cleaning ccache"
+        test_cmd="ccache -h | grep -c evict-older-than"
+        clean_cmd="ccache --evict-older-than $(($(date +%s) - $CCACHE_SETUP_START_TIME))s"
+        chroot $BUILD_ROOT su -c "$test_cmd && $clean_cmd" - $BUILD_USER
+    fi
+    echo "... saving ccache"
+    tar ${REMOVE_CCACHE:+--remove-files} -cf "$BUILD_ROOT/$TOPDIR/OTHER/_ccache.tar" -C "$BUILD_ROOT/.ccache/" .
+fi
+
 if test \( -n "$RPMS" -o -n "$DEBS" \) -a -n "$CREATE_BASELIBS"; then
     create_baselibs
 fi