Enable sccache in the default docker and CI. (#1009)

* replace ccache with sccache, pin package versions

* put ccache back temporarily to avoid breaking other CI jobs

* add sccashe_wrapper.sh script

* fix the package version syntax

* fix the pymysql package issue

* run sccache_wrapper before build if ccache server found

* set the paths before calling the sccache_wrapper

* use /tmp instead of /usr/local for cache

* try using sccache --start-server instead of wrapper

* try using redis server with sccache

* define SCCACHE_REDIS

* add redis and ping packages, and redis port

* use the new sccache redis server

* do not use sccache with staging compiler

* fix the condition syntax

* add stunnel to redis

* add tunnel verification

* separate caches for different architectures

* fix syntax for the cache tag

* quse double brackets for conditions

* add bash line to the script

* add a switch for sccache and only use it in build stage

* run check_host function when enabling sccache

* fix the invocation tags for sccache

* fix groovy syntax

* set the invocation tag in groovy

* disable sccache in clang-format stage

* try another syntax for invocation tags

* use local sccache server if can't connect to redis

* fix script syntax

* update README

* refresh readme

* readme updates

* remove the timing and verification caveat from readme

---------

Co-authored-by: Lisa Delaney <lisa.delaney@amd.com>

script/sccache_wrapper.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+set -e
+COMPILERS_HASH_DIR=${COMPILERS_HASH_DIR:-"/tmp/.sccache"}
+SCCACHE_EXTRAFILES=${SCCACHE_EXTRAFILES:-"${COMPILERS_HASH_DIR}/rocm_compilers_hash_file"}
+SCCACHE_BIN=${SCCACHE_BIN:-"${SCCACHE_INSTALL_LOCATION}/sccache"}
+ENFORCE_REDIS="false"
+while [ "$1" != "" ];
+do
+    case $1 in
+        --enforce_redis )
+            shift; ENFORCE_REDIS="true" ;;
+        --no-hipcc )
+            shift ;;
+        *)
+            break ;;
+    esac
+done
+setup_rocm_compilers_hash_file() {
+    mkdir -p "$COMPILERS_HASH_DIR"
+    HIPCC_MD5="$(md5sum "${ROCM_PATH}/bin/hipcc")"
+    pushd "${ROCM_PATH}/amdgcn/bitcode"
+        DEVICELIBS_BITCODES_MD5="$(find . -type f -exec md5sum {} \; | sort | md5sum)"
+    popd
+    HIPCC_HASH_VALUE="${HIPCC_MD5%% *}"
+    DEVICELIBS_BITCODES_HASH_VALUE="${DEVICELIBS_BITCODES_MD5%% *}"
+    # MD5 checksums of clang and clang-offload-bundler cannot be used since they will keep changing
+    # if the ROCM_PATH changes, ie; for every mainline build.
+    # This is because ROCM_PATH gets encoded into the clang/clang-offload-bundler binaries as part
+    # of RPATH.
+    # The versions themselves contain the commit hash of the compiler repo at the time of building.
+    # Hence, this should be a viable alternative to using the binary checksum itself.
+    CLANG_VERSION="$("${ROCM_PATH}/llvm/bin/clang" --version | head -n 1)"
+    CLANG_OFFLOAD_BUNDLER_VERSION="$("${ROCM_PATH}/llvm/bin/clang-offload-bundler" --version | head -n 1)"
+    printf '%s: %s\n' 'clang version' "${CLANG_VERSION}" | tee -a "$SCCACHE_EXTRAFILES"
+    printf '%s: %s\n' 'clang-offload-bundler version' "${CLANG_OFFLOAD_BUNDLER_VERSION}" | tee -a "$SCCACHE_EXTRAFILES"
+    printf '%s: %s\n' 'hipcc md5sum' "${HIPCC_HASH_VALUE}" | tee -a "$SCCACHE_EXTRAFILES"
+    printf '%s: %s\n' 'devicelibs bitcode md5sum' "${DEVICELIBS_BITCODES_HASH_VALUE}" | tee -a "$SCCACHE_EXTRAFILES"
+    echo "sccache-wrapper: compilers hash file set up at ${SCCACHE_EXTRAFILES}"
+    cat "$SCCACHE_EXTRAFILES"
+}
+if [ "${ENFORCE_REDIS}" == "true" ]; then
+    if [ -z "${SCCACHE_REDIS}" ]; then
+        echo "SCCACHE_REDIS not set. Not wrapping compilers with sccache."
+        exit 10
+    else
+        response=$(redis-cli -u ${SCCACHE_REDIS} ping) || true
+        if [ "${response}" != "PONG" ]; then
+            echo "Redis server unreachable. Not wrapping compilers with sccache."
+            exit 20
+        fi
+    fi
+fi
+setup_rocm_compilers_hash_file
+$SCCACHE_BIN --version
+$SCCACHE_BIN --start-server
+