diff options
Diffstat (limited to 'ci/setup-toolchain.sh')
| -rwxr-xr-x | ci/setup-toolchain.sh | 290 |
1 files changed, 250 insertions, 40 deletions
diff --git a/ci/setup-toolchain.sh b/ci/setup-toolchain.sh index 6ed09c6..a62f633 100755 --- a/ci/setup-toolchain.sh +++ b/ci/setup-toolchain.sh @@ -1,88 +1,298 @@ #!/bin/sh -# Provision the pinned Android build toolchain into TOOLCHAIN_DIR (default -# $HOME/toolchains): JDK 17, Android cmdline-tools, SDK platform, build-tools, -# NDK, and Gradle. Idempotent: re-running only fills what is missing. Writes an -# env.sh to source before building. This is what CI runs to get a build host. +# Provision the exact Android, Java, Go, and Gradle toolchain into +# TOOLCHAIN_DIR. Standalone tools are installed from staging directories; +# sdkmanager packages are checked for exact revisions after installation. set -eu -root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +# shellcheck source=versions . "$root/versions" dir=${TOOLCHAIN_DIR:-$HOME/toolchains} sdk="$dir/android-sdk" -mkdir -p "$dir/dl" -jdk="$dir/jdk-$JDK_RELEASE" cmdline="$sdk/cmdline-tools/$ANDROID_CMDLINE_TOOLS" +gradle="$dir/gradle-$GRADLE_VERSION" goroot="$dir/go-$GO_VERSION" +stage= +part= +trap 'if [ -n "$stage" ]; then rm -rf "$stage"; fi; if [ -n "$part" ]; then rm -f "$part"; fi' 0 1 2 3 15 + +if [ "$(uname -s)" != Linux ] || [ "$(uname -m)" != x86_64 ]; then + echo "setup: only Linux x86_64 build hosts are supported" >&2 + exit 1 +fi + +# Temurin publishes separate x86_64 archives for glibc and musl. Detect the +# host before naming the cache so an archive for one libc can never be reused +# on the other. +if getconf GNU_LIBC_VERSION >/dev/null 2>&1; then + jdk_libc=glibc + jdk_platform=linux + jdk_sha256=$JDK_GLIBC_SHA256 +else + musl_loader= + for candidate in /lib/ld-musl-x86_64.so.1 /lib64/ld-musl-x86_64.so.1 /usr/lib/ld-musl-x86_64.so.1; do + if [ -e "$candidate" ]; then + musl_loader=$candidate + break + fi + done + [ -n "$musl_loader" ] || { + echo "setup: host libc is neither glibc nor musl" >&2 + exit 1 + } + jdk_libc=musl + jdk_platform=alpine-linux + jdk_sha256=$JDK_MUSL_SHA256 +fi +jdk="$dir/jdk-$JDK_RELEASE-$jdk_libc" +jdk_mark="$JDK_RELEASE $jdk_libc" +vulncheck="$dir/govulncheck-$GOVULNCHECK_VERSION-$jdk_libc" +vulncheck_mark="$GOVULNCHECK_VERSION $GO_VERSION $jdk_libc" + +mkdir -p "$dir/dl" download() { url=$1 file=$2 want=$3 - curl -fsSL "$url" -o "$file" - printf '%s %s\n' "$want" "$file" | sha256sum -c - + if [ -f "$file" ]; then + actual=$(sha256sum "$file") + if [ "${actual%% *}" = "$want" ]; then + return + fi + echo "setup: removing corrupt $(basename "$file")" >&2 + rm -f "$file" + fi + part="$file.part.$$" + curl --fail --silent --show-error --location --retry 3 \ + --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + "$url" -o "$part" + actual=$(sha256sum "$part") + [ "${actual%% *}" = "$want" ] || { + echo "setup: checksum failed for $(basename "$file")" >&2 + exit 1 + } + mv "$part" "$file" + part= +} + +mark_complete() { + cat <<EOF > "$1/.rsend-complete" +$2 +EOF +} + +marked() { + [ -f "$1/.rsend-complete" ] && [ "$(cat "$1/.rsend-complete")" = "$2" ] +} + +java_works() { + java_root=$1 + version_output=$("$java_root/bin/java" -version 2>&1) || return 1 + case "$version_output" in + *"$JDK_RELEASE"*) return 0 ;; + *) return 1 ;; + esac +} + +gradle_works() { + gradle_root=$1 + version_output=$("$gradle_root/bin/gradle" --version 2>/dev/null) || return 1 + grep -Fqx "Gradle $GRADLE_VERSION" <<EOF +$version_output +EOF +} + +go_works() { + go_root=$1 + version_output=$(GOROOT="$go_root" "$go_root/bin/go" version 2>/dev/null) || return 1 + case "$version_output" in + *"go version go$GO_VERSION "*) return 0 ;; + *) return 1 ;; + esac } -# JDK (Temurin, exact pinned release; + must be %2B in the URL). -if [ ! -x "$jdk/bin/java" ]; then - echo "setup: JDK $JDK_RELEASE" +govulncheck_works() { + vulncheck_root=$1 + version_output=$("$vulncheck_root/bin/govulncheck" -version 2>/dev/null) || return 1 + grep -Fqx "Scanner: govulncheck@v$GOVULNCHECK_VERSION" <<EOF +$version_output +EOF +} + +java_ready() { + [ -x "$jdk/bin/java" ] && marked "$jdk" "$jdk_mark" && java_works "$jdk" +} + +gradle_ready() { + [ -x "$gradle/bin/gradle" ] && marked "$gradle" "$GRADLE_VERSION" && + gradle_works "$gradle" +} + +go_ready() { + [ -x "$goroot/bin/go" ] && marked "$goroot" "$GO_VERSION" && + go_works "$goroot" +} + +govulncheck_ready() { + [ -x "$vulncheck/bin/govulncheck" ] && marked "$vulncheck" "$vulncheck_mark" && + govulncheck_works "$vulncheck" +} + +# JDK (Temurin, exact pinned release and host libc; + must be %2B in the URL). +if ! java_ready; then + echo "setup: JDK $JDK_RELEASE ($jdk_libc)" rel=$(printf '%s' "$JDK_RELEASE" | sed 's/+/%2B/') - download "https://api.adoptium.net/v3/binary/version/jdk-${rel}/linux/x64/jdk/hotspot/normal/eclipse" \ - "$dir/dl/jdk-$JDK_RELEASE.tar.gz" "$JDK_SHA256" - mkdir -p "$jdk" - tar xzf "$dir/dl/jdk-$JDK_RELEASE.tar.gz" -C "$jdk" --strip-components=1 + jdk_archive="$dir/dl/jdk-$JDK_RELEASE-$jdk_libc.tar.gz" + download "https://api.adoptium.net/v3/binary/version/jdk-${rel}/${jdk_platform}/x64/jdk/hotspot/normal/eclipse" \ + "$jdk_archive" "$jdk_sha256" + stage=$(mktemp -d "$dir/.jdk-$JDK_RELEASE-$jdk_libc.XXXXXX") + tar xzf "$jdk_archive" -C "$stage" --strip-components=1 + if [ ! -x "$stage/bin/java" ] || ! java_works "$stage"; then + echo "setup: JDK cannot run on this host" >&2 + exit 1 + fi + mark_complete "$stage" "$jdk_mark" + rm -rf "$jdk" + mv "$stage" "$jdk" + stage= fi export JAVA_HOME="$jdk" export PATH="$JAVA_HOME/bin:$PATH" -# Android cmdline-tools. -if [ ! -x "$cmdline/bin/sdkmanager" ]; then +# Android command-line tools. +if [ ! -x "$cmdline/bin/sdkmanager" ] || ! marked "$cmdline" "$ANDROID_CMDLINE_TOOLS" || + ! "$cmdline/bin/sdkmanager" --sdk_root="$sdk" --version >/dev/null 2>&1; then echo "setup: cmdline-tools $ANDROID_CMDLINE_TOOLS" download "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_TOOLS}_latest.zip" \ "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS.zip" "$ANDROID_CMDLINE_TOOLS_SHA256" - rm -rf "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS" - python3 -m zipfile -e "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS.zip" "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS" mkdir -p "$sdk/cmdline-tools" + stage=$(mktemp -d "$sdk/cmdline-tools/.cmdline-$ANDROID_CMDLINE_TOOLS.XXXXXX") + python3 -m zipfile -e "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS.zip" "$stage" + chmod +x "$stage/cmdline-tools/bin/"* + [ -x "$stage/cmdline-tools/bin/sdkmanager" ] || { + echo "setup: command-line tools archive has no sdkmanager" >&2 + exit 1 + } + "$stage/cmdline-tools/bin/sdkmanager" --sdk_root="$sdk" --version >/dev/null + mark_complete "$stage/cmdline-tools" "$ANDROID_CMDLINE_TOOLS" rm -rf "$cmdline" - mv "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS/cmdline-tools" "$cmdline" - chmod +x "$cmdline/bin/"* + mv "$stage/cmdline-tools" "$cmdline" + rm -rf "$stage" + stage= fi -# SDK packages. -echo "setup: sdk packages" -yes | "$cmdline/bin/sdkmanager" --licenses >/dev/null -"$cmdline/bin/sdkmanager" \ - "platforms;android-${ANDROID_PLATFORM}" \ - "build-tools;${ANDROID_BUILD_TOOLS}" \ - "ndk;${ANDROID_NDK}" >/dev/null +# SDK packages. A complete pinned install needs no network on repeated runs. +export ANDROID_HOME="$sdk" +export ANDROID_SDK_ROOT="$sdk" +sdk_packages_ready() { + [ -f "$sdk/platforms/android-$ANDROID_PLATFORM/android.jar" ] && + grep -Fqx "AndroidVersion.ApiLevel=$ANDROID_PLATFORM" \ + "$sdk/platforms/android-$ANDROID_PLATFORM/source.properties" && + grep -Fqx "Pkg.Revision=$ANDROID_PLATFORM_REVISION" \ + "$sdk/platforms/android-$ANDROID_PLATFORM/source.properties" && + platform_sum=$(sha256sum "$sdk/platforms/android-$ANDROID_PLATFORM/android.jar") && + [ "${platform_sum%% *}" = "$ANDROID_PLATFORM_JAR_SHA256" ] && + [ -x "$sdk/build-tools/$ANDROID_BUILD_TOOLS/aapt" ] && + grep -Fqx "Pkg.Revision=$ANDROID_BUILD_TOOLS" \ + "$sdk/build-tools/$ANDROID_BUILD_TOOLS/source.properties" && + [ -d "$sdk/ndk/$ANDROID_NDK" ] && + grep -Fqx "Pkg.Revision = $ANDROID_NDK" \ + "$sdk/ndk/$ANDROID_NDK/source.properties" && + [ -s "$sdk/licenses/android-sdk-license" ] +} +if ! sdk_packages_ready; then + echo "setup: SDK packages" + license_answers="$dir/dl/android-license-answers" + awk 'BEGIN { for (i = 0; i < 100; i++) print "y" }' > "$license_answers" + "$cmdline/bin/sdkmanager" --sdk_root="$sdk" --licenses \ + < "$license_answers" >/dev/null + "$cmdline/bin/sdkmanager" --sdk_root="$sdk" \ + "platforms;android-${ANDROID_PLATFORM}" \ + "build-tools;${ANDROID_BUILD_TOOLS}" \ + "ndk;${ANDROID_NDK}" >/dev/null + sdk_packages_ready || { + echo "setup: Android SDK packages are incomplete" >&2 + exit 1 + } +fi +ndk_clang="$sdk/ndk/$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" +if [ ! -x "$ndk_clang" ] || ! "$ndk_clang" --version >/dev/null 2>&1; then + echo "setup: Android NDK cannot run on this host (Alpine needs gcompat)" >&2 + exit 1 +fi # Gradle. -if [ ! -x "$dir/gradle-${GRADLE_VERSION}/bin/gradle" ]; then +if ! gradle_ready; then echo "setup: Gradle $GRADLE_VERSION" download "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" \ "$dir/dl/gradle-$GRADLE_VERSION.zip" "$GRADLE_SHA256" - python3 -m zipfile -e "$dir/dl/gradle-$GRADLE_VERSION.zip" "$dir" - chmod +x "$dir/gradle-${GRADLE_VERSION}/bin/gradle" + stage=$(mktemp -d "$dir/.gradle-$GRADLE_VERSION.XXXXXX") + python3 -m zipfile -e "$dir/dl/gradle-$GRADLE_VERSION.zip" "$stage" + chmod +x "$stage/gradle-$GRADLE_VERSION/bin/gradle" + [ -x "$stage/gradle-$GRADLE_VERSION/bin/gradle" ] || { + echo "setup: Gradle archive has no gradle" >&2 + exit 1 + } + gradle_works "$stage/gradle-$GRADLE_VERSION" || { + echo "setup: Gradle cannot run on this host" >&2 + exit 1 + } + mark_complete "$stage/gradle-$GRADLE_VERSION" "$GRADLE_VERSION" + rm -rf "$gradle" + mv "$stage/gradle-$GRADLE_VERSION" "$gradle" + rm -rf "$stage" + stage= fi +export GRADLE_HOME="$gradle" -# Go (pinned; builds rsh). -if [ ! -x "$goroot/bin/go" ]; then +# Go. +if ! go_ready; then echo "setup: Go $GO_VERSION" download "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ "$dir/dl/go-$GO_VERSION.tar.gz" "$GO_SHA256" - mkdir -p "$goroot" - tar xzf "$dir/dl/go-$GO_VERSION.tar.gz" -C "$goroot" --strip-components=1 + stage=$(mktemp -d "$dir/.go-$GO_VERSION.XXXXXX") + tar xzf "$dir/dl/go-$GO_VERSION.tar.gz" -C "$stage" --strip-components=1 + if [ ! -x "$stage/bin/go" ] || ! go_works "$stage"; then + echo "setup: Go cannot run on this host" >&2 + exit 1 + fi + mark_complete "$stage" "$GO_VERSION" + rm -rf "$goroot" + mv "$stage" "$goroot" + stage= +fi +export GOROOT="$goroot" +export PATH="$GOROOT/bin:$PATH" + +# Source-aware vulnerability scanner. It queries the public Go vulnerability +# database when ci/test.sh invokes it; the source tree itself is not uploaded. +if ! govulncheck_ready; then + echo "setup: govulncheck $GOVULNCHECK_VERSION" + stage=$(mktemp -d "$dir/.govulncheck-$GOVULNCHECK_VERSION-$jdk_libc.XXXXXX") + mkdir -p "$stage/bin" + GOBIN="$stage/bin" "$goroot/bin/go" install \ + "golang.org/x/vuln/cmd/govulncheck@v${GOVULNCHECK_VERSION}" + if [ ! -x "$stage/bin/govulncheck" ] || ! govulncheck_works "$stage"; then + echo "setup: govulncheck install failed" >&2 + exit 1 + fi + mark_complete "$stage" "$vulncheck_mark" + rm -rf "$vulncheck" + mv "$stage" "$vulncheck" + stage= fi -# Emit the env to source before building. -cat > "$dir/env.sh" <<EOF +# Emit the environment to source before building. +cat <<EOF > "$dir/env.sh" export JAVA_HOME="$jdk" export GOROOT="$goroot" export ANDROID_HOME="$sdk" export ANDROID_SDK_ROOT="$sdk" export ANDROID_NDK_HOME="$sdk/ndk/${ANDROID_NDK}" -export GRADLE_HOME="$dir/gradle-${GRADLE_VERSION}" -export PATH="\$JAVA_HOME/bin:\$GOROOT/bin:\$GRADLE_HOME/bin:$cmdline/bin:$sdk/build-tools/${ANDROID_BUILD_TOOLS}:\$PATH" +export GRADLE_HOME="$gradle" +export PATH="\$JAVA_HOME/bin:\$GOROOT/bin:\$GRADLE_HOME/bin:$vulncheck/bin:$cmdline/bin:$sdk/build-tools/${ANDROID_BUILD_TOOLS}:\$PATH" EOF echo "setup: done; source $dir/env.sh" |