diff options
| -rw-r--r-- | README | 36 | ||||
| -rw-r--r-- | app/build.gradle | 1 | ||||
| -rwxr-xr-x | ci/build.sh | 4 | ||||
| -rwxr-xr-x | ci/release.sh | 88 | ||||
| -rwxr-xr-x | ci/setup-toolchain.sh | 62 | ||||
| -rwxr-xr-x | ci/test.sh | 21 | ||||
| -rwxr-xr-x | ci/verify-repro.sh | 24 | ||||
| -rw-r--r-- | metadata/reproducible-builds.md | 13 | ||||
| -rw-r--r-- | rsh/e2e_test.go | 2 | ||||
| -rw-r--r-- | versions | 9 |
10 files changed, 183 insertions, 77 deletions
@@ -30,21 +30,35 @@ Layout Build ----- -The repository is self-contained: clone it and the build fetches and pins -everything else from source. The only host prerequisites are a POSIX shell, -curl, python3, tar, and git. The toolchain (JDK, Go, Android SDK and NDK, -Gradle) and the rsync source are downloaded and version-pinned by the build -(see versions for every pin). +The repository is self-contained: clone it and the build fetches everything +else. The host prerequisites are a POSIX shell, coreutils, make, curl, python3, +tar, and git. The toolchain archives and rsync source are version- and +checksum-pinned by the build (see versions for every pin). make setup # provision the toolchain into $HOME/toolchains . "$HOME/toolchains/env.sh" # put it on PATH (do this in each shell) make # rsync (NDK) -> rsh (Go) -> APK - make test # Go tests plus the host-side end-to-end push - make verify-repro # build twice, diff the artifacts - -The APK lands under app/build/outputs/apk/. It is unsigned by default; signing -config is local and gitignored. Continuous integration runs the same scripts: -ci/setup-toolchain.sh then ci/build.sh and ci/test.sh. + make test # Go, real-rsync integration, and Kotlin tests + make verify-repro # build twice, compare the unsigned APKs + +`make test` also requires host rsync. The APK lands under +app/build/outputs/apk/. It is unsigned by default and cannot be installed until +signed. Continuous integration runs the same scripts: ci/setup-toolchain.sh, +ci/build.sh, and ci/test.sh. + +For a locally installable APK, create a keystore and a gitignored +keystore.properties before building: + + keytool -genkeypair -keystore rsend.jks -alias rsend -keyalg EC -validity 3650 + cat > keystore.properties <<EOF + storeFile=rsend.jks + storePassword=change-me + keyAlias=rsend + keyPassword=change-me + EOF + +Keep both files private and backed up. The signed output is +app/build/outputs/apk/release/app-release.apk. Run diff --git a/app/build.gradle b/app/build.gradle index 5cc640d..599b419 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -11,6 +11,7 @@ def keystoreProps = rootProject.file('keystore.properties') android { namespace 'invalid.lena.rsend' compileSdk 35 + buildToolsVersion '35.0.0' defaultConfig { applicationId 'invalid.lena.rsend' diff --git a/ci/build.sh b/ci/build.sh index 5afbfd4..c99ec5f 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -5,8 +5,8 @@ set -eu root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -make -C "$root" native make -C "$root" app +git -C "$root" rev-parse HEAD > "$root/app/build/outputs/apk/release/.source-commit" echo "ci: APK at app/build/outputs/apk/release/" -ls -l "$root"/app/build/outputs/apk/release/*.apk 2>/dev/null || true +ls -l "$root"/app/build/outputs/apk/release/*.apk diff --git a/ci/release.sh b/ci/release.sh index 260efba..1c96d1b 100755 --- a/ci/release.sh +++ b/ci/release.sh @@ -14,38 +14,90 @@ set -eu : "${CODEBERG_REPO:?set CODEBERG_REPO as owner/name}" tag=${TAG:-${CI_COMMIT_TAG:?set TAG or CI_COMMIT_TAG}} base=${CODEBERG_URL:-https://codeberg.org} -apk=app/build/outputs/apk/release/app-release.apk +base=${base%/} +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +apk="$root/app/build/outputs/apk/release/app-release.apk" +source_commit="$root/app/build/outputs/apk/release/.source-commit" [ -f "$apk" ] || { echo "release: $apk not found; run ci/build.sh first" >&2; exit 1; } +[ -f "$source_commit" ] || { echo "release: build provenance not found; run ci/build.sh first" >&2; exit 1; } +command -v apksigner >/dev/null 2>&1 || { echo "release: apksigner is required" >&2; exit 1; } +apksigner verify --verbose --print-certs "$apk" + +[ -z "$(git -C "$root" status --porcelain --untracked-files=normal)" ] || { + echo "release: working tree is not clean" >&2 + exit 1 +} + +case $CODEBERG_REPO in + *[!A-Za-z0-9._/-]*|*/*/*|/*|*/) + echo "release: invalid CODEBERG_REPO" >&2 + exit 1 + ;; + */*) ;; + *) + echo "release: CODEBERG_REPO must be owner/name" >&2 + exit 1 + ;; +esac + +head=$(git -C "$root" rev-parse HEAD) +IFS= read -r built_commit < "$source_commit" +[ "$built_commit" = "$head" ] || { echo "release: APK was not built from HEAD" >&2; exit 1; } +git check-ref-format "refs/tags/$tag" >/dev/null 2>&1 || { echo "release: invalid tag $tag" >&2; exit 1; } +tag_head=$(git -C "$root" rev-parse --verify "$tag^{commit}") +[ "$head" = "$tag_head" ] || { echo "release: $tag does not point to HEAD" >&2; exit 1; } +version=$(python3 -c 'import re,sys +s=open(sys.argv[1]).read() +m=re.search(r"^\s*versionName [\x27\x22]([^\x27\x22]+)", s, re.M) +print(m.group(1) if m else "")' "$root/app/build.gradle") +[ "$tag" = "$version" ] || { echo "release: tag $tag does not match app version $version" >&2; exit 1; } + +quote() { + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1" +} api="$base/api/v1/repos/$CODEBERG_REPO" auth="Authorization: token $CODEBERG_TOKEN" name="rsend-$tag.apk" +sum_name="$name.sha256" +tag_url=$(quote "$tag") +name_url=$(quote "$name") +sum_name_url=$(quote "$sum_name") +sum=$(mktemp) +trap 'rm -f "$sum"' 0 +sha256sum "$apk" | { read -r hash _; printf '%s %s\n' "$hash" "$name"; } > "$sum" -# Reuse the tag's release if it exists, else create it. Re-runs are idempotent: -# same release, and a stale asset from an earlier run is replaced below. -id=$(curl -fsS -H "$auth" "$api/releases/tags/$tag" 2>/dev/null \ +# Reuse the tag's release if it exists, else create it. Published assets are +# immutable: a rerun uploads only missing assets and never replaces one. +id=$(curl -fsS -H "$auth" "$api/releases/tags/$tag_url" 2>/dev/null \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' 2>/dev/null) || id= if [ -n "$id" ]; then - curl -fsS -H "$auth" "$api/releases/$id/assets" \ - | python3 -c 'import json,sys -for a in json.load(sys.stdin): - if a["name"] == sys.argv[1]: - print(a["id"])' "$name" \ - | while read -r aid; do - curl -fsS -X DELETE -H "$auth" "$api/releases/$id/assets/$aid" - done + assets=$(curl -fsS -H "$auth" "$api/releases/$id/assets") else + payload=$(python3 -c 'import json,sys; print(json.dumps({"tag_name":sys.argv[1],"name":sys.argv[1]}))' "$tag") id=$(curl -fsS -X POST "$api/releases" \ -H "$auth" -H "Content-Type: application/json" \ - -d "{\"tag_name\":\"$tag\",\"name\":\"$tag\"}" \ + -d "$payload" \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])') fi [ -n "$id" ] || { echo "release: could not create release for $tag" >&2; exit 1; } -# Upload the APK as a release asset. -curl -fsS -X POST "$api/releases/$id/assets?name=$name" \ - -H "$auth" \ - -F "attachment=@$apk" >/dev/null +has_asset() { + printf '%s' "${assets:-[]}" | python3 -c 'import json,sys +name=sys.argv[1] +raise SystemExit(0 if any(a["name"] == name for a in json.load(sys.stdin)) else 1)' "$1" +} + +if ! has_asset "$name"; then + curl -fsS -X POST "$api/releases/$id/assets?name=$name_url" \ + -H "$auth" \ + -F "attachment=@$apk" >/dev/null +fi +if ! has_asset "$sum_name"; then + curl -fsS -X POST "$api/releases/$id/assets?name=$sum_name_url" \ + -H "$auth" \ + -F "attachment=@$sum" >/dev/null +fi -echo "release: uploaded $name to $CODEBERG_REPO ($tag)" +echo "release: $name and $sum_name are present on $CODEBERG_REPO ($tag)" diff --git a/ci/setup-toolchain.sh b/ci/setup-toolchain.sh index 7aeaa42..6ed09c6 100755 --- a/ci/setup-toolchain.sh +++ b/ci/setup-toolchain.sh @@ -11,35 +11,47 @@ root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) dir=${TOOLCHAIN_DIR:-$HOME/toolchains} sdk="$dir/android-sdk" mkdir -p "$dir/dl" +jdk="$dir/jdk-$JDK_RELEASE" +cmdline="$sdk/cmdline-tools/$ANDROID_CMDLINE_TOOLS" +goroot="$dir/go-$GO_VERSION" + +download() { + url=$1 + file=$2 + want=$3 + curl -fsSL "$url" -o "$file" + printf '%s %s\n' "$want" "$file" | sha256sum -c - +} # JDK (Temurin, exact pinned release; + must be %2B in the URL). -if [ ! -x "$dir/jdk17/bin/java" ]; then +if [ ! -x "$jdk/bin/java" ]; then echo "setup: JDK $JDK_RELEASE" rel=$(printf '%s' "$JDK_RELEASE" | sed 's/+/%2B/') - curl -fsSL "https://api.adoptium.net/v3/binary/version/jdk-${rel}/linux/x64/jdk/hotspot/normal/eclipse" -o "$dir/dl/jdk.tar.gz" - mkdir -p "$dir/jdk17" - tar xzf "$dir/dl/jdk.tar.gz" -C "$dir/jdk17" --strip-components=1 + download "https://api.adoptium.net/v3/binary/version/jdk-${rel}/linux/x64/jdk/hotspot/normal/eclipse" \ + "$dir/dl/jdk-$JDK_RELEASE.tar.gz" "$JDK_SHA256" + mkdir -p "$jdk" + tar xzf "$dir/dl/jdk-$JDK_RELEASE.tar.gz" -C "$jdk" --strip-components=1 fi -export JAVA_HOME="$dir/jdk17" +export JAVA_HOME="$jdk" export PATH="$JAVA_HOME/bin:$PATH" # Android cmdline-tools. -if [ ! -x "$sdk/cmdline-tools/latest/bin/sdkmanager" ]; then +if [ ! -x "$cmdline/bin/sdkmanager" ]; then echo "setup: cmdline-tools $ANDROID_CMDLINE_TOOLS" - curl -fsSL "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_TOOLS}_latest.zip" -o "$dir/dl/clt.zip" - rm -rf "$dir/dl/clt" - python3 -m zipfile -e "$dir/dl/clt.zip" "$dir/dl/clt" + download "https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_CMDLINE_TOOLS}_latest.zip" \ + "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS.zip" "$ANDROID_CMDLINE_TOOLS_SHA256" + rm -rf "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS" + python3 -m zipfile -e "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS.zip" "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS" mkdir -p "$sdk/cmdline-tools" - rm -rf "$sdk/cmdline-tools/latest" - mv "$dir/dl/clt/cmdline-tools" "$sdk/cmdline-tools/latest" - chmod +x "$sdk/cmdline-tools/latest/bin/"* + rm -rf "$cmdline" + mv "$dir/dl/clt-$ANDROID_CMDLINE_TOOLS/cmdline-tools" "$cmdline" + chmod +x "$cmdline/bin/"* fi # SDK packages. echo "setup: sdk packages" -yes | "$sdk/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null 2>&1 || true -"$sdk/cmdline-tools/latest/bin/sdkmanager" \ - "platform-tools" \ +yes | "$cmdline/bin/sdkmanager" --licenses >/dev/null +"$cmdline/bin/sdkmanager" \ "platforms;android-${ANDROID_PLATFORM}" \ "build-tools;${ANDROID_BUILD_TOOLS}" \ "ndk;${ANDROID_NDK}" >/dev/null @@ -47,28 +59,30 @@ yes | "$sdk/cmdline-tools/latest/bin/sdkmanager" --licenses >/dev/null 2>&1 || t # Gradle. if [ ! -x "$dir/gradle-${GRADLE_VERSION}/bin/gradle" ]; then echo "setup: Gradle $GRADLE_VERSION" - curl -fsSL "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" -o "$dir/dl/gradle.zip" - python3 -m zipfile -e "$dir/dl/gradle.zip" "$dir" + download "https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip" \ + "$dir/dl/gradle-$GRADLE_VERSION.zip" "$GRADLE_SHA256" + python3 -m zipfile -e "$dir/dl/gradle-$GRADLE_VERSION.zip" "$dir" chmod +x "$dir/gradle-${GRADLE_VERSION}/bin/gradle" fi # Go (pinned; builds rsh). -if [ ! -x "$dir/go/bin/go" ]; then +if [ ! -x "$goroot/bin/go" ]; then echo "setup: Go $GO_VERSION" - curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" -o "$dir/dl/go.tar.gz" - rm -rf "$dir/go" - tar xzf "$dir/dl/go.tar.gz" -C "$dir" + download "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ + "$dir/dl/go-$GO_VERSION.tar.gz" "$GO_SHA256" + mkdir -p "$goroot" + tar xzf "$dir/dl/go-$GO_VERSION.tar.gz" -C "$goroot" --strip-components=1 fi # Emit the env to source before building. cat > "$dir/env.sh" <<EOF -export JAVA_HOME="$dir/jdk17" -export GOROOT="$dir/go" +export JAVA_HOME="$jdk" +export GOROOT="$goroot" export ANDROID_HOME="$sdk" export ANDROID_SDK_ROOT="$sdk" export ANDROID_NDK_HOME="$sdk/ndk/${ANDROID_NDK}" export GRADLE_HOME="$dir/gradle-${GRADLE_VERSION}" -export PATH="\$JAVA_HOME/bin:$dir/go/bin:\$GRADLE_HOME/bin:$sdk/cmdline-tools/latest/bin:$sdk/platform-tools:\$PATH" +export PATH="\$JAVA_HOME/bin:\$GOROOT/bin:\$GRADLE_HOME/bin:$cmdline/bin:$sdk/build-tools/${ANDROID_BUILD_TOOLS}:\$PATH" EOF echo "setup: done; source $dir/env.sh" @@ -1,12 +1,14 @@ #!/bin/sh -# Host-verifiable checks: pinned source checksum, Go formatting, vet, and tests -# (including the real-rsync-through-rsh end-to-end). Needs Go and rsync; no -# Android toolchain. This is what runs on every push. +# Full checks: source pins, Go formatting/vet/tests, real-rsync integration, +# and Android JVM tests. Requires the provisioned toolchain and host rsync. set -eu root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) . "$root/versions" +command -v rsync >/dev/null 2>&1 || { echo "ci: rsync is required" >&2; exit 1; } +command -v gradle >/dev/null 2>&1 || { echo "ci: gradle is required" >&2; exit 1; } + if [ -f "$root/rsync/rsync-${RSYNC_VERSION}.tar.gz" ]; then echo "ci: verifying pinned rsync checksum" ( cd "$root/rsync" && sha256sum -c "rsync-${RSYNC_VERSION}.tar.gz.sha256" ) @@ -16,15 +18,21 @@ fi # a shell file); fail loud if they drift. echo "ci: version pins" pin() { - grep -q "$2" "$root/$1" || { echo "ci: $1 does not pin '$2' (see versions)" >&2; exit 1; } + grep -Fq "$2" "$root/$1" || { echo "ci: $1 does not pin '$2' (see versions)" >&2; exit 1; } } pin build.gradle "version '${AGP_VERSION}'" pin build.gradle "version '${KOTLIN_VERSION}'" pin gradle/wrapper/gradle-wrapper.properties "gradle-${GRADLE_VERSION}-bin.zip" pin app/build.gradle "compileSdk ${ANDROID_PLATFORM}" +pin app/build.gradle "buildToolsVersion '${ANDROID_BUILD_TOOLS}'" pin app/build.gradle "minSdk ${ANDROID_MIN_SDK}" pin app/build.gradle "targetSdk ${ANDROID_TARGET_SDK}" -pin rsh/go.mod "^go ${GO_VERSION}" +pin rsh/go.mod "go ${GO_VERSION}" +pin app/build.gradle "androidx.core:core-ktx:${ANDROIDX_CORE_VERSION}" +pin app/build.gradle "androidx.appcompat:appcompat:${ANDROIDX_APPCOMPAT_VERSION}" +pin app/build.gradle "androidx.work:work-runtime-ktx:${ANDROIDX_WORK_VERSION}" +pin app/build.gradle "junit:junit:${JUNIT_VERSION}" +pin app/build.gradle "org.json:json:${JSON_VERSION}" echo "ci: gofmt" unformatted=$(gofmt -l "$root/rsh") @@ -40,4 +48,7 @@ echo "ci: go vet" echo "ci: go test" ( cd "$root/rsh" && go test ./... ) +echo "ci: Android unit tests" +( cd "$root" && gradle :app:testReleaseUnitTest ) + echo "ci: ok" diff --git a/ci/verify-repro.sh b/ci/verify-repro.sh index d412713..580ccd0 100755 --- a/ci/verify-repro.sh +++ b/ci/verify-repro.sh @@ -1,22 +1,26 @@ #!/bin/sh -# Build the native libs twice and diff them, catching nondeterminism early. -# Requires the Android NDK and Go. +# Build the complete unsigned APK twice and compare it byte for byte. set -eu root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) a=$(mktemp -d) b=$(mktemp -d) -trap 'rm -rf "$a" "$b"' EXIT +trap 'rm -rf "$a" "$b"' 0 -make -C "$root" native -cp -r "$root/app/src/main/jniLibs" "$a/jniLibs" +[ ! -f "$root/keystore.properties" ] || { + echo "verify-repro: keystore.properties must be absent" >&2 + exit 1 +} + +make -C "$root" app +cp "$root/app/build/outputs/apk/release/app-release-unsigned.apk" "$a/app.apk" -rm -rf "$root/app/src/main/jniLibs" "$root/rsync/work" "$root/out" -make -C "$root" native -cp -r "$root/app/src/main/jniLibs" "$b/jniLibs" +rm -rf "$root/app/build" "$root/app/src/main/jniLibs" "$root/rsync/work" "$root/out" +make -C "$root" app +cp "$root/app/build/outputs/apk/release/app-release-unsigned.apk" "$b/app.apk" -if diff -r "$a/jniLibs" "$b/jniLibs"; then - echo "verify-repro: native libs are bit-identical" +if cmp "$a/app.apk" "$b/app.apk"; then + echo "verify-repro: APKs are bit-identical" else echo "verify-repro: NONDETERMINISM detected" >&2 exit 1 diff --git a/metadata/reproducible-builds.md b/metadata/reproducible-builds.md index 91b34f6..6885aa7 100644 --- a/metadata/reproducible-builds.md +++ b/metadata/reproducible-builds.md @@ -5,11 +5,11 @@ How rsend stays reproducible and what F-Droid needs to build it. Pinning ------- -Every tool and source version lives in ./versions: the rsync source and its -sha256, the Go version, the JDK release, the Android NDK, SDK platform, -build-tools, Gradle, the Android Gradle Plugin, and Kotlin. The build never -floats a version. The gradle files and go.mod repeat some pins because gradle -cannot source a shell file; ci/test.sh fails if they drift from ./versions. +Every tool and source version lives in ./versions: the rsync source, Go, JDK, +Android NDK, SDK platform, build-tools, command-line tools, Gradle, the Android +Gradle Plugin, and Kotlin. Directly downloaded archives have repository-owned +checksums. The gradle files and go.mod repeat some pins because gradle cannot +source a shell file; ci/test.sh fails if they drift from ./versions. Determinism @@ -22,7 +22,8 @@ Determinism - APK: no code shrinking (minifyEnabled false), the Google dependency-metadata block is dropped (dependenciesInfo off), and native libs use legacy packaging. -Verify locally by building twice and diffing the native libs: +Verify locally by building the complete unsigned APK twice and comparing it +byte for byte: make verify-repro diff --git a/rsh/e2e_test.go b/rsh/e2e_test.go index 1fa80c7..88cd317 100644 --- a/rsh/e2e_test.go +++ b/rsh/e2e_test.go @@ -23,7 +23,7 @@ func TestEndToEndRealRsync(t *testing.T) { // Build the rsh binary so rsync can exec it as its remote shell. rshBin := filepath.Join(t.TempDir(), "rsh") - if out, err := exec.Command("go", "build", "-o", rshBin, ".").CombinedOutput(); err != nil { + if out, err := exec.Command("go", "build", "-buildvcs=false", "-o", rshBin, ".").CombinedOutput(); err != nil { t.Fatalf("build rsh: %v\n%s", err, out) } @@ -10,6 +10,7 @@ RSYNC_URL=https://download.samba.org/pub/rsync/src/rsync-${RSYNC_VERSION}.tar.gz # Go SSH transport (rsh). Matches the go.mod go directive; x/crypto needs 1.25+. GO_VERSION=1.25.0 +GO_SHA256=2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613 # Android toolchain. ANDROID_NDK is the sdkmanager package revision. # JDK_RELEASE is the exact Temurin GA build; JDK_VERSION its major. @@ -22,8 +23,16 @@ ANDROID_TARGET_SDK=35 AGP_VERSION=8.7.3 GRADLE_VERSION=8.9 KOTLIN_VERSION=2.0.21 +ANDROIDX_CORE_VERSION=1.13.1 +ANDROIDX_APPCOMPAT_VERSION=1.7.0 +ANDROIDX_WORK_VERSION=2.10.0 +JUNIT_VERSION=4.13.2 +JSON_VERSION=20240303 JDK_VERSION=17 JDK_RELEASE=17.0.19+10 +JDK_SHA256=d8afc263758141a66e0e3aafc321e783f7016696f4eaea067d340a269037d331 +ANDROID_CMDLINE_TOOLS_SHA256=2d2d50857e4eb553af5a6dc3ad507a17adf43d115264b1afc116f95c92e5e258 +GRADLE_SHA256=d725d707bfabd4dfdc958c624003b3c80accc03f7037b5122c4b1d0ef15cecab # Native ABIs to build. arm64-v8a ships on the phone; x86_64 is for host and # emulator tests. Add armeabi-v7a to support 32-bit phones. |