From dc0338f8c0c9e74b277169d812c432f7ea4888e3 Mon Sep 17 00:00:00 2001 From: Lena Date: Sun, 23 Aug 2026 00:00:00 +0000 Subject: app: rename RsyncRunner to Rsync Runner names the architecture rather than the thing. The object is the rsync invocation. --- app/src/main/java/invalid/lena/rsend/Rsync.kt | 202 +++++++++++++++++++++ .../main/java/invalid/lena/rsend/RsyncRunner.kt | 202 --------------------- app/src/main/java/invalid/lena/rsend/SyncWorker.kt | 4 +- 3 files changed, 204 insertions(+), 204 deletions(-) create mode 100644 app/src/main/java/invalid/lena/rsend/Rsync.kt delete mode 100644 app/src/main/java/invalid/lena/rsend/RsyncRunner.kt (limited to 'app/src/main') diff --git a/app/src/main/java/invalid/lena/rsend/Rsync.kt b/app/src/main/java/invalid/lena/rsend/Rsync.kt new file mode 100644 index 0000000..5fc73c8 --- /dev/null +++ b/app/src/main/java/invalid/lena/rsend/Rsync.kt @@ -0,0 +1,202 @@ +package invalid.lena.rsend + +import android.content.Context +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.util.concurrent.TimeUnit + +// Rsync builds and runs the rsync invocation for one folder, streaming its +// output into the log. It uses the bundled rsync and rsh binaries and passes +// key, known_hosts, and port to rsh through the environment. +object Rsync { + + // rsh prints this when it cannot reach the host at all. rsync only ever + // reports its own exit 12 for a remote shell that died, which does not say + // why, so the reason is read off rsh's merged stderr instead. + private const val UNREACHABLE = "rsh: unreachable:" + + // Outcome carries rsync's exit code plus whether the failure was the host + // being unreachable, which the caller uses to skip the rest of that remote. + // deleted is what a mirror removed on the server. A mirror is allowed to + // empty its destination, so the count is not a brake; it is how the run + // says what it did, loudly enough to be noticed the same day. + data class Outcome(val code: Int, val unreachable: Boolean, val deleted: Long = 0) + + // rsync exit 24 is "some files vanished before they could be transferred". + // For an additive push that is routine on a live phone: the camera, + // WhatsApp and the media scanner all move files while a sync runs. + // + // For a mirror it is not routine. rsync reports the same 24 when a whole + // source *directory* vanishes mid-run, and the receiver still deletes that + // directory's contents on the server, so calling 24 a success would record + // a run that destroyed remote data as "ok". + private const val EXIT_VANISHED = 24 + + fun succeeded(code: Int, mirror: Boolean): Boolean = + code == 0 || (code == EXIT_VANISHED && !mirror) + + // partialDir keeps a half-transferred file out of the destination name. + // Relative, so rsync excludes it automatically and --delete leaves it alone. + const val PARTIAL_DIR = ".rsend-partial" + + // " ": %o is send, recv or del. + const val OUT_FORMAT = "%o %n" + + // Transfers are counted rather than logged one line each. Deletions and + // everything else go to the log verbatim. + private const val SENT = "send " + private const val DELETED = "del. " + internal const val MAX_OUTPUT_LINE_BYTES = 16 * 1024 + + // args builds the rsync argument vector for one folder. Flags are tuned for + // media backup: recursive, preserve mtimes, resume partial files, and skip + // the ownership and permission bits that mean nothing across Android and a + // server. + // + // No --mkpath: rsync forwards it to the remote when sending, and remotes + // older than 3.2.3 (notably stock macOS) reject it. rsync creates the + // final component of the destination path on its own; deeper missing + // parents need a one-time mkdir on the server (see README). + // + // --out-format labels every line with the operation: "del. " for a + // deletion, "send " for a transfer. That is what makes the log an + // audit trail for mirrors, and it costs nothing in compatibility: rsync + // forwards it to the remote as --log-format, which every rsync back to the + // 2.6 series understands. --info=del would be tidier still, but rsync + // forwards --info verbatim and remotes older than 3.1.0 reject it, the same + // trap --mkpath fell into. + // + // Labelling also keeps filenames out of column 0, where an attacker-chosen + // name could otherwise forge rsh's "unreachable" marker. runFolder counts + // the "send" lines rather than logging each one, so a first sync of a large + // library cannot flood the capped log and rotate the deletions out of it. + fun args(rsh: String, remote: Remote, f: Folder): List { + require(RemoteRules.hostAllowed(remote.host) && RemoteRules.userAllowed(remote.user)) { + "invalid remote endpoint" + } + require(FolderRules.localPathAllowed(f.local)) { "invalid local path" } + require(FolderRules.remotePathAllowed(f.remotePath)) { "invalid remote path" } + require(FolderRules.excludesAllowed(f.excludes)) { "invalid exclude" } + val a = ArrayList() + a.add("-rt") + a.add("--out-format=$OUT_FORMAT") + // Without a partial dir an interrupted transfer renames the truncated + // temp file over the destination name, destroying a complete remote + // copy. Interruption is routine here: the worker is killed when a + // constraint is lost or the execution limit is hit. + a.add("--partial-dir=$PARTIAL_DIR") + // Abort rather than hang if the network stalls for 5 minutes. + a.add("--timeout=300") + a.add("--no-perms") + a.add("--no-owner") + a.add("--no-group") + a.add("--omit-dir-times") + a.add("-e") + a.add(rsh) + for (e in f.excludes) a.add("--exclude=$e") + // --delete-after, not --delete. Deleting during the transfer removes the + // server's copy of a file before its replacement has arrived, and this + // worker is routinely killed mid-run, so a rename on the phone plus one + // interruption leaves the server with neither copy. Deleting only after + // everything is transferred keeps the old copy until the new one is + // safely there. It is as old as rsync, so no remote is excluded. + if (f.delete) a.add("--delete-after") + a.add(withSlash(f.local)) + a.add("${remote.user}@${hostArg(remote.host)}:${withSlash(f.remotePath)}") + return a + } + + private fun withSlash(p: String): String = if (p.endsWith("/")) p else "$p/" + + // An IPv6 literal must be bracketed in rsync's USER@HOST:PATH, or rsync + // splits the destination on the first colon of the address. + private fun hostArg(h: String): String = if (h.contains(":")) "[$h]" else h + + // runFolder execs rsync for one folder, appending every output line to the + // log, and returns rsync's exit code. Blocks the calling thread (run it on + // an IO dispatcher). If the caller is cancelled (worker stopped, schedule + // replaced, constraints lost) the watchdog kills rsync rather than leave it + // running detached; killing it also closes its pipes, which unblocks the + // log reader below. destroy() on an already-exited process is a no-op. + suspend fun runFolder(ctx: Context, remote: Remote, f: Folder, log: SyncLog): Outcome = coroutineScope { + val cmd = listOf(Native.rsync(ctx).absolutePath) + args(Native.rsh(ctx).absolutePath, remote, f) + log.line("rsync ${cmd.drop(1).joinToString(" ")}") + val pb = ProcessBuilder(cmd).redirectErrorStream(true) + pb.environment().putAll(Keys.env(ctx, remote)) + val p = pb.start() + val watchdog = launch(Dispatchers.IO) { try { awaitCancellation() } finally { terminate(p) } } + try { + val t = tally(p.inputStream) { log.line(it) } + if (t.sent > 0) log.line("transferred ${t.sent} item(s)") + if (t.deleted > 0) log.line("deleted ${t.deleted} item(s) on the server") + Outcome(p.waitFor(), t.unreachable, t.deleted) + } finally { + watchdog.cancelAndJoin() + } + } + + // A hostile or broken remote can print an arbitrarily long line. Keep + // draining its pipe so rsync cannot deadlock, but retain only a bounded + // prefix for the log. + internal data class Tally(val sent: Long, val deleted: Long, val unreachable: Boolean) + + // tally reads rsync's merged output, counts what it did, and passes + // everything worth keeping to log. Separated from the process so it can be + // tested against real rsync output without spawning one. It streams: a + // first sync of a large library must not be buffered to be counted. + internal fun tally(input: InputStream, log: (String) -> Unit): Tally { + var sent = 0L + var deleted = 0L + var unreachable = false + boundedLines(input) { + if (it.startsWith(UNREACHABLE)) unreachable = true + if (it.startsWith(DELETED)) deleted++ + // One line per transferred file would bury the deletions and the + // errors, and on a first sync would outgrow the whole log. + if (it.startsWith(SENT)) sent++ else log(it) + } + return Tally(sent, deleted, unreachable) + } + + internal fun boundedLines(input: InputStream, emit: (String) -> Unit) { + val line = ByteArrayOutputStream() + val buf = ByteArray(4096) + var truncated = false + while (true) { + val n = input.read(buf) + if (n < 0) break + for (i in 0 until n) { + val b = buf[i].toInt() and 0xff + if (b == '\n'.code) { + emitLine(line, truncated, emit) + line.reset() + truncated = false + } else if (line.size() < MAX_OUTPUT_LINE_BYTES) { + line.write(b) + } else { + truncated = true + } + } + } + if (line.size() > 0 || truncated) emitLine(line, truncated, emit) + } + + private fun emitLine(line: ByteArrayOutputStream, truncated: Boolean, emit: (String) -> Unit) { + val value = String(line.toByteArray(), Charsets.UTF_8).removeSuffix("\r") + + if (truncated) " [truncated]" else "" + emit(value) + } + + private fun terminate(p: Process) { + p.destroy() + if (!p.waitFor(5, TimeUnit.SECONDS)) { + p.destroyForcibly() + p.waitFor() + } + } +} diff --git a/app/src/main/java/invalid/lena/rsend/RsyncRunner.kt b/app/src/main/java/invalid/lena/rsend/RsyncRunner.kt deleted file mode 100644 index f807288..0000000 --- a/app/src/main/java/invalid/lena/rsend/RsyncRunner.kt +++ /dev/null @@ -1,202 +0,0 @@ -package invalid.lena.rsend - -import android.content.Context -import kotlinx.coroutines.awaitCancellation -import kotlinx.coroutines.cancelAndJoin -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import java.io.ByteArrayOutputStream -import java.io.InputStream -import java.util.concurrent.TimeUnit - -// RsyncRunner builds and runs the rsync invocation for one folder, streaming -// rsync's output into the log. It uses the bundled rsync and rsh binaries and -// passes key, known_hosts, and port to rsh through the environment. -object RsyncRunner { - - // rsh prints this when it cannot reach the host at all. rsync only ever - // reports its own exit 12 for a remote shell that died, which does not say - // why, so the reason is read off rsh's merged stderr instead. - private const val UNREACHABLE = "rsh: unreachable:" - - // Outcome carries rsync's exit code plus whether the failure was the host - // being unreachable, which the caller uses to skip the rest of that remote. - // deleted is what a mirror removed on the server. A mirror is allowed to - // empty its destination, so the count is not a brake; it is how the run - // says what it did, loudly enough to be noticed the same day. - data class Outcome(val code: Int, val unreachable: Boolean, val deleted: Long = 0) - - // rsync exit 24 is "some files vanished before they could be transferred". - // For an additive push that is routine on a live phone: the camera, - // WhatsApp and the media scanner all move files while a sync runs. - // - // For a mirror it is not routine. rsync reports the same 24 when a whole - // source *directory* vanishes mid-run, and the receiver still deletes that - // directory's contents on the server, so calling 24 a success would record - // a run that destroyed remote data as "ok". - private const val EXIT_VANISHED = 24 - - fun succeeded(code: Int, mirror: Boolean): Boolean = - code == 0 || (code == EXIT_VANISHED && !mirror) - - // partialDir keeps a half-transferred file out of the destination name. - // Relative, so rsync excludes it automatically and --delete leaves it alone. - const val PARTIAL_DIR = ".rsend-partial" - - // " ": %o is send, recv or del. - const val OUT_FORMAT = "%o %n" - - // Transfers are counted rather than logged one line each. Deletions and - // everything else go to the log verbatim. - private const val SENT = "send " - private const val DELETED = "del. " - internal const val MAX_OUTPUT_LINE_BYTES = 16 * 1024 - - // args builds the rsync argument vector for one folder. Flags are tuned for - // media backup: recursive, preserve mtimes, resume partial files, and skip - // the ownership and permission bits that mean nothing across Android and a - // server. - // - // No --mkpath: rsync forwards it to the remote when sending, and remotes - // older than 3.2.3 (notably stock macOS) reject it. rsync creates the - // final component of the destination path on its own; deeper missing - // parents need a one-time mkdir on the server (see README). - // - // --out-format labels every line with the operation: "del. " for a - // deletion, "send " for a transfer. That is what makes the log an - // audit trail for mirrors, and it costs nothing in compatibility: rsync - // forwards it to the remote as --log-format, which every rsync back to the - // 2.6 series understands. --info=del would be tidier still, but rsync - // forwards --info verbatim and remotes older than 3.1.0 reject it, the same - // trap --mkpath fell into. - // - // Labelling also keeps filenames out of column 0, where an attacker-chosen - // name could otherwise forge rsh's "unreachable" marker. runFolder counts - // the "send" lines rather than logging each one, so a first sync of a large - // library cannot flood the capped log and rotate the deletions out of it. - fun args(rsh: String, remote: Remote, f: Folder): List { - require(RemoteRules.hostAllowed(remote.host) && RemoteRules.userAllowed(remote.user)) { - "invalid remote endpoint" - } - require(FolderRules.localPathAllowed(f.local)) { "invalid local path" } - require(FolderRules.remotePathAllowed(f.remotePath)) { "invalid remote path" } - require(FolderRules.excludesAllowed(f.excludes)) { "invalid exclude" } - val a = ArrayList() - a.add("-rt") - a.add("--out-format=$OUT_FORMAT") - // Without a partial dir an interrupted transfer renames the truncated - // temp file over the destination name, destroying a complete remote - // copy. Interruption is routine here: the worker is killed when a - // constraint is lost or the execution limit is hit. - a.add("--partial-dir=$PARTIAL_DIR") - // Abort rather than hang if the network stalls for 5 minutes. - a.add("--timeout=300") - a.add("--no-perms") - a.add("--no-owner") - a.add("--no-group") - a.add("--omit-dir-times") - a.add("-e") - a.add(rsh) - for (e in f.excludes) a.add("--exclude=$e") - // --delete-after, not --delete. Deleting during the transfer removes the - // server's copy of a file before its replacement has arrived, and this - // worker is routinely killed mid-run, so a rename on the phone plus one - // interruption leaves the server with neither copy. Deleting only after - // everything is transferred keeps the old copy until the new one is - // safely there. It is as old as rsync, so no remote is excluded. - if (f.delete) a.add("--delete-after") - a.add(withSlash(f.local)) - a.add("${remote.user}@${hostArg(remote.host)}:${withSlash(f.remotePath)}") - return a - } - - private fun withSlash(p: String): String = if (p.endsWith("/")) p else "$p/" - - // An IPv6 literal must be bracketed in rsync's USER@HOST:PATH, or rsync - // splits the destination on the first colon of the address. - private fun hostArg(h: String): String = if (h.contains(":")) "[$h]" else h - - // runFolder execs rsync for one folder, appending every output line to the - // log, and returns rsync's exit code. Blocks the calling thread (run it on - // an IO dispatcher). If the caller is cancelled (worker stopped, schedule - // replaced, constraints lost) the watchdog kills rsync rather than leave it - // running detached; killing it also closes its pipes, which unblocks the - // log reader below. destroy() on an already-exited process is a no-op. - suspend fun runFolder(ctx: Context, remote: Remote, f: Folder, log: SyncLog): Outcome = coroutineScope { - val cmd = listOf(Native.rsync(ctx).absolutePath) + args(Native.rsh(ctx).absolutePath, remote, f) - log.line("rsync ${cmd.drop(1).joinToString(" ")}") - val pb = ProcessBuilder(cmd).redirectErrorStream(true) - pb.environment().putAll(Keys.env(ctx, remote)) - val p = pb.start() - val watchdog = launch(Dispatchers.IO) { try { awaitCancellation() } finally { terminate(p) } } - try { - val t = tally(p.inputStream) { log.line(it) } - if (t.sent > 0) log.line("transferred ${t.sent} item(s)") - if (t.deleted > 0) log.line("deleted ${t.deleted} item(s) on the server") - Outcome(p.waitFor(), t.unreachable, t.deleted) - } finally { - watchdog.cancelAndJoin() - } - } - - // A hostile or broken remote can print an arbitrarily long line. Keep - // draining its pipe so rsync cannot deadlock, but retain only a bounded - // prefix for the log. - internal data class Tally(val sent: Long, val deleted: Long, val unreachable: Boolean) - - // tally reads rsync's merged output, counts what it did, and passes - // everything worth keeping to log. Separated from the process so it can be - // tested against real rsync output without spawning one. It streams: a - // first sync of a large library must not be buffered to be counted. - internal fun tally(input: InputStream, log: (String) -> Unit): Tally { - var sent = 0L - var deleted = 0L - var unreachable = false - boundedLines(input) { - if (it.startsWith(UNREACHABLE)) unreachable = true - if (it.startsWith(DELETED)) deleted++ - // One line per transferred file would bury the deletions and the - // errors, and on a first sync would outgrow the whole log. - if (it.startsWith(SENT)) sent++ else log(it) - } - return Tally(sent, deleted, unreachable) - } - - internal fun boundedLines(input: InputStream, emit: (String) -> Unit) { - val line = ByteArrayOutputStream() - val buf = ByteArray(4096) - var truncated = false - while (true) { - val n = input.read(buf) - if (n < 0) break - for (i in 0 until n) { - val b = buf[i].toInt() and 0xff - if (b == '\n'.code) { - emitLine(line, truncated, emit) - line.reset() - truncated = false - } else if (line.size() < MAX_OUTPUT_LINE_BYTES) { - line.write(b) - } else { - truncated = true - } - } - } - if (line.size() > 0 || truncated) emitLine(line, truncated, emit) - } - - private fun emitLine(line: ByteArrayOutputStream, truncated: Boolean, emit: (String) -> Unit) { - val value = String(line.toByteArray(), Charsets.UTF_8).removeSuffix("\r") + - if (truncated) " [truncated]" else "" - emit(value) - } - - private fun terminate(p: Process) { - p.destroy() - if (!p.waitFor(5, TimeUnit.SECONDS)) { - p.destroyForcibly() - p.waitFor() - } - } -} diff --git a/app/src/main/java/invalid/lena/rsend/SyncWorker.kt b/app/src/main/java/invalid/lena/rsend/SyncWorker.kt index 6bc2b2c..4f619dc 100644 --- a/app/src/main/java/invalid/lena/rsend/SyncWorker.kt +++ b/app/src/main/java/invalid/lena/rsend/SyncWorker.kt @@ -107,7 +107,7 @@ class SyncWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, log.line("error: local source \"${f.local}\" is not a readable directory") 1 } else try { - val r = RsyncRunner.runFolder(ctx, remote, f, log) + val r = Rsync.runFolder(ctx, remote, f, log) if (r.unreachable) unreachable.add(f.remoteName) deleted += r.deleted r.code @@ -118,7 +118,7 @@ class SyncWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx, 1 } log.line("exit=$code") - if (!RsyncRunner.succeeded(code, f.delete)) { + if (!Rsync.succeeded(code, f.delete)) { ok = false failed.add(label) } -- cgit v1.2.3