path: root/vmm

#!/bin/sh
#
# vmm - QEMU/KVM virtual machine manager. State is one directory of
# plain files per VM under $VMMDIR (default $HOME/.vm). The config is
# never sourced and never eval'd, and there is no way to inject raw
# QEMU arguments.

set -eu
umask 077
# Exported: the parser's [:print:] and every grep, sed and tr below must
# match bytes, not whatever the caller's locale calls a character.
export LC_ALL=C

SHUTDOWN_TIMEOUT=${SHUTDOWN_TIMEOUT:-10}
MONITOR_TIMEOUT=${MONITOR_TIMEOUT:-10}

QMPSEQ=0
VMPID=
PID=

log() {
	printf 'vmm: %s\n' "$*" >&2
}

fail() {
	_code=$1
	shift
	log "$@"
	exit "$_code"
}

usage() {
	cat >&2 <<'EOF'
Usage: vmm <command> [args]

  create <name> [size]   Create config and disk, then edit (default 10G).
  edit <name>            Edit config, then re-validate it.
  start <name>           Start VM. Does nothing if already running.
  stop <name>            Graceful ACPI shutdown. Exit 5 if it had to force.
  restart <name>         Stop then start.
  kill <name>            SIGKILL. Does nothing if already stopped.
  status <name>          Print state. Exit 0 running, 3 stopped, 4 unknown.
  list [-q]              List VMs and state. -q prints bare names.
  console <name>         Attach to the serial console. Ctrl-] detaches.
  viewer <name>          Open the display of a GRAPHICS=vnc or spice guest.
  monitor <name> [cmd]   Run one monitor command, or read them from stdin.
  logs [-n N] <name>     Show QEMU and console logs.
  clone <src> <dst>      Copy a stopped VM, new identity.
  delete [-f] <name>     Remove a stopped VM and its state.
  dryrun <name>          Validate the config and print what would run.

VM names are 1 to 32 characters of A-Za-z0-9._- and may not start with
a dot or a dash.

Environment:
  VMMDIR=$HOME/.vm         where VMs live
  SHUTDOWN_TIMEOUT=10      seconds to wait for ACPI shutdown
  MONITOR_TIMEOUT=10       seconds to wait for a monitor reply
  EDITOR=vi                editor for 'vmm create' and 'vmm edit'
  VIEWER=remote-viewer     display client for 'vmm viewer'
EOF
	exit 1
}

vm_check_name() {
	case ${1:-} in
	'')		fail 1 "empty VM name" ;;
	-* | .*)	fail 1 "invalid VM name: $1" ;;
	*[!A-Za-z0-9_.-]*)	fail 1 "invalid VM name: $1" ;;
	esac
	if [ ${#1} -gt 32 ]; then
		fail 1 "VM name longer than 32 characters: $1"
	fi
}

# vm_pid matches the pidfile path against QEMU's argv byte for byte, so
# a second spelling of VMMDIR would hide every guest.
vmmdir_setup() {
	if [ -z "${VMMDIR:-}" ]; then
		if [ -z "${HOME:-}" ]; then
			fail 1 "neither VMMDIR nor HOME is set"
		fi
		VMMDIR=$HOME/.vm
	fi
	case $VMMDIR in
	/*)	;;
	*)	fail 1 "VMMDIR must be an absolute path: $VMMDIR" ;;
	esac
	if [ -d "$VMMDIR" ]; then
		VMMDIR=$(cd "$VMMDIR" && pwd -P) ||
			fail 1 "cannot enter VMMDIR: $VMMDIR"
	fi
}

vm_exists() {
	if [ ! -d "$1" ]; then
		fail 2 "no such VM: ${1##*/}"
	fi
	if [ ! -f "$1/config" ]; then
		fail 2 "no config: $1/config"
	fi
}

# True when PID is dead, unreaped, and down to its last thread. It holds
# no files or sockets then, whatever its procfs directory still shows.
proc_is_zombie() {
	_zstat=$(cat "/proc/$1/stat" 2>/dev/null) || return 1
	_zrest=${_zstat##*) }
	case $_zrest in
	Z\ *)	;;
	*)	return 1 ;;
	esac
	# The group leader reaches Z while a sibling thread can still hold the
	# file table they share, and QEMU's lock on the disk image with it. One
	# word is the leader alone, or a pattern that matched nothing.
	set -- "/proc/$1/task"/*
	[ $# -eq 1 ]
}

# Set PID to the QEMU that owns this VM, or empty when it is stopped.
vm_pid() {
	PID=
	_pf=$1/pid
	if [ ! -f "$_pf" ]; then
		return 0
	fi
	_p=
	IFS= read -r _p 2>/dev/null < "$_pf" || :
	case $_p in
	'' | *[!0-9]*)	return 0 ;;
	esac
	# -pidfile is vmm's final argv pair. ash and dash discard procfs's NUL
	# separators here, leaving an exact suffix that needs no helper process.
	_cmdline=
	IFS= read -r _cmdline 2>/dev/null < "/proc/$_p/cmdline" || :
	case $_cmdline in
	qemu-system-*"-pidfile$_pf" | /*/qemu-system-*"-pidfile$_pf")
		PID=$_p
		;;
	esac
}

# True when state left after vm_pid returned empty is safe to remove.
# An unrecognised live QEMU is preserved so the caller can report it.
vm_state_is_stale() {
	_sf=$1/pid
	if [ ! -f "$_sf" ]; then
		return 0
	fi
	_sp=$(cat "$_sf" 2>/dev/null) || return 0
	case $_sp in
	'' | *[!0-9]*)	return 0 ;;
	esac
	if proc_is_zombie "$_sp"; then
		return 0
	fi
	# A recycled pid running something else is not this VM. An unreadable
	# exe link is another user's process: refused too, because
	# unidentifiable is not the same as dead.
	if _exe=$(readlink "/proc/$_sp/exe" 2>/dev/null); then
		:
	elif [ -d "/proc/$_sp" ]; then
		return 1
	else
		return 0
	fi
	case ${_exe##*/} in
	qemu-system-*)	return 1 ;;
	esac
	return 0
}

# Exit 3 when the guest is stopped, exit 4 when live state vmm cannot
# identify is in the way.
vm_require_running() {
	vm_pid "$1"
	if [ -n "$PID" ]; then
		return 0
	fi
	if ! vm_state_is_stale "$1"; then
		fail 4 "$1 has unrecognised live QEMU state"
	fi
	fail 3 "${1##*/} is not running"
}

# Echoes nothing for a malformed uuid. QEMU rejects one before it opens
# the monitor FIFOs, so it must never reach the argv.
vm_uuid() {
	_u=$(cat "$1/uuid" 2>/dev/null) || return 0
	case $_u in
	????????-????-????-????-????????????)	;;
	*)					return 0 ;;
	esac
	_hex=$(printf '%s' "$_u" | tr -d '-')
	if [ ${#_hex} -ne 32 ]; then
		return 0
	fi
	case $_hex in
	*[!0-9a-f]*)	return 0 ;;
	esac
	printf '%s' "$_u"
}

# The kernel releases the lock when vmm exits or is killed, so there is no
# stale state to recover. Every child that can outlive vmm closes fd 9, or
# the lock outlives vmm with it.
lock_acquire() {
	command -v flock >/dev/null 2>&1 || fail 1 "flock is not on PATH"
	exec 9<"$1"
	flock -n 9 || fail 4 "busy: another vmm holds $1"
}

config_fail() {
	fail 1 "$CONFIG_PATH:$CONFIG_LINE: $*"
}

# QEMU passes the address to inet_aton, which also reads 127.1, octal
# and hexadecimal. Only the dotted quad is accepted, so an address means
# what it looks like.
ipv4_check() {
	case $1 in
	.* | *. | *..* | *[!0-9.]*)	return 1 ;;
	esac
	_qr=$1
	_qn=0
	while [ -n "$_qr" ]; do
		case $_qr in
		*.*)	_qo=${_qr%%.*}; _qr=${_qr#*.} ;;
		*)	_qo=$_qr; _qr= ;;
		esac
		# A leading zero is octal to inet_aton, and four digits are out
		# of range before [ has to compare them.
		case $_qo in
		0?* | ????*)	return 1 ;;
		esac
		if [ "$_qo" -gt 255 ]; then
			return 1
		fi
		_qn=$((_qn + 1))
	done
	[ "$_qn" -eq 4 ]
}

# Validated while the config is read, so config_fail names the line.
hostfwd_check() {
	_rest=$1
	_found=no
	while [ -n "$_rest" ]; do
		case $_rest in
		*,*)	_e=${_rest%%,*}; _rest=${_rest#*,} ;;
		*)	_e=$_rest; _rest= ;;
		esac
		case $_e in
		'')	continue ;;
		*:*:*)	_a=${_e%%:*}; _t=${_e#*:}; _p1=${_t%%:*}; _p2=${_t#*:} ;;
		*:*)	_a=127.0.0.1; _p1=${_e%%:*}; _p2=${_e#*:} ;;
		*)	config_fail "HOSTFWD entry is not HOST:GUEST or ADDR:HOST:GUEST: $_e" ;;
		esac
		_found=yes
		if ! ipv4_check "$_a"; then
			config_fail "HOSTFWD bind address must be A.B.C.D: $_e"
		fi
		# Six digits or more is out of range without arithmetic, which
		# stops [ from choking on a number too big for it.
		for _p in "$_p1" "$_p2"; do
			case $_p in
			'' | *[!0-9]* | 0* | ??????*)
				config_fail "HOSTFWD port out of range 1-65535: $_e"
				;;
			esac
			if [ "$_p" -gt 65535 ]; then
				config_fail "HOSTFWD port out of range 1-65535: $_e"
			fi
		done
	done
	if [ -n "$1" ] && [ "$_found" = no ]; then
		config_fail "HOSTFWD has no usable entries: $1"
	fi
}

# The two field form binds loopback, where a bare tcp::PORT would listen
# on every interface.
hostfwd_args() {
	_rest=$1
	while [ -n "$_rest" ]; do
		case $_rest in
		*,*)	_e=${_rest%%,*}; _rest=${_rest#*,} ;;
		*)	_e=$_rest; _rest= ;;
		esac
		case $_e in
		'')	continue ;;
		*:*:*)	printf ',hostfwd=tcp:%s-:%s' "${_e%:*}" "${_e##*:}" ;;
		*)	printf ',hostfwd=tcp:127.0.0.1:%s-:%s' "${_e%%:*}" "${_e#*:}" ;;
		esac
	done
}

# Sets the globals every other function reads.
config_load() {
	CONFIG_PATH=$1
	CR=$(printf '\r')
	HOSTARCH=$(uname -m)
	if [ ! -r "$CONFIG_PATH" ]; then
		fail 1 "$CONFIG_PATH: not readable"
	fi
	# read(1) cannot carry NUL and would silently remove it.
	_nul_line=$(od -An -tu1 -v "$CONFIG_PATH" | awk '
		{ for (i = 1; i <= NF; i++) {
			if ($i == 0) { print line + 1; exit }
			if ($i == 10) line++
		} }')
	if [ -n "$_nul_line" ]; then
		CONFIG_LINE=$_nul_line
		config_fail "NUL byte in line"
	fi

	CPUS=2
	MEM=2048
	ARCH=$HOSTARCH
	HWACCEL=yes
	FIRMWARE=
	DISK=${CONFIG_PATH%/*}/disk.qcow2
	IMAGE_FORMAT=qcow2
	GRAPHICS=no
	NETWORK=yes
	SNAPSHOT=no
	BOOT_ORDER=c
	BALLOON=yes
	CDROM=
	HOSTFWD=

	_seen=' '
	CONFIG_LINE=0

	# Fed by a redirect, never a pipe: ash runs a pipeline in a subshell
	# and every value set here would be lost.
	while IFS= read -r _line || [ -n "$_line" ]; do
		CONFIG_LINE=$((CONFIG_LINE + 1))

		_line=${_line%"$CR"}
		_ws=${_line%%[![:blank:]]*}; _line=${_line#"$_ws"}
		_ws=${_line##*[![:blank:]]}; _line=${_line%"$_ws"}

		case $_line in
		'' | '#'*)	continue ;;
		*[![:print:]]*)	config_fail "non-printable byte in line" ;;
		*=*)		;;
		*)		config_fail "not a KEY=VALUE line: $_line" ;;
		esac

		_key=${_line%%=*}
		_val=${_line#*=}

		_ws=${_key##*[![:blank:]]}; _key=${_key%"$_ws"}
		_ws=${_val%%[![:blank:]]*}; _val=${_val#"$_ws"}

		case $_key in
		'' | *[!A-Za-z0-9_]*)
			config_fail "not a valid key: $_key"
			;;
		esac

		case $_val in
		'"'*'"')
			_val=${_val#\"}
			_val=${_val%\"}
			;;
		'"'* | *'"')
			config_fail "unbalanced quote in value for $_key"
			;;
		esac

		case $_seen in
		*" $_key "*)	config_fail "duplicate key: $_key" ;;
		esac
		_seen="$_seen$_key "

		case $_key in
		CPUS)
			case $_val in
			'' | *[!0-9]* | 0*)
				config_fail "CPUS must be a positive integer: $_val"
				;;
			esac
			CPUS=$_val
			;;
		MEM)
			case $_val in
			'' | *[!0-9]* | 0*)
				config_fail "MEM must be a positive integer in MiB: $_val"
				;;
			esac
			MEM=$_val
			;;
		ARCH)
			case $_val in
			x86_64 | aarch64)	ARCH=$_val ;;
			*)			config_fail "ARCH must be x86_64 or aarch64: $_val" ;;
			esac
			;;
		HWACCEL)
			case $_val in
			yes | no)	HWACCEL=$_val ;;
			*)		config_fail "HWACCEL must be yes or no: $_val" ;;
			esac
			;;
		FIRMWARE)
			case $_val in
			/*)	FIRMWARE=$_val ;;
			*)	config_fail "FIRMWARE must be an absolute path: $_val" ;;
			esac
			;;
		CDROM)
			case $_val in
			/*)	CDROM=$_val ;;
			*)	config_fail "CDROM must be an absolute path: $_val" ;;
			esac
			;;
		IMAGE_FORMAT)
			# An allowlist because the value is interpolated into a
			# comma-separated option string: "qcow2,readonly=on"
			# would start a VM whose writes go nowhere.
			case $_val in
			qcow2 | raw)	IMAGE_FORMAT=$_val ;;
			*)		config_fail "IMAGE_FORMAT must be qcow2 or raw: $_val" ;;
			esac
			;;
		GRAPHICS)
			case $_val in
			no | vnc | spice)	GRAPHICS=$_val ;;
			*)			config_fail "GRAPHICS must be no, vnc or spice: $_val" ;;
			esac
			;;
		NETWORK)
			case $_val in
			yes | no | hostonly)	NETWORK=$_val ;;
			*)			config_fail "NETWORK must be yes, no or hostonly: $_val" ;;
			esac
			;;
		SNAPSHOT)
			case $_val in
			yes | no)	SNAPSHOT=$_val ;;
			*)		config_fail "SNAPSHOT must be yes or no: $_val" ;;
			esac
			;;
		BALLOON)
			case $_val in
			yes | no)	BALLOON=$_val ;;
			*)		config_fail "BALLOON must be yes or no: $_val" ;;
			esac
			;;
		BOOT_ORDER)
			# Every legal value, rather than a loop working out which
			# strings of c, d and n name a device no more than once.
			case $_val in
			c | d | n | cd | cn | dc | dn | nc | nd | \
			cdn | cnd | dcn | dnc | ncd | ndc)
				BOOT_ORDER=$_val
				;;
			*)
				config_fail "BOOT_ORDER must be 1-3 distinct of c, d, n: $_val"
				;;
			esac
			;;
		HOSTFWD)
			HOSTFWD=$_val
			hostfwd_check "$_val"
			;;
		*)
			config_fail "unknown key: $_key"
			;;
		esac
	done < "$CONFIG_PATH"

	# Without firmware the machine starts and executes nothing.
	if [ "$ARCH" = aarch64 ] && [ -z "$FIRMWARE" ]; then
		fail 1 "$CONFIG_PATH: ARCH=aarch64 requires FIRMWARE"
	fi
	# -nic none has nowhere to forward to, and the builder would drop
	# the request without a word.
	if [ "$NETWORK" = no ] && [ -n "$HOSTFWD" ]; then
		fail 1 "$CONFIG_PATH: NETWORK=no leaves HOSTFWD nothing to forward"
	fi
	# -boot strict=on restricts the guest to the devices carrying a
	# bootindex, and only a device this config creates gets one. A list
	# naming none of them restricts nothing: the guest boots the disk
	# that the list left out.
	_have=c
	[ -z "$CDROM" ] || _have=${_have}d
	[ "$NETWORK" = no ] || _have=${_have}n
	case $BOOT_ORDER in
	*[$_have]*)	;;
	*)		fail 1 "$CONFIG_PATH: BOOT_ORDER=$BOOT_ORDER has no device to boot" ;;
	esac
	arch_setup
}

# KVM needs the guest to be the machine it runs on, so HWACCEL=no and a
# foreign ARCH come to the same thing: emulation, where -cpu host does
# not exist.
arch_setup() {
	case $ARCH in
	x86_64)
		MACHINE=q35
		VGA=virtio-vga
		;;
	aarch64)
		MACHINE=virt,gic-version=max
		VGA=virtio-gpu-pci
		;;
	*)
		fail 1 "$CONFIG_PATH: unsupported host architecture: $ARCH"
		;;
	esac
	if [ "$HWACCEL" = yes ] && [ "$ARCH" = "$HOSTARCH" ]; then
		MACHINE=$MACHINE,accel=kvm
		CPU=host
	else
		MACHINE=$MACHINE,accel=tcg
		CPU=max
	fi
	QEMU=qemu-system-$ARCH
}

# Returns 1 and prints nothing when no reply came within MONITOR_TIMEOUT,
# so a silent monitor is not an empty answer. No liveness precheck: start
# has to talk to a QEMU that has no pidfile yet.
qmp() {
	_d=$1
	_body=$2
	QMPSEQ=$((QMPSEQ + 1))
	_id="vmm.$$.$QMPSEQ"

	# Without the FIFOs, tee would create a regular file where the pipe
	# belongs and wedge the monitor for the life of the guest.
	if [ ! -p "$_d/qmp.in" ] || [ ! -p "$_d/qmp.out" ]; then
		return 1
	fi

	# qmp_capabilities is valid once per QEMU process, not once per
	# writer; the duplicate error carries no id, so the filter skips it.
	#
	# Never write the FIFO with a shell redirect: the shell opens it
	# before exec'ing timeout, so the open(2) blocks forever. Only
	# "printf | timeout N tee fifo" is bounded. The subshell keeps the
	# shell's report of the killed job off the terminal.
	(
		printf '{"execute":"qmp_capabilities"}\n{%s,"id":"%s"}\n' \
			"$_body" "$_id" \
			| timeout "$MONITOR_TIMEOUT" tee "$_d/qmp.in" 9>&-
	) >/dev/null 2>&1 || :
	# The id picks this reply out of the greeting, of events, and of
	# replies orphaned by an earlier timeout, which would leave the FIFO
	# off by one forever. busybox timeout exits 143 where GNU exits 124.
	_reply=$( { timeout "$MONITOR_TIMEOUT" grep -m1 -F "\"$_id\"" \
		"$_d/qmp.out" 9>&-; } 2>/dev/null ) || :
	if [ -z "$_reply" ]; then
		return 1
	fi
	printf '%s\n' "$_reply"
}

# The command is interpolated into a JSON string, so its backslashes and
# quotes escape.
monitor_send() {
	case $2 in
	*[![:print:]]*)	fail 1 "monitor command has a non-printable byte" ;;
	esac
	_esc=$(printf '%s' "$2" | sed 's/\\/\\\\/g; s/"/\\"/g')
	qmp "$1" "\"execute\":\"human-monitor-command\",\"arguments\":{\"command-line\":\"$_esc\"}"
}

# QEMU splits option strings on commas, so an interpolated path must
# double its own. A raw comma is only a deprecation warning: the VM
# starts misconfigured.
qemu_comma() {
	printf '%s' "$1" | sed 's/,/,,/g'
}

# dryrun's output has to paste back into a shell unchanged. An embedded
# quote ends the string, escapes itself and starts a new one: '\''
shell_quote() {
	printf "'"
	printf '%s' "$1" | sed "s/'/'\\\\''/g"
	printf "'"
}

# One builder, so what dryrun prints and what start runs cannot drift
# apart.
vm_build() {
	_action=$1
	_name=$2
	_d=$VMMDIR/$_name
	_uuid=$(vm_uuid "$_d")
	if [ -z "$_uuid" ]; then
		fail 1 "$_d/uuid is missing or malformed"
	fi

	# bootindex supersedes -boot order= entirely, so emitting both would
	# guarantee a config that lies about itself.
	_bi_disk=
	_bi_cd=
	_bi_net=
	_r=$BOOT_ORDER
	_bn=0
	while [ -n "$_r" ]; do
		_c=${_r%"${_r#?}"}
		_r=${_r#?}
		_bn=$((_bn + 1))
		case $_c in
		c)	_bi_disk=$_bn ;;
		d)	_bi_cd=$_bn ;;
		n)	_bi_net=$_bn ;;
		esac
	done

	# A locally administered MAC derived from the uuid, so a VM keeps its
	# DHCP lease and a clone never collides with its source.
	_h=${_uuid%%-*}
	_m1=${_h%??????}
	_h=${_h#??}
	_m2=${_h%????}
	_h=${_h#??}
	_m3=${_h%??}

	_disk=$DISK
	_dfmt=$IMAGE_FORMAT
	if [ "$SNAPSHOT" = yes ]; then
		# QEMU's -snapshot is ignored for -blockdev nodes: the base
		# would be opened read-write with no overlay at all.
		_disk=$_d/ephemeral.qcow2
		_dfmt=qcow2
	fi

	set -- "$QEMU" \
		-nodefaults \
		-no-user-config \
		-name "guest=$_name,process=vmm/$_name" \
		-uuid "$_uuid" \
		-machine "$MACHINE" \
		-cpu "$CPU" \
		-smp "$CPUS" \
		-m "$MEM" \
		-rtc base=utc \
		-boot strict=on \
		-action reboot=reset \
		-action shutdown=poweroff \
		-action panic=pause \
		-device pvpanic-pci \
		-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
		-msg timestamp=on \
		-blockdev "node-name=disk0f,driver=file,filename=$(qemu_comma "$_disk"),discard=unmap" \
		-blockdev "node-name=disk0,driver=$_dfmt,file=disk0f,discard=unmap,detect-zeroes=unmap" \
		-device "virtio-blk-pci,id=blk0,drive=disk0,serial=vmm-$_name${_bi_disk:+,bootindex=$_bi_disk}" \
		-device virtio-rng-pci,id=rng0

	if [ "$BALLOON" = yes ]; then
		# free-page-reporting lets the guest hand freed pages back to the
		# host continuously, with no ballooning policy to manage.
		set -- "$@" -device virtio-balloon-pci,id=balloon0,free-page-reporting=on
	fi

	if [ -n "$FIRMWARE" ]; then
		set -- "$@" -bios "$FIRMWARE"
	fi

	# SCSI rather than IDE, which only x86 machines have.
	if [ -n "$CDROM" ]; then
		set -- "$@" \
			-device virtio-scsi-pci,id=scsi0 \
			-blockdev "node-name=cd0f,driver=file,filename=$(qemu_comma "$CDROM"),read-only=on" \
			-blockdev node-name=cd0,driver=raw,file=cd0f,read-only=on \
			-device "scsi-cd,id=cd0dev,drive=cd0,bus=scsi0.0${_bi_cd:+,bootindex=$_bi_cd}"
	fi

	if [ "$NETWORK" = no ]; then
		set -- "$@" -nic none
	else
		_nd="user,id=n0"
		if [ "$NETWORK" = hostonly ]; then
			_nd="$_nd,restrict=on"
		fi
		_nd="$_nd$(hostfwd_args "$HOSTFWD")"
		set -- "$@" \
			-netdev "$_nd" \
			-device "virtio-net-pci,id=nic0,netdev=n0,mac=52:54:00:$_m1:$_m2:$_m3${_bi_net:+,bootindex=$_bi_net}"
	fi

	# -vga none always: a non-VGA display device does not suppress the
	# default adapter, and two adapters is a confusing guest.
	set -- "$@" -vga none

	case $GRAPHICS in
	no)
		set -- "$@" -vnc none
		;;
	vnc)
		set -- "$@" \
			-device "$VGA" \
			-device qemu-xhci,id=xhci \
			-device usb-tablet,bus=xhci.0 \
			-vnc "unix:$(qemu_comma "$_d/vnc.sock")"
		;;
	spice)
		set -- "$@" \
			-device "$VGA" \
			-device qemu-xhci,id=xhci \
			-device usb-tablet,bus=xhci.0 \
			-display none \
			-spice "unix=on,addr=$(qemu_comma "$_d/spice.sock"),disable-ticketing=on"
		;;
	esac

	set -- "$@" \
		-chardev "pty,id=con0,logfile=$(qemu_comma "$_d/console.log"),logappend=on" \
		-serial chardev:con0 \
		-chardev "pipe,id=qmp0,path=$(qemu_comma "$_d/qmp")" \
		-mon chardev=qmp0,mode=control \
		-pidfile "$_d/pid"

	case $_action in
	print)
		# The overlay and the FIFOs first: QEMU will not make them for
		# itself. No side effects, printing never touches the filesystem.
		if [ "$SNAPSHOT" = yes ]; then
			printf 'qemu-img create -f qcow2 -b '
			shell_quote "$DISK"
			printf ' -F %s ' "$IMAGE_FORMAT"
			shell_quote "$_d/ephemeral.qcow2"
			printf '\n'
		fi
		printf 'mkfifo '
		shell_quote "$_d/qmp.in"
		printf ' '
		shell_quote "$_d/qmp.out"
		printf ' 2>/dev/null || :\n'
		shell_quote "$1"
		shift
		for _a in "$@"; do
			case $_a in
			-*)	printf ' \\\n%s' "$_a" ;;
			*)	printf ' '; shell_quote "$_a" ;;
			esac
		done
		printf '\n'
		;;
	spawn)
		# Never -daemonize: with the sandbox denying elevateprivileges
		# it fails with exit 1 and empty stderr. setsid detaches the
		# guest from this terminal.
		setsid "$@" 9>&- >"$_d/stdout" 2>"$_d/stderr" &
		VMPID=$!
		;;
	*)
		fail 1 "vm_build: no such action: $_action"
		;;
	esac
}

vm_wait_gone() {
	_left=$2
	vm_pid "$1"
	while [ -n "$PID" ]; do
		if [ "$_left" -le 0 ]; then
			return 1
		fi
		_left=$((_left - 1))
		sleep 1
		vm_pid "$1"
	done
}

# What survives SIGKILL is stuck in the kernel, not deciding, so five
# seconds is generous.
vm_force_kill() {
	vm_pid "$1"
	_current=$PID
	if [ -z "$_current" ]; then
		if vm_state_is_stale "$1"; then
			return 0
		fi
		return 1
	fi
	if [ "$_current" != "$2" ]; then
		return 1
	fi
	# ESRCH is a guest that died between vm_pid and the signal.
	kill -9 "$2" 2>/dev/null || :
	# Wait for staleness, not for vm_pid to go empty: a dying QEMU loses its
	# argv and its /proc/PID/exe link before it leaves /proc, and that window
	# is indistinguishable from state vm_state_is_stale must refuse to touch.
	_fk=5
	while ! vm_state_is_stale "$1"; do
		if [ "$_fk" -le 0 ]; then
			return 1
		fi
		_fk=$((_fk - 1))
		sleep 1
	done
}

# A directory whose pidfile still names a live QEMU is left exactly as
# it is.
vm_cleanup() {
	if ! vm_state_is_stale "$1"; then
		return 0
	fi
	rm -f "$1/qmp.in" "$1/qmp.out" "$1/vnc.sock" "$1/spice.sock" \
		"$1/ephemeral.qcow2" "$1/pid"
}

cmd_create() {
	case $# in
	1 | 2)	;;
	*)	usage ;;
	esac
	name=$1
	size=${2:-10G}
	vm_check_name "$name"
	case $size in
	*[KMGTkmgt])	_amount=${size%?} ;;
	*)		_amount=$size ;;
	esac
	case $_amount in
	'' | *[!0-9]* | 0*)	fail 1 "invalid disk size: $size" ;;
	esac
	command -v qemu-img >/dev/null 2>&1 || fail 1 "qemu-img is not on PATH"

	mkdir -p "$VMMDIR"
	VMMDIR=$(cd "$VMMDIR" && pwd -P) ||
		fail 1 "cannot enter VMMDIR: $VMMDIR"
	d=$VMMDIR/$name

	mkdir "$d" 2>/dev/null || fail 1 "cannot create VM directory: $d"
	cat /proc/sys/kernel/random/uuid > "$d/uuid"
	qemu-img create -f qcow2 "$d/disk.qcow2" "$size" >/dev/null

	cat > "$d/config" <<EOF
# vmm config for '$name'. The README documents the keys; vmm names the
# legal values of any it refuses.

CPUS=2
MEM=2048
IMAGE_FORMAT=qcow2
GRAPHICS=no
NETWORK=yes
SNAPSHOT=no
BOOT_ORDER=c
BALLOON=yes

# Optional:
# CDROM="/srv/iso/alpine-virt-x86_64.iso"
# HOSTFWD="2222:22,8080:80"
# ARCH=aarch64
# FIRMWARE="/usr/share/qemu/edk2-aarch64-code.fd"
# HWACCEL=no
EOF

	printf 'created %s (%s)\n' "$d" "$size"
	if [ -t 0 ]; then
		cmd_edit "$name"
	else
		printf 'next: vmm edit %s\n' "$name"
	fi
}

cmd_edit() {
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"
	${EDITOR:-vi} "$d/config"
	config_load "$d/config"
	printf 'config ok\n'
}

cmd_dryrun() {
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"
	config_load "$d/config"
	vm_build print "$1"
}

cmd_start() {
	[ $# -eq 1 ] || usage
	name=$1
	vm_check_name "$name"
	d=$VMMDIR/$name
	vm_exists "$d"

	lock_acquire "$d"
	vm_pid "$d"
	if [ -n "$PID" ]; then
		printf '%s is already running\n' "$name"
		return 0
	fi
	# vm_pid says no, but something alive still owns this state. Never
	# clear another process's files on a guess.
	if ! vm_state_is_stale "$d"; then
		fail 4 "$d has unrecognised live QEMU state; refusing to touch it"
	fi

	config_load "$d/config"
	if [ -z "$(vm_uuid "$d")" ]; then
		fail 1 "$d/uuid is missing or malformed"
	fi

	# A missing binary would fail inside setsid, and the monitor exchange
	# would wait out both timeouts before anything said so.
	if ! command -v "$QEMU" >/dev/null 2>&1; then
		fail 1 "$QEMU is not on PATH"
	fi

	# sun_path holds 108 bytes and /spice.sock is the longest name added.
	if [ "$GRAPHICS" != no ] && [ ${#d} -gt 96 ]; then
		fail 1 "$d is too long for a unix socket path; shorten VMMDIR or the VM name"
	fi

	# QEMU unlinks its own pidfile on a clean exit but not after SIGKILL.
	vm_cleanup "$d"

	if [ "$SNAPSHOT" = yes ]; then
		if ! qemu-img create -f qcow2 -b "$DISK" -F "$IMAGE_FORMAT" \
			"$d/ephemeral.qcow2" >/dev/null; then
			rm -f "$d/ephemeral.qcow2"
			fail 1 "$name failed to create its snapshot overlay"
		fi
	fi

	mkfifo "$d/qmp.in" "$d/qmp.out"
	: > "$d/console.log"

	vm_build spawn "$name"

	# Opening the QMP FIFO is the readiness barrier: it cannot complete
	# until QEMU is far enough along to open its end. No sleep required.
	if ! qmp "$d" '"execute":"query-status"' >/dev/null; then
		# A QEMU that has not written its pidfile yet is invisible to
		# vm_force_kill, which would leave it running. This child is
		# vmm's own and unreaped, so the pid is still its own to signal.
		kill -9 "$VMPID" 2>/dev/null || :
		if vm_force_kill "$d" "$VMPID"; then
			vm_cleanup "$d"
		else
			log "$name could not stop after its startup failure; state was left intact"
		fi
		log "$name failed to start, last lines of $d/stderr:"
		tail -n 20 "$d/stderr" >&2 || :
		exit 1
	fi

	# The monitor answered, so QEMU has opened every blockdev. Unlinked
	# now, the overlay is held by that descriptor alone: the guest runs on
	# it, and QEMU exiting frees the blocks.
	if [ "$SNAPSHOT" = yes ]; then
		rm -f "$d/ephemeral.qcow2"
	fi

	vm_pid "$d"
	printf '%s started (pid %s)\n' "$name" "$PID"
}

cmd_stop() {
	[ $# -eq 1 ] || usage
	name=$1
	vm_check_name "$name"
	d=$VMMDIR/$name
	vm_exists "$d"

	lock_acquire "$d"
	vm_pid "$d"
	pid=$PID
	if [ -z "$pid" ]; then
		if ! vm_state_is_stale "$d"; then
			fail 4 "$d has unrecognised live QEMU state; refusing to stop it"
		fi
		printf '%s is not running\n' "$name"
		vm_cleanup "$d"
		return 0
	fi

	if ! qmp "$d" '"execute":"system_powerdown"' >/dev/null; then
		log "$name did not acknowledge system_powerdown"
	fi

	if vm_wait_gone "$d" "$SHUTDOWN_TIMEOUT"; then
		vm_cleanup "$d"
		printf '%s stopped\n' "$name"
		return 0
	fi

	log "$name ignored ACPI shutdown for ${SHUTDOWN_TIMEOUT}s, killing"
	if ! vm_force_kill "$d" "$pid"; then
		fail 4 "$name did not die after SIGKILL; state was left intact"
	fi
	vm_cleanup "$d"
	fail 5 "$name was stopped by force"
}

cmd_kill() {
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"

	lock_acquire "$d"
	vm_pid "$d"
	pid=$PID
	if [ -z "$pid" ]; then
		if ! vm_state_is_stale "$d"; then
			fail 4 "$d has unrecognised live QEMU state; refusing to kill it"
		fi
		printf '%s is not running\n' "$1"
		vm_cleanup "$d"
		return 0
	fi
	if ! vm_force_kill "$d" "$pid"; then
		fail 4 "$1 did not die after SIGKILL; state was left intact"
	fi
	vm_cleanup "$d"
	printf '%s killed\n' "$1"
}

cmd_restart() {
	[ $# -eq 1 ] || usage
	# In a subshell so a forced stop, which exits 5, still starts. Any
	# other failure propagates.
	( cmd_stop "$1" ) && rc=0 || rc=$?
	if [ "$rc" != 0 ] && [ "$rc" != 5 ]; then
		exit "$rc"
	fi
	cmd_start "$1"
}

cmd_status() {
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"

	# The config is not read here: what the guest got is only knowable
	# from what is on the filesystem while it runs.
	vm_pid "$d"
	pid=$PID
	if [ -z "$pid" ]; then
		if ! vm_state_is_stale "$d"; then
			pid=$(cat "$d/pid" 2>/dev/null) || :
			printf 'STATE=unknown\nPID=%s\n' "${pid:-?}"
			exit 4
		fi
		printf 'STATE=stopped\nPID=-\n'
		exit 3
	fi
	printf 'STATE=running\nPID=%s\n' "$pid"
	if [ -S "$d/vnc.sock" ]; then
		printf 'VNC=%s\n' "$d/vnc.sock"
	fi
	if [ -S "$d/spice.sock" ]; then
		printf 'SPICE=%s\n' "$d/spice.sock"
	fi
}

cmd_list() {
	quiet=no
	if [ $# -gt 0 ] && [ "$1" = -q ]; then
		quiet=yes
		shift
	fi
	[ $# -eq 0 ] || usage
	if [ ! -d "$VMMDIR" ]; then
		return 0
	fi

	w=4
	for e in "$VMMDIR"/*; do
		if [ ! -f "$e/config" ]; then
			continue
		fi
		n=${e##*/}
		if [ ${#n} -gt "$w" ]; then
			w=${#n}
		fi
	done

	if [ "$quiet" = no ]; then
		printf "%-${w}s  %-7s  %s\n" NAME STATE PID
	fi
	for e in "$VMMDIR"/*; do
		if [ ! -f "$e/config" ]; then
			continue
		fi
		n=${e##*/}
		if [ "$quiet" = yes ]; then
			printf '%s\n' "$n"
			continue
		fi
		vm_pid "$e"
		p=$PID
		if [ -n "$p" ]; then
			printf "%-${w}s  %-7s  %s\n" "$n" running "$p"
		elif ! vm_state_is_stale "$e"; then
			p=$(cat "$e/pid" 2>/dev/null) || :
			printf "%-${w}s  %-7s  %s\n" "$n" unknown "${p:-?}"
		else
			printf "%-${w}s  %-7s  %s\n" "$n" stopped -
		fi
	done
}

console_restore() {
	kill "$reader" "$writer" 2>/dev/null || :
	stty "$old" 2>/dev/null || :
	wait "$reader" 2>/dev/null || :
	wait "$writer" 2>/dev/null || :
}

cmd_console() {
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"
	if [ ! -t 0 ]; then
		fail 1 "console needs a terminal on stdin; try: tail -f $d/console.log"
	fi

	lock_acquire "$d"
	vm_require_running "$d"
	reply=$(qmp "$d" '"execute":"query-chardev"') || reply=
	# The pty is allocated afresh at every start, so it is asked for, never
	# cached.
	pts=$(printf '%s' "$reply" | sed -n 's/.*"filename": *"pty:\([^"]*\)".*/\1/p')
	# Dropped before attaching: a console stays for as long as a human
	# wants it, and stop must not wait that out.
	exec 9>&-

	if [ -z "$pts" ] || [ ! -c "$pts" ]; then
		fail 1 "$1 has no serial pty; see $d/console.log"
	fi

	printf 'console %s (%s), Ctrl-] detaches\n' "$1" "$pts" >&2
	old=$(stty -g)
	reader=
	writer=
	trap 'console_restore' EXIT
	trap 'exit 130' INT
	trap 'exit 143' TERM
	trap 'exit 129' HUP
	# isig with intr ^] keeps Ctrl-C for the guest and makes Ctrl-] the
	# detach key. susp undef stops Ctrl-Z suspending vmm while raw.
	stty raw -echo isig intr '^]' susp undef
	# The reader signals this shell when the guest goes away, so a dead
	# guest detaches. Its cat is a background child under a trap because
	# console_restore kills the subshell alone, leaving a cat holding the
	# guest's pty. $! expands when the signal arrives, not before.
	{
		trap 'kill $! 2>/dev/null || :; exit 0' HUP INT TERM
		cat < "$pts" 2>/dev/null &
		wait
		kill -INT $$ 2>/dev/null || :
	} &
	reader=$!
	cat /dev/tty > "$pts" 2>/dev/null &
	writer=$!
	# wait, not a foreground cat: a signal interrupts wait at once, where
	# read(2) would defer the trap and leave the terminal raw.
	wait "$writer" 2>/dev/null || :
}

# The socket that is there is what the running guest got. No lock and no
# monitor round trip: unlike the serial pty, the path is fixed.
cmd_viewer() {
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"
	vm_require_running "$d"

	if [ -S "$d/vnc.sock" ]; then
		uri=vnc+unix://$d/vnc.sock
	elif [ -S "$d/spice.sock" ]; then
		uri=spice+unix://$d/spice.sock
	else
		fail 1 "$1 has no display; set GRAPHICS and restart it"
	fi

	# Unquoted so VIEWER can carry its own options, as EDITOR does. exec
	# replaces vmm, so no shell waits on the client.
	exec ${VIEWER:-remote-viewer} "$uri"
}

cmd_monitor() {
	[ $# -ge 1 ] || usage
	name=$1
	shift
	vm_check_name "$name"
	d=$VMMDIR/$name
	vm_exists "$d"

	# Held for the whole session: the monitor is one FIFO with one reply
	# stream, so a second reader would answer with the first one's replies.
	lock_acquire "$d"
	vm_require_running "$d"

	if [ $# -gt 0 ]; then
		if ! monitor_send "$d" "$*"; then
			fail 3 "no reply from the monitor of $name within ${MONITOR_TIMEOUT}s"
		fi
		return 0
	fi

	rc=0
	while :; do
		if [ -t 0 ]; then
			printf '(qemu) ' >&2
		fi
		IFS= read -r line || [ -n "$line" ] || break
		if [ -z "$line" ]; then
			continue
		fi
		if ! monitor_send "$d" "$line"; then
			log "no reply for: $line"
			rc=3
		fi
	done
	if [ "$rc" != 0 ]; then
		exit "$rc"
	fi
}

cmd_logs() {
	n=50
	if [ $# -gt 1 ] && [ "$1" = -n ]; then
		n=$2
		shift 2
		# Capped in length, so tail is never handed a count it refuses.
		case $n in
		'' | *[!0-9]* | 0* | ??????????*)
			fail 1 "invalid line count: $n"
			;;
		esac
	fi
	[ $# -eq 1 ] || usage
	vm_check_name "$1"
	d=$VMMDIR/$1
	vm_exists "$d"

	for f in "$d/stdout" "$d/stderr" "$d/console.log"; do
		printf '==> %s <==\n' "$f"
		if [ -f "$f" ]; then
			tail -n "$n" "$f"
		else
			printf '(none)\n'
		fi
	done
}

cmd_clone() {
	[ $# -eq 2 ] || usage
	src=$1
	dst=$2
	vm_check_name "$src"
	vm_check_name "$dst"
	sd=$VMMDIR/$src
	dd=$VMMDIR/$dst
	vm_exists "$sd"

	# The source is locked for the whole copy: without it, a concurrent
	# start could boot the guest halfway through reading its disk.
	lock_acquire "$sd"
	vm_pid "$sd"
	if [ -n "$PID" ]; then
		fail 4 "$src is running; stop it before cloning"
	fi
	if ! vm_state_is_stale "$sd"; then
		fail 4 "$sd has unrecognised live QEMU state; refusing to clone it"
	fi
	mkdir "$dd" 2>/dev/null || fail 1 "cannot create VM directory: $dd"
	cat /proc/sys/kernel/random/uuid > "$dd/uuid"
	cp "$sd/disk.qcow2" "$dd/disk.qcow2"
	cp "$sd/config" "$dd/config"
	printf 'cloned %s to %s\n' "$src" "$dst"
}

cmd_delete() {
	force=no
	if [ $# -gt 0 ] && [ "$1" = -f ]; then
		force=yes
		shift
	fi
	[ $# -eq 1 ] || usage
	name=$1
	vm_check_name "$name"
	d=$VMMDIR/$name
	vm_exists "$d"

	if [ "$force" = no ]; then
		if [ ! -t 0 ]; then
			fail 1 "refusing to delete $d without -f when stdin is not a tty"
		fi
		printf 'about to remove %s\n' "$d"
		du -sh "$d" 2>/dev/null || :
		printf 'type the VM name to confirm: '
		read -r answer || answer=
		if [ "$answer" != "$name" ]; then
			fail 1 "not confirmed"
		fi
	fi

	# Locked after the prompt, so a human typing cannot block start and
	# stop, and re-checked because the answer may have changed since.
	lock_acquire "$d"
	vm_pid "$d"
	if [ -n "$PID" ]; then
		fail 4 "$name is running; stop it before deleting"
	fi
	if ! vm_state_is_stale "$d"; then
		fail 4 "$d has unrecognised live QEMU state; refusing to delete"
	fi
	rm -rf "$d"
	printf 'deleted %s\n' "$d"
}

main() {
	[ $# -ge 1 ] || usage
	cmd=$1
	shift

	# Capped in length, so [ never has to compare a number too big for it.
	case $SHUTDOWN_TIMEOUT in
	'' | *[!0-9]* | 0* | ??????????*)
		fail 1 "SHUTDOWN_TIMEOUT must be a positive integer: $SHUTDOWN_TIMEOUT"
		;;
	esac
	case $MONITOR_TIMEOUT in
	'' | *[!0-9]* | 0* | ??????????*)
		fail 1 "MONITOR_TIMEOUT must be a positive integer: $MONITOR_TIMEOUT"
		;;
	esac
	vmmdir_setup

	case $cmd in
	create)		cmd_create "$@" ;;
	edit)		cmd_edit "$@" ;;
	start)		cmd_start "$@" ;;
	stop)		cmd_stop "$@" ;;
	restart)	cmd_restart "$@" ;;
	kill)		cmd_kill "$@" ;;
	status)		cmd_status "$@" ;;
	list)		cmd_list "$@" ;;
	console)	cmd_console "$@" ;;
	viewer)		cmd_viewer "$@" ;;
	monitor)	cmd_monitor "$@" ;;
	logs)		cmd_logs "$@" ;;
	clone)		cmd_clone "$@" ;;
	delete)		cmd_delete "$@" ;;
	dryrun)		cmd_dryrun "$@" ;;
	*)		usage ;;
	esac
}

main "$@"