diff --git a/.gitignore b/.gitignore index 3aadeef6..f87f117f 100644 --- a/.gitignore +++ b/.gitignore @@ -319,6 +319,9 @@ pyrightconfig.json # Custom release release.tar.gz + +# Offline install bundles +dweos-offline*.tar.gz **/device_settings.json **/server_preferences.json .env diff --git a/README.md b/README.md index 66b3e505..aab6a3f9 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,30 @@ To install for any *supported* Linux system, run the following command: `curl -s https://raw.githubusercontent.com/DeepwaterExploration/DWE_OS_2/main/install.sh | sudo bash -s` +### Installing or updating a device with no internet access + +Build a bundle on a machine that has internet, then push it to the device over +SSH. The device never reaches out to GitHub, PyPI or apt: + +```sh +./deploy-offline.sh pi@192.168.2.2 +``` + +If the device is not reachable from the machine with internet either, build the +bundle and carry it across yourself: + +```sh +# on the machine with internet, describing the device +./package-offline.sh --arch aarch64 --python 3.11 --glibc 2.36 + +# on the device +tar -xzf dweos-offline-*.tar.gz +sudo ./dweos-offline/install-offline.sh +``` + +Python packages are updated from the bundle's wheelhouse, so dependency changes +ship with the update. See [docs/offline-update.md](docs/offline-update.md). + ### Raspberry Pi Hardware PWM In order to enable hardware PWM on your Raspberry Pi, you need to edit `/boot/firmware/config.txt`. See [Raspberry Pi documentation](https://www.raspberrypi.com/documentation/computers/config_txt.html) for more information. diff --git a/create_release.sh b/create_release.sh index 170e0212..8a412870 100755 --- a/create_release.sh +++ b/create_release.sh @@ -39,7 +39,11 @@ echo "Successfully packaged frontend" cp run_release.py release cp install_requirements.sh release cp create_venv.sh release +cp install-local.sh release cp run_release.sh release cp -r service release +# Record the version so the installer can report what it just installed +python3 -c "import json; print(json.load(open('frontend/package.json'))['version'])" > release/VERSION + tar -czvf release.tar.gz release diff --git a/create_venv.sh b/create_venv.sh index 153fe779..30f31c65 100755 --- a/create_venv.sh +++ b/create_venv.sh @@ -1,11 +1,119 @@ #!/bin/bash -echo "Creating virtual python environment in .venv directory" +set -e -python3 -m venv .venv +VENV_DIR=".venv" +REQUIREMENTS="backend_py/requirements.txt" +PYTHON_BIN="python3" +WHEELHOUSE="" +RECREATE=0 +UPGRADE=1 + +usage() { + cat <<'USAGE' +Usage: create_venv.sh [options] + +Creates (or refreshes) the Python virtual environment used by DWE OS. + +Options: + --venv DIR Virtual environment directory (default: .venv) + --requirements FILE Requirements file (default: backend_py/requirements.txt) + --python BIN Python interpreter used to create the venv (default: python3) + --wheelhouse DIR Install from a local directory of wheels instead of PyPI. + Implies a fully offline pip run (--no-index). + --recreate Delete and rebuild the virtual environment from scratch + --no-upgrade Leave already-satisfied packages at their current version + -h, --help Show this help +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --venv) + VENV_DIR="$2" + shift 2 + ;; + --requirements) + REQUIREMENTS="$2" + shift 2 + ;; + --python) + PYTHON_BIN="$2" + shift 2 + ;; + --wheelhouse) + WHEELHOUSE="$2" + shift 2 + ;; + --recreate) + RECREATE=1 + shift + ;; + --no-upgrade) + UPGRADE=0 + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ ! -f "$REQUIREMENTS" ]; then + echo "Error: requirements file not found: $REQUIREMENTS" >&2 + exit 1 +fi + +PIP_ARGS=() +if [ -n "$WHEELHOUSE" ]; then + if [ ! -d "$WHEELHOUSE" ]; then + echo "Error: wheelhouse directory not found: $WHEELHOUSE" >&2 + exit 1 + fi + # Absolute path, since pip is invoked after the venv is activated + WHEELHOUSE=$(cd "$WHEELHOUSE" && pwd) + PIP_ARGS+=(--no-index --find-links "$WHEELHOUSE") + echo "Installing Python packages offline from $WHEELHOUSE" +fi + +if [ "$UPGRADE" -eq 1 ]; then + PIP_ARGS+=(--upgrade) +fi + +if [ "$RECREATE" -eq 1 ] && [ -d "$VENV_DIR" ]; then + echo "Removing existing virtual environment in $VENV_DIR" + rm -rf "$VENV_DIR" +fi + +if [ -x "$VENV_DIR/bin/python3" ]; then + echo "Reusing existing virtual environment in $VENV_DIR directory" +else + echo "Creating virtual python environment in $VENV_DIR directory" + # A leftover, half-built venv would make the next step fail in confusing ways + rm -rf "$VENV_DIR" + "$PYTHON_BIN" -m venv "$VENV_DIR" +fi + +if [ ! -x "$VENV_DIR/bin/pip" ]; then + echo "Error: $VENV_DIR has no pip. Install the python3-venv package and retry." >&2 + exit 1 +fi + +# Keep pip itself current where we can. A stale pip cannot read newer wheel +# metadata, but a failure here is never a reason to abort the install. +if [ -n "$WHEELHOUSE" ]; then + "$VENV_DIR/bin/pip" install --no-index --find-links "$WHEELHOUSE" --upgrade pip || + echo "Note: pip not upgraded (no newer pip wheel in the wheelhouse)" +fi echo "Installing requirements..." -. .venv/bin/activate && pip install -r backend_py/requirements.txt +"$VENV_DIR/bin/pip" install "${PIP_ARGS[@]}" -r "$REQUIREMENTS" -echo "Virtual environment created." +echo "Virtual environment ready." diff --git a/deploy-offline.sh b/deploy-offline.sh new file mode 100755 index 00000000..d6eea947 --- /dev/null +++ b/deploy-offline.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# +# Updates DWE OS on a remote device over SSH. Only this machine needs internet +# access: the device is given a bundle that already contains everything, so it +# never reaches out to GitHub, PyPI or apt. +# +# ./deploy-offline.sh pi@192.168.2.2 +# +# That probes the device, builds a bundle matching it, copies the bundle over +# and runs the installer there. + +set -e + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +BUNDLE="" +VERSION="latest" +RELEASE_TARBALL="" +BUILD_FROM_SOURCE=0 +TARGET_ARCH="" +TARGET_PYTHON="" +TARGET_GLIBC="" +DEBS_SOURCE="" +REMOTE_DIR="/tmp" +SSH_PORT="" +SSH_IDENTITY="" +SSH_EXTRA=() +INSTALL_ARGS=() +KEEP_REMOTE=0 +KEEP_BUNDLE=0 +PROBE=1 + +usage() { + cat <<'USAGE' +Usage: deploy-offline.sh [options] [user@]host + +Updates DWE OS on a device over SSH without the device needing internet. + +By default the device is probed for its architecture and Python version, a +matching bundle is built here, copied over, and installed. + +Bundle: + --bundle FILE Use an existing bundle from package-offline.sh instead + of building one (skips probing) + --version TAG Release to package (default: latest) + --release-tarball FILE Package a release.tar.gz that is already on disk + --build-from-source Build the release from this checkout + --debs DIR .deb packages to carry along to the device + --keep-bundle Keep the bundle that was built (default: discard it) + +Target overrides (skip probing the device): + --no-probe Do not ask the device what it is + --arch ARCH x86_64 | aarch64 | armv7l + --python VERSION Target Python minor version, e.g. 3.11 + --glibc VERSION Target glibc version, e.g. 2.36 + +SSH: + --port N SSH port + --identity FILE SSH private key + --ssh-option OPT Extra ssh -o option (repeatable) + --remote-dir DIR Where to stage the bundle on the device (default: /tmp) + --keep-remote Leave the bundle on the device after installing + +Install: + --install-arg ARG Extra argument for install-offline.sh (repeatable), + e.g. --install-arg --recreate-venv + -h, --help Show this help + +Examples: + ./deploy-offline.sh pi@192.168.2.2 + ./deploy-offline.sh --version v0.7.4 --identity ~/.ssh/id_dwe pi@192.168.2.2 + ./deploy-offline.sh --build-from-source --install-arg --recreate-venv pi@dwe.local +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --bundle) + BUNDLE="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --release-tarball) + RELEASE_TARBALL="$2" + shift 2 + ;; + --build-from-source) + BUILD_FROM_SOURCE=1 + shift + ;; + --debs) + DEBS_SOURCE="$2" + shift 2 + ;; + --keep-bundle) + KEEP_BUNDLE=1 + shift + ;; + --no-probe) + PROBE=0 + shift + ;; + --arch) + TARGET_ARCH="$2" + shift 2 + ;; + --python) + TARGET_PYTHON="$2" + shift 2 + ;; + --glibc) + TARGET_GLIBC="$2" + shift 2 + ;; + --port) + SSH_PORT="$2" + shift 2 + ;; + --identity) + SSH_IDENTITY="$2" + shift 2 + ;; + --ssh-option) + SSH_EXTRA+=(-o "$2") + shift 2 + ;; + --remote-dir) + REMOTE_DIR="$2" + shift 2 + ;; + --keep-remote) + KEEP_REMOTE=1 + shift + ;; + --install-arg) + INSTALL_ARGS+=("$2") + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + if [ -n "${REMOTE_TARGET:-}" ]; then + echo "Error: more than one host given ($REMOTE_TARGET, $1)" >&2 + exit 1 + fi + REMOTE_TARGET="$1" + shift + ;; + esac +done + +if [ -z "${REMOTE_TARGET:-}" ]; then + echo "Error: no host given" >&2 + usage >&2 + exit 1 +fi + +SSH_ARGS=() +SCP_ARGS=() +if [ -n "$SSH_PORT" ]; then + SSH_ARGS+=(-p "$SSH_PORT") + SCP_ARGS+=(-P "$SSH_PORT") +fi +if [ -n "$SSH_IDENTITY" ]; then + SSH_ARGS+=(-i "$SSH_IDENTITY") + SCP_ARGS+=(-i "$SSH_IDENTITY") +fi +SSH_ARGS+=("${SSH_EXTRA[@]}") +SCP_ARGS+=("${SSH_EXTRA[@]}") + +run_ssh() { + ssh "${SSH_ARGS[@]}" "$REMOTE_TARGET" "$@" +} + +# Single-quote a value for the remote shell +quote_remote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} + +# ------------------------------------------------------------------- probing + +REMOTE_IS_ROOT=0 + +# Emitted as key=value so a command that produces no output on the device +# cannot shift the rest of the answers +PROBE_CMD=$(cat <<'PROBE' +echo "arch=$(uname -m)" +echo "python=$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null)" +echo "glibc=$(getconf GNU_LIBC_VERSION 2>/dev/null | cut -d" " -f2)" +echo "uid=$(id -u)" +PROBE +) + +probe_value() { + printf '%s\n' "$PROBE_OUTPUT" | sed -n "s/^$1=//p" +} + +if [ "$PROBE" -eq 1 ] && [ -z "$BUNDLE" ]; then + echo "Probing $REMOTE_TARGET..." + PROBE_OUTPUT=$(run_ssh "$PROBE_CMD") + + PROBED_ARCH=$(probe_value arch) + PROBED_PYTHON=$(probe_value python) + PROBED_GLIBC=$(probe_value glibc) + PROBED_UID=$(probe_value uid) + + echo " architecture: $PROBED_ARCH" + echo " python: ${PROBED_PYTHON:-not found}" + echo " glibc: ${PROBED_GLIBC:-unknown}" + + if [ -z "$PROBED_PYTHON" ]; then + echo "Error: no python3 on $REMOTE_TARGET. DWE OS needs python3 and" >&2 + echo "python3-venv installed on the device before it can be updated offline." >&2 + exit 1 + fi + + [ -z "$TARGET_ARCH" ] && TARGET_ARCH="$PROBED_ARCH" + [ -z "$TARGET_PYTHON" ] && TARGET_PYTHON="$PROBED_PYTHON" + if [ -z "$TARGET_GLIBC" ] && echo "$PROBED_GLIBC" | grep -qE '^[0-9]+\.[0-9]+$'; then + TARGET_GLIBC="$PROBED_GLIBC" + fi + [ "$PROBED_UID" = "0" ] && REMOTE_IS_ROOT=1 +else + PROBE_OUTPUT=$(run_ssh "$PROBE_CMD") + [ "$(probe_value uid)" = "0" ] && REMOTE_IS_ROOT=1 +fi + +# ------------------------------------------------------------ build the bundle + +BUILT_BUNDLE="" + +if [ -z "$BUNDLE" ]; then + PACKAGE_ARGS=() + [ -n "$TARGET_ARCH" ] && PACKAGE_ARGS+=(--arch "$TARGET_ARCH") + [ -n "$TARGET_PYTHON" ] && PACKAGE_ARGS+=(--python "$TARGET_PYTHON") + [ -n "$TARGET_GLIBC" ] && PACKAGE_ARGS+=(--glibc "$TARGET_GLIBC") + [ -n "$DEBS_SOURCE" ] && PACKAGE_ARGS+=(--debs "$DEBS_SOURCE") + + if [ "$BUILD_FROM_SOURCE" -eq 1 ]; then + PACKAGE_ARGS+=(--build-from-source) + elif [ -n "$RELEASE_TARBALL" ]; then + PACKAGE_ARGS+=(--release-tarball "$RELEASE_TARBALL") + else + PACKAGE_ARGS+=(--version "$VERSION") + fi + + if [ "$KEEP_BUNDLE" -eq 1 ]; then + BUNDLE_OUT="$PWD/dweos-offline-${TARGET_ARCH}-py${TARGET_PYTHON}.tar.gz" + else + BUNDLE_STAGE=$(mktemp -d) + BUNDLE_OUT="$BUNDLE_STAGE/dweos-offline.tar.gz" + trap 'rm -rf "$BUNDLE_STAGE"' EXIT + fi + + echo + PACKAGE_HIDE_HINTS=1 bash "$SCRIPT_DIR/package-offline.sh" "${PACKAGE_ARGS[@]}" --output "$BUNDLE_OUT" + BUNDLE="$BUNDLE_OUT" + BUILT_BUNDLE="$BUNDLE_OUT" +fi + +if [ ! -f "$BUNDLE" ]; then + echo "Error: bundle not found: $BUNDLE" >&2 + exit 1 +fi + +# ------------------------------------------------------------------- transfer + +BUNDLE_NAME=$(basename "$BUNDLE") +REMOTE_BUNDLE="$REMOTE_DIR/$BUNDLE_NAME" +REMOTE_STAGE="$REMOTE_DIR/dweos-offline" + +echo +echo "Copying $BUNDLE_NAME to $REMOTE_TARGET:$REMOTE_DIR ..." +scp "${SCP_ARGS[@]}" "$BUNDLE" "$REMOTE_TARGET:$REMOTE_BUNDLE" + +if [ "$REMOTE_IS_ROOT" -eq 1 ]; then + SUDO_PREFIX="" +else + SUDO_PREFIX="sudo " +fi + +REMOTE_INSTALL_ARGS="" +for arg in "${INSTALL_ARGS[@]}"; do + REMOTE_INSTALL_ARGS="$REMOTE_INSTALL_ARGS $(quote_remote "$arg")" +done + +REMOTE_SCRIPT="set -e +rm -rf $(quote_remote "$REMOTE_STAGE") +tar -xzf $(quote_remote "$REMOTE_BUNDLE") -C $(quote_remote "$REMOTE_DIR") +${SUDO_PREFIX}bash $(quote_remote "$REMOTE_STAGE/install-offline.sh")$REMOTE_INSTALL_ARGS" + +if [ "$KEEP_REMOTE" -eq 0 ]; then + REMOTE_SCRIPT="$REMOTE_SCRIPT +rm -rf $(quote_remote "$REMOTE_STAGE") $(quote_remote "$REMOTE_BUNDLE")" +fi + +echo +echo "Installing on $REMOTE_TARGET ..." +echo + +# -t so sudo can prompt for a password on the device +ssh -t "${SSH_ARGS[@]}" "$REMOTE_TARGET" "$REMOTE_SCRIPT" + +echo +echo "Done. DWE OS updated on $REMOTE_TARGET." +if [ "$KEEP_BUNDLE" -eq 1 ] && [ -n "$BUILT_BUNDLE" ]; then + echo "Bundle kept at $BUILT_BUNDLE" +fi diff --git a/docs/offline-update.md b/docs/offline-update.md new file mode 100644 index 00000000..837f2aa8 --- /dev/null +++ b/docs/offline-update.md @@ -0,0 +1,140 @@ +# Updating a device without internet access + +Devices in the field usually cannot reach GitHub, PyPI or a Debian mirror. The +offline flow moves that requirement onto your laptop: you build a bundle on a +machine that *does* have internet, then hand the whole thing to the device over +SSH. The device never makes an outbound connection. + +A bundle contains everything an update needs: + +| Path | What it is | +| -------------------- | ----------------------------------------------------- | +| `release.tar.gz` | The DWE OS release being installed | +| `wheelhouse/` | Every Python wheel, built for the device's arch/Python | +| `bin/ttyd` | A matching `ttyd` binary | +| `debs/` | Optional `.deb` packages (empty by default) | +| `install-offline.sh` | The entry point you run on the device | +| `install-local.sh` | The installer itself | +| `MANIFEST` | What the bundle was built for | + +## One command + +From a checkout, on a machine with internet: + +```sh +./deploy-offline.sh pi@192.168.2.2 +``` + +That asks the device what it is, builds a bundle for it, copies it over, and +runs the installer. Useful options: + +```sh +# a specific release instead of the latest +./deploy-offline.sh --version v0.7.4 pi@192.168.2.2 + +# what is in this checkout, built from source +./deploy-offline.sh --build-from-source pi@192.168.2.2 + +# non-default SSH settings +./deploy-offline.sh --port 2222 --identity ~/.ssh/id_dwe pi@192.168.2.2 + +# force a clean virtual environment on the device +./deploy-offline.sh --install-arg --recreate-venv pi@192.168.2.2 +``` + +## Two steps + +When the device is not reachable from the machine with internet — an air-gapped +site, or a USB stick in between — build the bundle and carry it across +yourself. + +On the machine with internet: + +```sh +./package-offline.sh --arch aarch64 --python 3.11 --glibc 2.36 +``` + +`--arch`, `--python` and `--glibc` describe **the device**, not the machine +building the bundle. Read them off the device with: + +```sh +uname -m +python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])' +getconf GNU_LIBC_VERSION +``` + +Then, on the device: + +```sh +tar -xzf dweos-offline-v0.7.4-aarch64-py3.11.tar.gz +sudo ./dweos-offline/install-offline.sh +``` + +`install-offline.sh` checks the bundle against the device before touching +anything and stops if the architecture or Python version does not match, since +mismatched wheels install cleanly and then fail at import time. + +## Updating the Python packages + +`requirements.txt` is resolved at bundle time, so a bundle always carries the +exact set of wheels that release wants. On the device, pip installs from the +wheelhouse with `--no-index --upgrade`: pinned packages move to their pinned +version, unpinned ones move to whatever was resolved when the bundle was built, +and nothing reaches out to PyPI. The bundle also carries `pip`, `setuptools` +and `wheel`, so the device's own pip gets updated too. + +The virtual environment is kept between updates and upgraded in place. Only the +packages that actually changed are rewritten, which makes a repeat update take +a second or two. Pass `--recreate-venv` to rebuild it from scratch. + +## System packages + +`apt` is skipped entirely in offline mode. DWE OS's system dependencies +(GStreamer, `python3-venv`, exiftool) come from the initial install or the +Raspberry Pi image and rarely change, so an update does not need them. The +installer checks for them and warns if something is missing rather than +failing silently. + +If a system package genuinely has to change, carry the `.deb` files along. On a +machine running the same distribution and architecture as the device: + +```sh +mkdir debs +sudo apt-get install --reinstall --download-only -o Dir::Cache::archives="$PWD/debs" +``` + +Then build the bundle with `--debs debs`, and the installer will `dpkg -i` them +before the rest of the update. `dpkg` does not resolve dependencies, so the +directory has to hold the full dependency closure. + +## When a dependency has no wheel + +`package-offline.sh` selects wheels for the target platform; pip cannot do that +for source-only packages. When a dependency ships no wheel at all, the script +builds one and keeps it only if it came out architecture independent +(`*-none-any.whl`). `PyEventEmitter` is the current example. + +If a source-only dependency ever turns out to be a C extension, the build stops +with an error, because the wheel would be for the build machine rather than the +device. Build the bundle on a machine matching the device instead — either the +real hardware, or a container for that architecture: + +```sh +docker run --rm --platform linux/arm64 -v "$PWD:/src" -w /src \ + python:3.11-slim-bookworm \ + sh -c 'apt-get update && apt-get install -y curl build-essential && ./package-offline.sh' +``` + +## Script reference + +| Script | Where it runs | What it does | +| ------------------------ | ---------------------- | ---------------------------------------------- | +| `deploy-offline.sh` | machine with internet | probe, package, copy over SSH, install | +| `package-offline.sh` | machine with internet | build a bundle for a given target | +| `install-offline.sh` | device (in the bundle) | check the bundle fits, then install offline | +| `install-local.sh` | device | install from a release tarball already on disk | +| `install.sh` | device | download the latest release and install it | +| `install_requirements.sh`| device | system packages (`--offline` skips apt) | +| `create_venv.sh` | device | the venv (`--wheelhouse` installs offline) | + +Each takes `--help`. diff --git a/install-local.sh b/install-local.sh index caf4805b..1cd7c105 100755 --- a/install-local.sh +++ b/install-local.sh @@ -2,56 +2,197 @@ set -e +INSTALL_DIR=/opt/DWE_OS_2 +SERVICE_NAME=dwe_os_2 +SCRIPT_RUN_DIR=$PWD +TARBALL="release.tar.gz" + +OFFLINE=0 +WHEELHOUSE="" +TTYD_BINARY="" +DEBS_DIR="" +RECREATE_VENV=0 +SKIP_REQUIREMENTS=0 +USE_LOCAL_VENV=0 + +usage() { + cat <<'USAGE' +Usage: install-local.sh [options] [release.tar.gz] + +Installs or updates DWE OS from a release tarball that is already on this +machine. Nothing is downloaded unless the requirements step needs it. + +Options: + --offline Do not use the network at all. apt is skipped and pip + installs from --wheelhouse. Use this when updating a + device with no internet access. + --wheelhouse DIR Directory of wheels to install the Python packages from + --ttyd-binary FILE Prebuilt ttyd binary to install if ttyd is missing + --debs DIR Directory of .deb packages to install before anything else + --recreate-venv Rebuild .venv from scratch instead of updating in place + --skip-requirements Do not run install_requirements.sh or create_venv.sh + --install-dir DIR Where to install (default: /opt/DWE_OS_2) + --local Developer shortcut: copy ./.venv into the install + directory instead of building one + -h, --help Show this help + +The existing .venv is kept across updates unless --recreate-venv is passed, so +an offline update only has to move the wheels that actually changed. +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --offline) + OFFLINE=1 + shift + ;; + --wheelhouse) + WHEELHOUSE="$2" + shift 2 + ;; + --ttyd-binary) + TTYD_BINARY="$2" + shift 2 + ;; + --debs) + DEBS_DIR="$2" + shift 2 + ;; + --recreate-venv) + RECREATE_VENV=1 + shift + ;; + --skip-requirements) + SKIP_REQUIREMENTS=1 + shift + ;; + --install-dir) + INSTALL_DIR="$2" + shift 2 + ;; + --local) + USE_LOCAL_VENV=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + TARBALL="$1" + shift + ;; + esac +done + # Check if running as root if [ "$EUID" -ne 0 ]; then echo "This script must be run as root" exit 1 fi -INSTALL_DIR=/opt/DWE_OS_2 -SCRIPT_RUN_DIR=$PWD - -tar -xvzf release.tar.gz +if [ ! -f "$TARBALL" ]; then + echo "Error: release tarball not found: $TARBALL" >&2 + exit 1 +fi +TARBALL=$(cd "$(dirname "$TARBALL")" && pwd)/$(basename "$TARBALL") -if [ -d "$INSTALL_DIR" ]; then - echo "$INSTALL_DIR exists, updating files." - rm -rf $INSTALL_DIR +if [ -n "$WHEELHOUSE" ]; then + WHEELHOUSE=$(cd "$WHEELHOUSE" && pwd) fi -mkdir -p ${INSTALL_DIR} +# Unpack somewhere temporary so the caller keeps their tarball and a failed +# install does not leave a stray release/ directory behind +WORK_DIR=$(mktemp -d) +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT -cp -r release/* ${INSTALL_DIR} +echo "Extracting $TARBALL" +tar -xzf "$TARBALL" -C "$WORK_DIR" -cd ${INSTALL_DIR} +RELEASE_DIR="$WORK_DIR/release" +if [ ! -d "$RELEASE_DIR" ]; then + echo "Error: $TARBALL does not contain a release/ directory" >&2 + exit 1 +fi -# Comment the next two lines and uncomment the one after if you are developing locally and already have an install +VERSION="unknown version" +if [ -f "$RELEASE_DIR/VERSION" ]; then + VERSION=$(cat "$RELEASE_DIR/VERSION") +fi -if [ "$1" != "--local" ]; then - sh install_requirements.sh && - sh create_venv.sh -else - cp -r $SCRIPT_RUN_DIR/.venv $INSTALL_DIR +echo "Installing DWE OS 2 ($VERSION) to $INSTALL_DIR" + +# Stop the service before swapping files out from under it +if systemctl is-active --quiet "$SERVICE_NAME"; then + echo "Stopping currently running $SERVICE_NAME service..." + systemctl stop "$SERVICE_NAME" fi -# check if the service is already running, and stop it if necessary -if systemctl is-active --quiet dwe_os_2; then - echo "Stopping currently running dwe_os_2 service..." - systemctl stop dwe_os_2 +if [ -d "$INSTALL_DIR" ]; then + if [ "$RECREATE_VENV" -eq 1 ] || [ "$USE_LOCAL_VENV" -eq 1 ]; then + echo "$INSTALL_DIR exists, replacing it." + rm -rf "$INSTALL_DIR" + else + # Keep .venv: rebuilding it needs either the network or a wheelhouse, + # and its contents are tied to this absolute path + echo "$INSTALL_DIR exists, updating files (keeping .venv)." + find "$INSTALL_DIR" -mindepth 1 -maxdepth 1 ! -name '.venv' -exec rm -rf {} + + fi fi -# copy and enable service -cp ${INSTALL_DIR}/service/* /etc/systemd/system/ -systemctl enable dwe_os_2 -systemctl start dwe_os_2 +mkdir -p "$INSTALL_DIR" -# cleanup -cd $SCRIPT_RUN_DIR +cp -r "$RELEASE_DIR"/* "$INSTALL_DIR" -rm -rf release -rm release.tar.gz +cd "$INSTALL_DIR" -if [ "$VERSION" == "latest" ]; then - echo "Successfully installed DWE_OS 2 (latest version)" +if [ "$USE_LOCAL_VENV" -eq 1 ]; then + echo "Copying $SCRIPT_RUN_DIR/.venv into $INSTALL_DIR" + cp -r "$SCRIPT_RUN_DIR/.venv" "$INSTALL_DIR" +elif [ "$SKIP_REQUIREMENTS" -eq 1 ]; then + echo "Skipping system requirements and virtual environment setup." else - echo "Successfully installed DWE_OS 2 ($VERSION)" + REQUIREMENTS_ARGS=() + VENV_ARGS=() + + if [ "$OFFLINE" -eq 1 ]; then + REQUIREMENTS_ARGS+=(--offline) + fi + if [ -n "$TTYD_BINARY" ]; then + REQUIREMENTS_ARGS+=(--ttyd-binary "$TTYD_BINARY") + fi + if [ -n "$DEBS_DIR" ]; then + REQUIREMENTS_ARGS+=(--debs "$DEBS_DIR") + fi + if [ -n "$WHEELHOUSE" ]; then + VENV_ARGS+=(--wheelhouse "$WHEELHOUSE") + elif [ "$OFFLINE" -eq 1 ]; then + echo "Error: --offline needs --wheelhouse DIR to install Python packages" >&2 + exit 1 + fi + if [ "$RECREATE_VENV" -eq 1 ]; then + VENV_ARGS+=(--recreate) + fi + + bash install_requirements.sh "${REQUIREMENTS_ARGS[@]}" + bash create_venv.sh "${VENV_ARGS[@]}" fi + +# copy and enable service +cp "$INSTALL_DIR"/service/* /etc/systemd/system/ +systemctl daemon-reload +systemctl enable "$SERVICE_NAME" +systemctl restart "$SERVICE_NAME" + +cd "$SCRIPT_RUN_DIR" + +echo "Successfully installed DWE OS 2 ($VERSION)" diff --git a/install.sh b/install.sh index 50cde5e2..395abe7b 100755 --- a/install.sh +++ b/install.sh @@ -2,15 +2,16 @@ set -e +GITHUB_REPO="${GITHUB_REPO:-DeepwaterExploration/DWE_OS_2}" +SOURCE_REPO="${SOURCE_REPO:-DeepWaterExploration/dweOS}" +INSTALLER_URL="${INSTALLER_URL:-https://raw.githubusercontent.com/${SOURCE_REPO}/main/install-local.sh}" + # Check if running as root if [ "$EUID" -ne 0 ]; then echo "This script must be run as root" exit 1 fi -INSTALL_DIR=/opt/DWE_OS_2 -SCRIPT_RUN_DIR=$PWD - # Check if a tag name was provided as an argument if [ -z "$1" ]; then VERSION="latest" @@ -20,49 +21,30 @@ fi if [ "$VERSION" == "latest" ]; then echo "Installing DWE_OS 2 (latest release)" - DOWNLOAD_URL="https://github.com/DeepwaterExploration/DWE_OS_2/releases/latest/download/release.tar.gz" + DOWNLOAD_URL="https://github.com/${GITHUB_REPO}/releases/latest/download/release.tar.gz" else echo "Installing DWE_OS 2 ($VERSION)" - DOWNLOAD_URL="https://github.com/DeepwaterExploration/DWE_OS_2/releases/download/$VERSION/release.tar.gz" -fi - -wget $DOWNLOAD_URL -O release.tar.gz - -tar -xvzf release.tar.gz - -if [ -d "$INSTALL_DIR" ]; then - echo "$INSTALL_DIR does exist, deleting." - rm -rf $INSTALL_DIR + DOWNLOAD_URL="https://github.com/${GITHUB_REPO}/releases/download/$VERSION/release.tar.gz" fi -mkdir -p ${INSTALL_DIR} - -cp -r release/* ${INSTALL_DIR} - -cd ${INSTALL_DIR} +WORK_DIR=$(mktemp -d) +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT -sh install_requirements.sh && -sh create_venv.sh && +wget "$DOWNLOAD_URL" -O "$WORK_DIR/release.tar.gz" -# check if the service is already running, and stop it if necessary -if systemctl is-active --quiet dwe_os_2; then - echo "Stopping currently running dwe_os_2 service..." - systemctl stop dwe_os_2 -fi - -# copy and enable service -cp ${INSTALL_DIR}/service/* /etc/systemd/system/ -systemctl enable dwe_os_2 -systemctl start dwe_os_2 - -# cleanup -cd $SCRIPT_RUN_DIR - -rm -rf release -rm release.tar.gz - -if [ "$VERSION" == "latest" ]; then - echo "Successfully installed DWE_OS 2 (latest version)" +# Releases carry their own installer, so online and offline installs run +# exactly the same code. Releases cut before that was true fall back to +# fetching the installer from the branch this script came from. +INSTALLER="$WORK_DIR/release/install-local.sh" +if tar -tzf "$WORK_DIR/release.tar.gz" | grep -qx "release/install-local.sh"; then + tar -xzf "$WORK_DIR/release.tar.gz" -C "$WORK_DIR" release/install-local.sh else - echo "Successfully installed DWE_OS 2 ($VERSION)" + echo "This release predates the bundled installer, fetching it from $SOURCE_REPO" + mkdir -p "$WORK_DIR/release" + wget "$INSTALLER_URL" -O "$INSTALLER" fi + +bash "$INSTALLER" "$WORK_DIR/release.tar.gz" diff --git a/install_requirements.sh b/install_requirements.sh index 39fd3d12..6c4280ea 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -1,5 +1,68 @@ #!/bin/bash +TTYD_VERSION="1.6.3" + +OFFLINE=0 +TTYD_BINARY="" +DEBS_DIR="" + +usage() { + cat <<'USAGE' +Usage: install_requirements.sh [options] + +Installs the system packages DWE OS needs (Python, GStreamer, ttyd). + +Options: + --offline Do not touch the network. apt is skipped entirely; ttyd + and any .deb packages are taken from the local files + given below. Used when updating a device that has no + internet access. + --ttyd-binary FILE Install this prebuilt ttyd binary instead of downloading + one. Ignored if ttyd is already installed. + --debs DIR Install every .deb in DIR with dpkg before anything else. + Optional; used to carry system packages onto an offline + device. + -h, --help Show this help +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --offline) + OFFLINE=1 + shift + ;; + --ttyd-binary) + TTYD_BINARY="$2" + shift 2 + ;; + --debs) + DEBS_DIR="$2" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +# Run privileged commands directly when already root; offline devices are not +# guaranteed to have sudo installed. +if [ "$(id -u)" -eq 0 ]; then + SUDO="" +elif command -v sudo >/dev/null 2>&1; then + SUDO="sudo" +else + echo "Error: not running as root and sudo is not available" >&2 + exit 1 +fi + # detect system architecture get_arch() { local arch=$(uname -m) @@ -19,6 +82,19 @@ get_arch() { esac } +# install a ttyd binary that is already on disk +install_ttyd_binary() { + local source_file="$1" + + if ! $SUDO install -m 755 "$source_file" /usr/local/bin/ttyd; then + echo "Error: Failed to install ttyd from $source_file" + return 1 + fi + + echo "Successfully installed ttyd from $source_file" + return 0 +} + # download and install ttyd install_ttyd() { local arch=$(get_arch) @@ -27,7 +103,7 @@ install_ttyd() { exit 1 fi - local version="1.6.3" + local version="$TTYD_VERSION" local filename="ttyd.x86_64" # set filename based on architecture @@ -60,7 +136,7 @@ install_ttyd() { chmod +x "$filename" # move to /usr/local/bin so it can be executed by the program - if ! sudo mv "$filename" /usr/local/bin/ttyd; then + if ! $SUDO mv "$filename" /usr/local/bin/ttyd; then echo "Error: Failed to install ttyd" rm -rf "$temp_dir" exit 1 @@ -82,25 +158,76 @@ install_ttyd() { fi } -# update dependencies -sudo apt-get update -y +# report on system packages we cannot install without the network +check_offline_dependencies() { + local missing="" -# Install python and gstreamer dependencies -echo "Installing Python dependencies..." -sudo apt-get install python3 python3-venv -y + command -v python3 >/dev/null 2>&1 || missing="$missing python3" + python3 -c "import venv" >/dev/null 2>&1 || missing="$missing python3-venv" + command -v gst-inspect-1.0 >/dev/null 2>&1 || missing="$missing gstreamer1.0-tools" + command -v exiftool >/dev/null 2>&1 || missing="$missing libimage-exiftool-perl" -echo "Installing GStreamer dependencies..." -sudo apt-get install -y libglib2.0-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-plugins-ugly libimage-exiftool-perl + if [ -n "$missing" ]; then + echo "Warning: these system packages look missing and cannot be installed offline:" + echo " $missing" + echo " Install them on the device (or pass --debs DIR) before DWE OS will run." + return 1 + fi + echo "System dependencies present." + return 0 +} + +# install .deb files carried along with an offline bundle +install_local_debs() { + local debs_dir="$1" + + # A bundle always has the directory; it is usually empty + if ! ls "$debs_dir"/*.deb >/dev/null 2>&1; then + return 0 + fi + + echo "Installing local .deb packages from $debs_dir..." + if ! $SUDO dpkg -i "$debs_dir"/*.deb; then + echo "Warning: some .deb packages failed to install (unmet dependencies?)." + echo " Run 'dpkg -l | grep ^i[^i]' on the device to inspect." + fi +} + +if [ -n "$DEBS_DIR" ] && [ -d "$DEBS_DIR" ]; then + install_local_debs "$DEBS_DIR" +fi + +if [ "$OFFLINE" -eq 1 ]; then + echo "Offline mode: skipping apt-get." + check_offline_dependencies || true +else + # update dependencies + $SUDO apt-get update -y + + # Install python and gstreamer dependencies + echo "Installing Python dependencies..." + $SUDO apt-get install python3 python3-venv -y + + echo "Installing GStreamer dependencies..." + $SUDO apt-get install -y libglib2.0-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-libav gstreamer1.0-plugins-ugly libimage-exiftool-perl +fi -# Attempt to install ttyd through apt. If it fails, download from GitHub echo "Installing ttyd..." -if ! sudo apt-get install -y ttyd; then +if command -v ttyd >/dev/null 2>&1 && [ "$OFFLINE" -eq 1 ] && [ -z "$TTYD_BINARY" ]; then + echo "ttyd already installed at $(command -v ttyd)" +elif [ -n "$TTYD_BINARY" ] && [ -f "$TTYD_BINARY" ]; then + install_ttyd_binary "$TTYD_BINARY" || exit 1 +elif [ "$OFFLINE" -eq 1 ]; then + echo "Warning: ttyd is not installed and no local binary was provided." + echo " The web terminal will be unavailable until ttyd is installed." +# Attempt to install ttyd through apt. If it fails, download from GitHub +elif ! $SUDO apt-get install -y ttyd; then echo "ttyd not available in repositories, downloading from GitHub..." install_ttyd else # Disable ttyd service - sudo systemctl disable ttyd + $SUDO systemctl disable ttyd echo "ttyd installed from repositories" fi diff --git a/package-offline.sh b/package-offline.sh new file mode 100755 index 00000000..7cc66716 --- /dev/null +++ b/package-offline.sh @@ -0,0 +1,502 @@ +#!/bin/bash +# +# Builds a self-contained bundle that installs or updates DWE OS on a device +# with no internet access. Run this on a machine that *does* have internet, +# then ship the bundle to the device (deploy-offline.sh does that over SSH). +# +# The bundle carries the release tarball, every Python wheel needed for the +# target's architecture and Python version, a matching ttyd binary, and the +# installer itself. + +set -e + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +GITHUB_REPO="${GITHUB_REPO:-DeepwaterExploration/DWE_OS_2}" +TTYD_VERSION="1.6.3" + +# Oldest glibc any manylinux wheel we pick may require. Bumping the ceiling to +# whatever the device actually has (--glibc, or the probe in deploy-offline.sh) +# widens the set of wheels we can use. +MIN_GLIBC_MINOR=17 +DEFAULT_GLIBC_MINOR=28 + +VERSION="latest" +RELEASE_TARBALL="" +BUILD_FROM_SOURCE=0 +TARGET_ARCH="" +TARGET_PYTHON="" +TARGET_GLIBC="" +OUTPUT="" +SKIP_TTYD=0 +DEBS_SOURCE="" + +usage() { + cat <<'USAGE' +Usage: package-offline.sh [options] + +Builds an offline install/update bundle for a device with no internet access. +Run this on a machine that has internet. + +Release source (pick one, default --version latest): + --version TAG Download this release from GitHub (e.g. v0.7.4, latest) + --release-tarball FILE Use a release.tar.gz that is already on disk + --build-from-source Run ./create_release.sh to build the release here + +Target description (what the device is, not what this machine is): + --arch ARCH x86_64 | aarch64 | armv7l (default: this machine's) + --python VERSION Target Python minor version, e.g. 3.11 (default: this + machine's). Wheels are selected for this version, so + it has to match the device. + --glibc VERSION Target glibc version, e.g. 2.36 (default: 2.28). + Only wheels needing this or older are selected. + +Other: + --output FILE Bundle path (default: dweos-offline--.tar.gz) + --debs DIR Copy these .deb packages into the bundle. They get + installed with dpkg before the rest of the update. + --skip-ttyd Do not put a ttyd binary in the bundle + -h, --help Show this help + +Example, for a Raspberry Pi running Python 3.11 on Debian bookworm: + ./package-offline.sh --arch aarch64 --python 3.11 --glibc 2.36 +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --version) + VERSION="$2" + shift 2 + ;; + --release-tarball) + RELEASE_TARBALL="$2" + shift 2 + ;; + --build-from-source) + BUILD_FROM_SOURCE=1 + shift + ;; + --arch) + TARGET_ARCH="$2" + shift 2 + ;; + --python) + TARGET_PYTHON="$2" + shift 2 + ;; + --glibc) + TARGET_GLIBC="$2" + shift 2 + ;; + --output) + OUTPUT="$2" + shift 2 + ;; + --debs) + DEBS_SOURCE="$2" + shift 2 + ;; + --skip-ttyd) + SKIP_TTYD=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +# ---------------------------------------------------------------- target info + +if [ -z "$TARGET_ARCH" ]; then + TARGET_ARCH=$(uname -m) + echo "No --arch given, assuming this machine's: $TARGET_ARCH" +fi + +case "$TARGET_ARCH" in + x86_64 | amd64) + TARGET_ARCH="x86_64" + TTYD_SUFFIX="x86_64" + ;; + aarch64 | arm64) + TARGET_ARCH="aarch64" + TTYD_SUFFIX="aarch64" + ;; + armv7l | armhf | arm) + TARGET_ARCH="armv7l" + TTYD_SUFFIX="arm" + ;; + *) + echo "Error: unsupported target architecture: $TARGET_ARCH" >&2 + exit 1 + ;; +esac + +if [ -z "$TARGET_PYTHON" ]; then + TARGET_PYTHON=$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])') + echo "No --python given, assuming this machine's: $TARGET_PYTHON" +fi + +if ! echo "$TARGET_PYTHON" | grep -qE '^3\.[0-9]+$'; then + echo "Error: --python expects a major.minor version like 3.11, got: $TARGET_PYTHON" >&2 + exit 1 +fi +PYTHON_TAG="cp$(echo "$TARGET_PYTHON" | tr -d '.')" + +if [ -z "$TARGET_GLIBC" ]; then + GLIBC_MINOR="$DEFAULT_GLIBC_MINOR" + echo "No --glibc given, limiting wheels to glibc 2.${GLIBC_MINOR} or older" +else + GLIBC_MINOR=$(echo "$TARGET_GLIBC" | cut -d. -f2) + if ! echo "$GLIBC_MINOR" | grep -qE '^[0-9]+$'; then + echo "Error: --glibc expects a version like 2.36, got: $TARGET_GLIBC" >&2 + exit 1 + fi + if [ "$GLIBC_MINOR" -lt "$MIN_GLIBC_MINOR" ]; then + GLIBC_MINOR="$MIN_GLIBC_MINOR" + fi +fi + +echo +echo "Target: $TARGET_ARCH, Python $TARGET_PYTHON, glibc <= 2.${GLIBC_MINOR}" +echo + +WORK_DIR=$(mktemp -d) +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +BUNDLE_DIR="$WORK_DIR/dweos-offline" +mkdir -p "$BUNDLE_DIR/wheelhouse" "$BUNDLE_DIR/bin" "$BUNDLE_DIR/debs" + +# ------------------------------------------------------------- release source + +SOURCE_DESCRIPTION="" + +if [ -n "$RELEASE_TARBALL" ] && [ "$BUILD_FROM_SOURCE" -eq 1 ]; then + echo "Error: --release-tarball and --build-from-source are mutually exclusive" >&2 + exit 1 +fi + +if [ "$BUILD_FROM_SOURCE" -eq 1 ]; then + echo "Building release from source..." + (cd "$SCRIPT_DIR" && ./create_release.sh) + RELEASE_TARBALL="$SCRIPT_DIR/release.tar.gz" + SOURCE_DESCRIPTION="built from source" +elif [ -n "$RELEASE_TARBALL" ]; then + SOURCE_DESCRIPTION="local tarball" +else + if [ "$VERSION" = "latest" ]; then + DOWNLOAD_URL="https://github.com/${GITHUB_REPO}/releases/latest/download/release.tar.gz" + else + DOWNLOAD_URL="https://github.com/${GITHUB_REPO}/releases/download/${VERSION}/release.tar.gz" + fi + echo "Downloading release ($VERSION) from $GITHUB_REPO..." + curl -fL --progress-bar -o "$WORK_DIR/release.tar.gz" "$DOWNLOAD_URL" + RELEASE_TARBALL="$WORK_DIR/release.tar.gz" + SOURCE_DESCRIPTION="github:${GITHUB_REPO}@${VERSION}" +fi + +if [ ! -f "$RELEASE_TARBALL" ]; then + echo "Error: release tarball not found: $RELEASE_TARBALL" >&2 + exit 1 +fi + +cp "$RELEASE_TARBALL" "$BUNDLE_DIR/release.tar.gz" + +# The requirements file and the installer both come out of the release, so the +# bundle always matches the version it installs +tar -xzf "$BUNDLE_DIR/release.tar.gz" -C "$WORK_DIR" release/backend_py/requirements.txt +REQUIREMENTS="$WORK_DIR/release/backend_py/requirements.txt" + +if tar -tzf "$BUNDLE_DIR/release.tar.gz" | grep -qx "release/install-local.sh"; then + tar -xzf "$BUNDLE_DIR/release.tar.gz" -C "$WORK_DIR" release/install-local.sh + cp "$WORK_DIR/release/install-local.sh" "$BUNDLE_DIR/install-local.sh" +else + # Older release: use the installer sitting next to this script + echo "Release has no bundled installer, using $SCRIPT_DIR/install-local.sh" + cp "$SCRIPT_DIR/install-local.sh" "$BUNDLE_DIR/install-local.sh" +fi +chmod +x "$BUNDLE_DIR/install-local.sh" + +RELEASE_VERSION="unknown" +if tar -tzf "$BUNDLE_DIR/release.tar.gz" | grep -qx "release/VERSION"; then + tar -xzf "$BUNDLE_DIR/release.tar.gz" -C "$WORK_DIR" release/VERSION + RELEASE_VERSION=$(cat "$WORK_DIR/release/VERSION") +elif [ "$VERSION" != "latest" ]; then + RELEASE_VERSION="$VERSION" +fi + +echo "Release version: $RELEASE_VERSION" + +# ------------------------------------------------------------------ wheelhouse + +# pip runs inside a throwaway venv: a distribution-patched system pip can fail +# to build sdists that build fine in a clean environment. +echo +echo "Preparing a build environment for pip..." +python3 -m venv "$WORK_DIR/buildenv" >/dev/null +BUILD_PIP="$WORK_DIR/buildenv/bin/pip" + +# Downloads here are large and often go over a slow or proxied link +PIP_NET_ARGS=(--retries 5 --timeout 60) + +"$BUILD_PIP" install --quiet "${PIP_NET_ARGS[@]}" --upgrade pip setuptools wheel + +WHEELHOUSE="$BUNDLE_DIR/wheelhouse" +PIP_LOG="$WORK_DIR/pip-download.log" + +# Platform tags the device can install, newest manylinux first +PLATFORM_ARGS=() +for minor in $(seq "$GLIBC_MINOR" -1 "$MIN_GLIBC_MINOR"); do + PLATFORM_ARGS+=(--platform "manylinux_2_${minor}_${TARGET_ARCH}") +done +PLATFORM_ARGS+=(--platform "manylinux2014_${TARGET_ARCH}") +if [ "$TARGET_ARCH" = "x86_64" ]; then + PLATFORM_ARGS+=(--platform "manylinux2010_${TARGET_ARCH}") + PLATFORM_ARGS+=(--platform "manylinux1_${TARGET_ARCH}") +fi +PLATFORM_ARGS+=(--platform "linux_${TARGET_ARCH}") +PLATFORM_ARGS+=(--platform any) + +download_wheels() { + "$BUILD_PIP" download \ + -r "$REQUIREMENTS" \ + -d "$WHEELHOUSE" \ + --find-links "$WHEELHOUSE" \ + "${PIP_NET_ARGS[@]}" \ + --only-binary=:all: \ + "${PLATFORM_ARGS[@]}" \ + --python-version "$TARGET_PYTHON" \ + --implementation cp \ + --abi "$PYTHON_TAG" --abi abi3 --abi none \ + >"$PIP_LOG" 2>&1 +} + +# pip can only cross-select wheels, never sdists. When a dependency ships no +# wheel at all, build one here and drop it in the wheelhouse -- but only keep +# it if it came out architecture independent, since anything else would be +# built for this machine rather than the device. +build_portable_wheel() { + local spec="$1" + local sdist_dir="$WORK_DIR/sdist" + local built_dir="$WORK_DIR/built" + + echo " $spec ships no wheel, building one from its source distribution..." + rm -rf "$sdist_dir" "$built_dir" + mkdir -p "$sdist_dir" "$built_dir" + + if ! "$BUILD_PIP" download "${PIP_NET_ARGS[@]}" --no-deps --no-binary :all: "$spec" -d "$sdist_dir" >>"$PIP_LOG" 2>&1; then + echo "Error: could not download a source distribution for $spec" >&2 + tail -n 20 "$PIP_LOG" >&2 + exit 1 + fi + + if ! "$BUILD_PIP" wheel --no-deps "$sdist_dir"/* -w "$built_dir" >>"$PIP_LOG" 2>&1; then + echo "Error: could not build a wheel for $spec" >&2 + tail -n 20 "$PIP_LOG" >&2 + exit 1 + fi + + local wheel + for wheel in "$built_dir"/*.whl; do + case "$(basename "$wheel")" in + *-none-any.whl) + mv "$wheel" "$WHEELHOUSE/" + echo " built $(basename "$wheel")" + ;; + *) + echo "Error: $spec built into $(basename "$wheel"), which is specific to" >&2 + echo " this machine rather than $TARGET_ARCH/Python $TARGET_PYTHON." >&2 + echo " Run package-offline.sh on a machine matching the device (or in a" >&2 + echo " container for that architecture) to package this dependency." >&2 + exit 1 + ;; + esac + done +} + +echo "Collecting Python wheels for $TARGET_ARCH / Python $TARGET_PYTHON..." + +RESOLVED="" +attempt=0 +while ! download_wheels; do + attempt=$((attempt + 1)) + if [ "$attempt" -gt 25 ]; then + echo "Error: gave up resolving Python dependencies after $attempt attempts" >&2 + tail -n 30 "$PIP_LOG" >&2 + exit 1 + fi + + MISSING=$(sed -n 's/^ERROR: No matching distribution found for //p' "$PIP_LOG" | head -n 1) + if [ -z "$MISSING" ]; then + echo "Error: could not collect Python wheels" >&2 + tail -n 30 "$PIP_LOG" >&2 + exit 1 + fi + + case " $RESOLVED " in + *" $MISSING "*) + echo "Error: $MISSING still cannot be resolved after building it" >&2 + tail -n 30 "$PIP_LOG" >&2 + exit 1 + ;; + esac + RESOLVED="$RESOLVED $MISSING" + + build_portable_wheel "$MISSING" +done + +# pip, setuptools and wheel go in too: pip needs them to build anything on the +# device, and it lets the device upgrade its own pip without internet +"$BUILD_PIP" download --quiet "${PIP_NET_ARGS[@]}" -d "$WHEELHOUSE" --only-binary=:all: --platform any \ + --python-version "$TARGET_PYTHON" --implementation py --abi none \ + pip setuptools wheel >>"$PIP_LOG" 2>&1 || + echo "Note: could not add pip/setuptools/wheel to the wheelhouse" + +cp "$REQUIREMENTS" "$WHEELHOUSE/requirements.txt" + +WHEEL_COUNT=$(find "$WHEELHOUSE" -name '*.whl' | wc -l) +echo "Wheelhouse: $WHEEL_COUNT wheels" + +# ----------------------------------------------------------------------- ttyd + +if [ "$SKIP_TTYD" -eq 1 ]; then + echo "Skipping ttyd" +else + echo "Downloading ttyd ${TTYD_VERSION} for ${TTYD_SUFFIX}..." + if curl -fL --progress-bar -o "$BUNDLE_DIR/bin/ttyd" \ + "https://github.com/tsl0922/ttyd/releases/download/${TTYD_VERSION}/ttyd.${TTYD_SUFFIX}"; then + chmod +x "$BUNDLE_DIR/bin/ttyd" + else + echo "Warning: could not download ttyd, the bundle will not carry it" + rm -f "$BUNDLE_DIR/bin/ttyd" + fi +fi + +# ----------------------------------------------------------------- deb packages + +if [ -n "$DEBS_SOURCE" ]; then + if [ ! -d "$DEBS_SOURCE" ]; then + echo "Error: --debs directory not found: $DEBS_SOURCE" >&2 + exit 1 + fi + if ls "$DEBS_SOURCE"/*.deb >/dev/null 2>&1; then + cp "$DEBS_SOURCE"/*.deb "$BUNDLE_DIR/debs/" + echo "Included $(find "$BUNDLE_DIR/debs" -name '*.deb' | wc -l) .deb packages" + else + echo "Warning: no .deb files found in $DEBS_SOURCE" + fi +fi + +# -------------------------------------------------------------------- manifest + +cat > "$BUNDLE_DIR/MANIFEST" < "$BUNDLE_DIR/install-offline.sh" <<'INSTALLER' +#!/bin/bash +# +# Installs or updates DWE OS from this bundle. Nothing here touches the +# network. Run as root on the device: +# +# sudo ./install-offline.sh +# +# Any extra arguments are passed straight through to install-local.sh, so +# --recreate-venv and --install-dir work here too. + +set -e + +BUNDLE_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +if [ "$EUID" -ne 0 ]; then + echo "This script must be run as root" >&2 + exit 1 +fi + +manifest_value() { + sed -n "s/^$1=//p" "$BUNDLE_DIR/MANIFEST" +} + +BUNDLE_ARCH=$(manifest_value TARGET_ARCH) +BUNDLE_PYTHON=$(manifest_value TARGET_PYTHON) +BUNDLE_VERSION=$(manifest_value DWEOS_VERSION) + +DEVICE_ARCH=$(uname -m) +DEVICE_PYTHON=$(python3 -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null || echo "none") + +echo "Bundle: DWE OS $BUNDLE_VERSION for $BUNDLE_ARCH / Python $BUNDLE_PYTHON" +echo "Device: $DEVICE_ARCH / Python $DEVICE_PYTHON" + +# The wheels in this bundle were picked for one architecture and one Python +# version. Installing them anywhere else fails at import time rather than +# install time, so stop here instead. +if [ "$DEVICE_ARCH" != "$BUNDLE_ARCH" ]; then + echo >&2 + echo "Error: this bundle was built for $BUNDLE_ARCH but this device is $DEVICE_ARCH." >&2 + echo "Rebuild it with: ./package-offline.sh --arch $DEVICE_ARCH" >&2 + exit 1 +fi + +if [ "$DEVICE_PYTHON" != "$BUNDLE_PYTHON" ]; then + echo >&2 + echo "Error: this bundle was built for Python $BUNDLE_PYTHON but this device" >&2 + echo "runs Python $DEVICE_PYTHON." >&2 + echo "Rebuild it with: ./package-offline.sh --arch $DEVICE_ARCH --python $DEVICE_PYTHON" >&2 + exit 1 +fi + +TTYD_ARGS=() +if [ -f "$BUNDLE_DIR/bin/ttyd" ]; then + TTYD_ARGS=(--ttyd-binary "$BUNDLE_DIR/bin/ttyd") +fi + +exec bash "$BUNDLE_DIR/install-local.sh" \ + --offline \ + --wheelhouse "$BUNDLE_DIR/wheelhouse" \ + --debs "$BUNDLE_DIR/debs" \ + "${TTYD_ARGS[@]}" \ + "$@" \ + "$BUNDLE_DIR/release.tar.gz" +INSTALLER +chmod +x "$BUNDLE_DIR/install-offline.sh" + +# ---------------------------------------------------------------------- output + +if [ -z "$OUTPUT" ]; then + OUTPUT="$PWD/dweos-offline-${RELEASE_VERSION}-${TARGET_ARCH}-py${TARGET_PYTHON}.tar.gz" +fi + +tar -czf "$OUTPUT" -C "$WORK_DIR" dweos-offline + +echo +echo "Bundle written to $OUTPUT ($(du -h "$OUTPUT" | cut -f1))" + +# deploy-offline.sh sets this: it is about to copy the bundle over itself +if [ -z "${PACKAGE_HIDE_HINTS:-}" ]; then + echo + echo "Copy it to the device and run:" + echo " tar -xzf $(basename "$OUTPUT")" + echo " sudo ./dweos-offline/install-offline.sh" + echo + echo "Or do both at once: ./deploy-offline.sh --bundle $OUTPUT user@device" +fi