diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 90b9a38409..52afac8649 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -262,6 +262,10 @@ When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`): rootful Podman uses absolute host IDs (e.g. `uidmap = ["0:1000:1", "1:100000:65536"]`). - `nomap` (without hyphen) is accepted as input but canonicalized to `no-map` for Podman's API. +- A workload remains in `stopping` until Podman resorts to `SIGKILL`: inspect + supervisor logs for `failed to signal entrypoint process group`. The + supervisor must retain `CAP_KILL` so its root process can forward `SIGTERM` + to a workload that runs as the sandbox user. ### Step 6: Check Kubernetes Helm Gateways diff --git a/.github/actions/setup-e2e-podman/action.yml b/.github/actions/setup-e2e-podman/action.yml deleted file mode 100644 index 47f57f5be8..0000000000 --- a/.github/actions/setup-e2e-podman/action.yml +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Setup E2E Podman -description: Configure the supported rootless Podman environment for E2E tests - -inputs: - mise-version: - description: mise release to install - required: false - default: v2026.4.25 - podman-major: - description: Expected Podman major version - required: true - podman-package-version: - description: Exact Ubuntu Podman package version - required: true - conmon-package-version: - description: Exact Ubuntu conmon package version - required: true - -runs: - using: composite - steps: - - uses: ./.github/actions/setup-mise - with: - version: ${{ inputs.mise-version }} - - - name: Install Podman and build dependencies - shell: bash - run: | # zizmor: ignore[github-env] persists only literal and trusted runner-derived paths - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - apparmor \ - build-essential \ - fuse-overlayfs \ - libssl-dev \ - openssh-client \ - passt \ - pkg-config \ - "conmon=${INPUTS_CONMON_PACKAGE_VERSION}" \ - "podman=${INPUTS_PODMAN_PACKAGE_VERSION}" \ - uidmap - # Hosted runners can place newer binaries under /usr/local. Select the - # distro CLI and use Podman's supported final conmon-path override. - podman_config="${RUNNER_TEMP}/openshell-containers.conf" - printf '%s\n' \ - '[engine]' \ - 'conmon_path = ["/usr/bin/conmon"]' \ - > "${podman_config}" - echo "/usr/bin" >> "${GITHUB_PATH}" - echo "CONTAINERS_CONF_OVERRIDE=${podman_config}" >> "${GITHUB_ENV}" - env: - INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} - INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} - - - name: Allow pasta to receive Podman stop signals - shell: bash - run: | - set -euo pipefail - # Ubuntu's packaged profile blocks this signal and makes Podman wait - # for its SIGKILL fallback. - profile=/etc/apparmor.d/usr.bin.pasta - rule=' signal (receive) peer=podman,' - if ! sudo grep -Fqx "${rule}" "${profile}"; then - sudo sed -i '\|^ include $|a\ signal (receive) peer=podman,' "${profile}" - fi - sudo grep -Fqx "${rule}" "${profile}" - sudo apparmor_parser --replace "${profile}" - - - name: Configure rootless Podman - shell: bash - run: | # zizmor: ignore[github-env] runtime path is derived solely from the runner UID - set -euo pipefail - if ! grep -q "^${USER}:" /etc/subuid; then - sudo usermod --add-subuids 100000-165535 "$USER" - fi - if ! grep -q "^${USER}:" /etc/subgid; then - sudo usermod --add-subgids 100000-165535 "$USER" - fi - runtime_dir="/run/user/$(id -u)" - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "$runtime_dir" - echo "XDG_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" - - - name: Verify rootless Podman environment - shell: bash - run: | - set -euo pipefail - podman_version="$(podman version --format '{{.Client.Version}}')" - case "$podman_version" in - "${INPUTS_PODMAN_MAJOR}".*) ;; - *) echo "ERROR: expected Podman ${INPUTS_PODMAN_MAJOR}.x, found $podman_version" >&2; exit 1 ;; - esac - test "$(dpkg-query -W -f='${Version}' podman)" = "${INPUTS_PODMAN_PACKAGE_VERSION}" - test "$(dpkg-query -W -f='${Version}' conmon)" = "${INPUTS_CONMON_PACKAGE_VERSION}" - test "$(command -v podman)" = "/usr/bin/podman" - test "$(podman info --format '{{.Host.Conmon.Path}}')" = "/usr/bin/conmon" - test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true" - test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta" - test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1" - echo "=== host ===" - uname -a - echo "=== AppArmor ===" - cat /proc/self/attr/current - sudo aa-status || true - echo "=== Podman ===" - podman version - podman info --debug - env: - INPUTS_PODMAN_MAJOR: ${{ inputs.podman-major }} - INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} - INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} - - - name: Probe rootless capability bounding set - shell: bash - run: | - set -euo pipefail - probe="$RUNNER_TEMP/openshell-capbset-probe" - cc -static -O2 -Wall -Wextra -Werror \ - e2e/support/capbset-probe.c \ - -o "$probe" - podman run --rm \ - --cap-add=SETPCAP \ - --volume "$probe:/openshell-capbset-probe:ro" \ - docker.io/library/alpine:3.22 \ - /openshell-capbset-probe diff --git a/.github/actions/setup-e2e-sandbox/action.yml b/.github/actions/setup-e2e-sandbox/action.yml new file mode 100644 index 0000000000..943bd75618 --- /dev/null +++ b/.github/actions/setup-e2e-sandbox/action.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup E2E Sandbox +description: Download an architecture-matched prebuilt OpenShell sandbox binary for E2E tests + +runs: + using: composite + steps: + - name: Download prebuilt sandbox + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ runner.arch == 'X64' && 'openshell-sandbox-x86_64-unknown-linux-musl' || 'openshell-sandbox-aarch64-unknown-linux-musl' }} + path: .e2e/prebuilt-sandbox + + - name: Configure prebuilt sandbox + shell: bash + run: | # zizmor: ignore[github-env] persists only a trusted runner-derived path + set -euo pipefail + sandbox="$GITHUB_WORKSPACE/.e2e/prebuilt-sandbox/openshell-sandbox" + if [[ ! -f "$sandbox" ]]; then + echo "downloaded artifact is missing $sandbox" >&2 + exit 1 + fi + chmod +x "$sandbox" + "$sandbox" --version + echo "OPENSHELL_SANDBOX_BIN=$sandbox" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 0b050d0e51..419dc011cb 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -168,22 +168,6 @@ jobs: interpreter: /lib/ld-linux-aarch64.so.1 secrets: inherit - build-driver-podman: - needs: [pr_metadata, version] - if: needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - contents: read - uses: ./.github/workflows/build-binaries.yml - with: - package: openshell-driver-podman - binary: openshell-driver-podman - triple: x86_64-unknown-linux-gnu - runner: linux-amd64-cpu8 - dev-shell: .#devShells.x86_64-linux.glibc-2-28 - cargo-version: ${{ needs.version.outputs.cargo }} - interpreter: /lib64/ld-linux-x86-64.so.2 - secrets: inherit - build-driver-kubernetes: needs: [pr_metadata, version] if: needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -253,16 +237,13 @@ jobs: runner: linux-arm64-cpu8 podman-e2e: - needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image] + needs: [pr_metadata, build-cli, build-gateway, build-sandbox] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - vm-e2e: needs: [pr_metadata, build-cli, build-gateway, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -288,21 +269,6 @@ jobs: suite-matrix: >- [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] - podman-external-driver-e2e: - needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-podman, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - external-driver-binary: openshell-driver-podman - suite-matrix: >- - [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] - vm-external-driver-e2e: needs: [pr_metadata, build-cli, build-gateway-plain, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -428,7 +394,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, podman-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index 4a5a6d38d7..74f83b6c5f 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -6,9 +6,6 @@ name: Podman E2E Test on: workflow_call: inputs: - image-tag: - required: true - type: string checkout-ref: required: false type: string @@ -17,15 +14,6 @@ on: required: false type: string default: "" - external-driver-binary: - required: false - type: string - default: "" - suite-matrix: - required: false - type: string - default: >- - [{"suite":"provider-refresh-keycloak","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:provider-refresh-keycloak"}] permissions: actions: read @@ -34,18 +22,14 @@ permissions: jobs: e2e: - name: E2E (rust-podman-${{ matrix.suite }}, ${{ matrix.runner }}) - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.suite-matrix) }} + name: E2E (rust-podman-rootless, Ubuntu 26.04 Nix VM) + runs-on: ubuntu-26.04 + # Run rootless Podman inside a Nix-managed Ubuntu guest so Podman, pasta, + # and user-namespace setup are provisioned by versioned repository tooling + # rather than mutable hosted-runner packages. + timeout-minutes: 60 env: - IMAGE_TAG: ${{ inputs.image-tag }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell - OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -58,34 +42,48 @@ jobs: with: artifact-name: ${{ inputs.gateway-artifact }} - - if: inputs.external-driver-binary != '' - uses: ./.github/actions/setup-e2e-driver + - uses: ./.github/actions/setup-e2e-sandbox + + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 with: - binary: ${{ inputs.external-driver-binary }} + github_access_token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/setup-e2e-podman + - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17 with: - podman-major: ${{ matrix.podman_major }} - podman-package-version: ${{ matrix.podman_package_version }} - conmon-package-version: ${{ matrix.conmon_package_version }} + name: openshell - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + - name: Install mise + run: | + curl https://mise.run | MISE_VERSION=v2026.4.25 sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - - name: Run tests - run: ${{ matrix.cmd }} + - name: Install tools + run: mise install --locked - - name: Print AppArmor denials - if: always() - run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libclang-dev \ + libssl-dev \ + libz3-dev \ + openssh-client \ + pkg-config - - name: Fail on pasta SIGTERM AppArmor denial - if: always() + - name: Run tests run: | set -euo pipefail - denials="$(sudo dmesg | grep -E 'profile="pasta".*requested_mask="receive".*signal=term.*peer="podman"' || true)" - if [ -n "${denials}" ]; then - echo "::error::pasta denied Podman's SIGTERM; Podman will use its SIGKILL fallback" - printf '%s\n' "${denials}" - exit 1 - fi + + mise x -- e2e/run.sh \ + --vm ubuntu-26-04 \ + --with podman-rootless \ + --tests-in-vm \ + --cli-bin "$OPENSHELL_BIN" \ + --gateway-bin "$OPENSHELL_GATEWAY_BIN" \ + --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \ + --gateway-config e2e/configs/gateway/podman.toml \ + --features e2e-podman diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 7982a81643..c756f2c5e2 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -136,15 +136,12 @@ jobs: runner: linux-arm64-cpu8 podman-e2e: - needs: [build-cli, build-gateway, build-supervisor-image] + needs: [build-cli, build-gateway, build-sandbox] permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - vm-e2e: needs: [build-cli, build-gateway, build-vm-driver] permissions: diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 92eb4adf14..a2f819d19f 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -165,14 +165,13 @@ jobs: runner: linux-arm64-cpu8 podman-e2e: - needs: [compute-versions, build-cli, build-gateway, build-supervisor-image] + needs: [compute-versions, build-cli, build-gateway, build-sandbox] permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml with: - image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} vm-e2e: diff --git a/TESTING.md b/TESTING.md index 6c0829060d..94e457d6ba 100644 --- a/TESTING.md +++ b/TESTING.md @@ -175,6 +175,12 @@ Run the Podman-backed Rust CLI e2e suite: mise run e2e:podman ``` +Run the rootless Podman suite in an Ubuntu 26.04 Nix test guest: + +```shell +mise run e2e:podman:rootless +``` + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index abb9d69dd2..a81ee13e1d 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1114,8 +1114,6 @@ pub fn build_container_spec_for_image( "DAC_OVERRIDE".into(), // Not needed: the supervisor does not create setuid/setgid executables. "FSETID".into(), - // Not needed: the supervisor does not send signals to arbitrary processes. - "KILL".into(), // Not needed: the supervisor does not bind privileged ports (<1024). "NET_BIND_SERVICE".into(), // Not in Podman's default set but explicitly denied in case the image @@ -1146,6 +1144,9 @@ pub fn build_container_spec_for_image( // Child setup clears the capability bounding set before exec, which // requires CAP_SETPCAP in the supervisor until drop_privileges(). "SETPCAP".into(), + // Forwarding shutdown signals to the canonical workload process + // group after it drops to the sandbox UID requires CAP_KILL. + "KILL".into(), ], // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from // Podman's default set and not dropped: @@ -1885,6 +1886,7 @@ mod tests { "missing DAC_READ_SEARCH" ); assert!(added.contains(&"SETPCAP"), "missing SETPCAP"); + assert!(added.contains(&"KILL"), "missing KILL"); // SETUID and SETGID are NOT in cap_add — they remain available from the // default bounding set because we no longer use cap_drop:ALL. Verify they @@ -1916,6 +1918,10 @@ mod tests { !dropped.contains(&"SETPCAP"), "SETPCAP must not be dropped (needed for child bounding-set clear)" ); + assert!( + !dropped.contains(&"KILL"), + "KILL must not be dropped (needed to signal the sandbox workload on shutdown)" + ); assert!( !dropped.contains(&"ALL"), "must not use cap_drop:ALL in rootless Podman" diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index b2820d9588..8c47e789ba 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -453,24 +453,34 @@ pub async fn run_process( .build() ); - if let Some(tx) = sidecar_exit_tx.as_ref() { - report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code).await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + if outcome.should_report_main_process_exit() { + if let Some(tx) = sidecar_exit_tx.as_ref() { + report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?; + } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { + report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code) + .await; + info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + } + } else { + info!( + instance_id = %main_instance_id, + "skipping main-process exit report during supervisor shutdown" + ); } main_session.mark_terminal_reported(); - if drain_terminal && terminal_delivery_pending { + if outcome.should_report_main_process_exit() && drain_terminal && terminal_delivery_pending { // The peer's SSH channel-close confirms that the terminal frames sent // above traversed russh and the relay. Detached commands have no active // attachment and never enter this wait. main_session.wait_for_terminal_attachments().await; } - if let Some(tx) = sidecar_exit_tx.as_ref() { - finalize_sidecar_main_process_exit(tx, &main_instance_id).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await; - info!(instance_id = %main_instance_id, "main-process terminal delivery finalized"); + if outcome.should_report_main_process_exit() { + if let Some(tx) = sidecar_exit_tx.as_ref() { + finalize_sidecar_main_process_exit(tx, &main_instance_id).await?; + } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { + finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await; + info!(instance_id = %main_instance_id, "main-process terminal delivery finalized"); + } } supervisor_terminating.store(true, Ordering::Release); @@ -572,6 +582,16 @@ enum ProcessWaitOutcome { }, } +impl ProcessWaitOutcome { + /// A gateway acknowledgement is required for ordinary canonical-process + /// completion, but cannot be awaited after the supervisor itself has been + /// asked to terminate. At that point the gateway may already be shutting + /// down and no longer able to acknowledge the report. + fn should_report_main_process_exit(&self) -> bool { + !matches!(self, Self::ShutdownSignal { .. }) + } +} + async fn wait_for_process_exit_or_shutdown( handle: &mut ProcessHandle, timeout_secs: u64, @@ -789,4 +809,22 @@ mod tests { assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); } + + #[cfg(unix)] + #[test] + fn supervisor_shutdown_exit_skips_gateway_acknowledgement() { + use std::os::unix::process::ExitStatusExt; + + let status = ProcessStatus::from(std::process::ExitStatus::from_raw(libc::SIGTERM)); + + assert!(ProcessWaitOutcome::Exited(status).should_report_main_process_exit()); + assert!(ProcessWaitOutcome::TimedOut.should_report_main_process_exit()); + assert!( + !ProcessWaitOutcome::ShutdownSignal { + signal: "SIGTERM", + status, + } + .should_report_main_process_exit() + ); + } } diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index c1549cd933..48e84a4bde 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -26,3 +26,5 @@ image_pull_policy = "missing" network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +# Keep rootless Podman E2E teardown bounded. +stop_timeout_secs = 15 diff --git a/e2e/run.sh b/e2e/run.sh index 0505730f05..bf364f6176 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Build the current checkout, run its gateway on the host or in a disposable -# Nix test guest, and execute one named host-side E2E suite against that gateway. +# Nix test guest, and execute E2E tests against that gateway. set -Eeuo pipefail @@ -20,19 +20,27 @@ usage() { cat <<'EOF' Usage: e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ - --gateway-config PATH --suite NAME + --gateway-config PATH [--features FEATURES] [--suite NAME] Options: --vm DISTRO Run the gateway in a Nix test guest --with CONFIG Apply a Nix test-guest configuration; repeatable + --tests-in-vm Prebuild Linux Rust E2E test binaries on the host, + copy them into the Nix test guest, and run them there + --cli-bin PATH Use a prebuilt openshell CLI instead of building it + --gateway-bin PATH Use a prebuilt openshell-gateway instead of building it + --sandbox-bin PATH Use a prebuilt openshell-sandbox instead of building it --gateway-config PATH Fully resolved gateway TOML + --features FEATURES Rust e2e feature set to enable (default: e2e) --suite NAME Rust suite at e2e/rust/tests/NAME.rs -h, --help Show this help Omit --vm and --with to run the gateway on the host. Supplying --with without --vm selects Fedora for the Podman driver and Ubuntu otherwise. Set -OPENSHELL_E2E_KEEP=1 to retain state. +OPENSHELL_CLI_BIN for a default --cli-bin; otherwise --tests-in-vm +cross-builds the guest CLI with cargo-zigbuild. Set OPENSHELL_E2E_KEEP=1 to +retain state. EOF } @@ -92,7 +100,12 @@ catalog_has_entry() { vm= gateway_config= +gateway_bin= +cli_bin= +sandbox_bin= +e2e_features=e2e suite_name= +tests_in_vm=0 with_configurations=() while [ "$#" -gt 0 ]; do @@ -107,11 +120,35 @@ while [ "$#" -gt 0 ]; do with_configurations+=("$2") shift 2 ;; + --tests-in-vm) + tests_in_vm=1 + shift + ;; + --cli-bin) + require_value "$1" "$#" "${2:-}" + cli_bin="$(resolve_file "$2")" || die "--cli-bin does not name a file: $2" + shift 2 + ;; + --gateway-bin) + require_value "$1" "$#" "${2:-}" + gateway_bin="$(resolve_file "$2")" || die "--gateway-bin does not name a file: $2" + shift 2 + ;; + --sandbox-bin) + require_value "$1" "$#" "${2:-}" + sandbox_bin="$(resolve_file "$2")" || die "--sandbox-bin does not name a file: $2" + shift 2 + ;; --gateway-config) require_value "$1" "$#" "${2:-}" gateway_config=$2 shift 2 ;; + --features) + require_value "$1" "$#" "${2:-}" + e2e_features=$2 + shift 2 + ;; --suite) require_value "$1" "$#" "${2:-}" suite_name=$2 @@ -130,26 +167,31 @@ done if [ -z "${gateway_config}" ]; then die "--gateway-config is required" fi -if [ -z "${suite_name}" ]; then - die "--suite is required" -fi if ! command -v python3 >/dev/null 2>&1; then die "python3 is required" fi +if ! command -v mise >/dev/null 2>&1; then + die "mise is required to build OpenShell" +fi gateway_config_source=${gateway_config} if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then die "gateway config does not exist: ${gateway_config_source}" fi -gateway_driver="$(python3 -c ' +gateway_driver="$(mise x -- python3 -c ' import sys, tomllib print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) ' "${gateway_config}")" -if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then - die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" +if [ -z "${e2e_features}" ]; then + die "--features must not be empty" fi -suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" -if [ ! -f "${suite_path}" ]; then - die "unknown suite: ${suite_name}" +if [ -n "${suite_name}" ]; then + if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then + die "suite name must contain only lowercase letters, digits, underscores, and hyphens: ${suite_name}" + fi + suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" + if [ ! -f "${suite_path}" ]; then + die "unknown suite: ${suite_name}" + fi fi mode=host if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then @@ -158,7 +200,7 @@ if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then if [ "${gateway_driver}" = podman ]; then vm=fedora else - vm=ubuntu + vm=ubuntu-24-04 fi fi fi @@ -171,9 +213,6 @@ if [ "${mode}" = vm ]; then die "invalid VM configuration name: ${configuration}" fi done - if [ "${gateway_driver}" = podman ] && [ "${vm}" = ubuntu ]; then - die "the Ubuntu 24.04 guest lacks the Podman 5 pasta helper required for sandbox callbacks; use --vm fedora --with podman" - fi if ! command -v nix >/dev/null 2>&1; then die "Nix is required for VM mode" fi @@ -191,15 +230,17 @@ if [ "${mode}" = vm ]; then die "unknown VM configuration in the Nix test-guest catalog: ${configuration}" fi done +elif [ "${tests_in_vm}" -eq 1 ]; then + die "--tests-in-vm requires --vm" +fi +if [ -z "${cli_bin}" ] && [ -n "${OPENSHELL_CLI_BIN:-}" ]; then + cli_bin="$(resolve_file "${OPENSHELL_CLI_BIN}")" || + die "OPENSHELL_CLI_BIN does not name a file: ${OPENSHELL_CLI_BIN}" fi - gateway_ready_timeout=${OPENSHELL_E2E_GATEWAY_READY_TIMEOUT:-600} if [[ ! ${gateway_ready_timeout} =~ ^[1-9][0-9]*$ ]]; then die "OPENSHELL_E2E_GATEWAY_READY_TIMEOUT must be a positive integer" fi -if ! command -v mise >/dev/null 2>&1; then - die "mise is required to build OpenShell" -fi if ! command -v openssl >/dev/null 2>&1; then die "OpenSSL is required to generate sandbox JWT keys" fi @@ -230,56 +271,93 @@ target_dir="$(e2e_cargo_target_dir "${ROOT}" mise x -- cargo)" ensure_build_nofile_limit -echo "==> Building native host openshell CLI" -mise x -- cargo build "${cargo_jobs[@]}" -p openshell-cli --bin openshell -host_cli_bin="${target_dir}/debug/openshell" - -echo "==> Preparing ${linux_musl_target} build target" -mise x -- rustup target add "${linux_musl_target}" >/dev/null +if [ "${tests_in_vm}" -eq 1 ]; then + if [ -n "${cli_bin}" ]; then + echo "==> Using Linux guest openshell CLI: ${cli_bin}" + else + echo "==> Building Linux guest openshell CLI (${linux_musl_target})" + mise x -- rustup target add "${linux_musl_target}" >/dev/null + ( + export CXXSTDLIB=c++ + mise x -- cargo zigbuild "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-cli \ + --bin openshell + ) + cli_bin="${target_dir}/${linux_musl_target}/release/openshell" + fi +elif [ -n "${cli_bin}" ]; then + echo "==> Using host openshell CLI: ${cli_bin}" +else + echo "==> Building native host openshell CLI" + mise x -- cargo build "${cargo_jobs[@]+"${cargo_jobs[@]}"}" -p openshell-cli --bin openshell + cli_bin="${target_dir}/debug/openshell" +fi -echo "==> Building Linux openshell-sandbox (${linux_musl_target})" -mise x -- cargo zigbuild "${cargo_jobs[@]}" \ - --release \ - --target "${linux_musl_target}" \ - -p openshell-sandbox \ - --bin openshell-sandbox -linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" +if [ -n "${sandbox_bin}" ]; then + echo "==> Using Linux openshell-sandbox: ${sandbox_bin}" + linux_sandbox_bin="${sandbox_bin}" +else + echo "==> Preparing ${linux_musl_target} build target" + mise x -- rustup target add "${linux_musl_target}" >/dev/null + + echo "==> Building Linux openshell-sandbox (${linux_musl_target})" + mise x -- cargo zigbuild "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-sandbox \ + --bin openshell-sandbox + linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" +fi host_gateway_bin= guest_gateway_bin= if [ "${mode}" = host ]; then - echo "==> Building native host openshell-gateway" - mise x -- cargo build "${cargo_jobs[@]}" \ - -p openshell-server \ - --bin openshell-gateway \ - --features bundled-z3 - host_gateway_bin="${target_dir}/debug/openshell-gateway" -else - echo "==> Preparing ${linux_gateway_rust_target} build target" - mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null - echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" - ( - eval "$( - "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ - "${linux_gateway_zig_target}" \ - "${linux_gateway_zig_target}" \ - "${target_dir}/zig-gnu-wrapper/e2e" - )" - mise x -- cargo zigbuild "${cargo_jobs[@]}" \ - --release \ - --target "${linux_gateway_zig_target}" \ + if [ -n "${gateway_bin}" ]; then + echo "==> Using host openshell-gateway: ${gateway_bin}" + host_gateway_bin="${gateway_bin}" + else + echo "==> Building native host openshell-gateway" + mise x -- cargo build "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ -p openshell-server \ --bin openshell-gateway \ --features bundled-z3 - ) - guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" + host_gateway_bin="${target_dir}/debug/openshell-gateway" + fi +else + if [ -n "${gateway_bin}" ]; then + echo "==> Using Linux openshell-gateway: ${gateway_bin}" + guest_gateway_bin="${gateway_bin}" + else + echo "==> Preparing ${linux_gateway_rust_target} build target" + mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null + echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" + ( + eval "$( + "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ + "${linux_gateway_zig_target}" \ + "${linux_gateway_zig_target}" \ + "${target_dir}/zig-gnu-wrapper/e2e" + )" + mise x -- cargo zigbuild "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ + --release \ + --target "${linux_gateway_zig_target}" \ + -p openshell-server \ + --bin openshell-gateway \ + --features bundled-z3 + ) + guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" + fi fi -expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}") -if [ "${mode}" = host ]; then - expected_binaries+=("${host_gateway_bin}") +expected_binaries=("${linux_sandbox_bin}") +if [ "${tests_in_vm}" -eq 1 ]; then + expected_binaries+=("${cli_bin}" "${guest_gateway_bin}") +elif [ "${mode}" = host ]; then + expected_binaries+=("${cli_bin}" "${host_gateway_bin}") else - expected_binaries+=("${guest_gateway_bin}") + expected_binaries+=("${cli_bin}" "${guest_gateway_bin}") fi for binary in "${expected_binaries[@]}"; do if [ ! -x "${binary}" ]; then @@ -300,6 +378,8 @@ supervisor_archive="${run_dir}/supervisor.tar" mkdir -p "${supervisor_rootfs}" install -m 0555 "${linux_sandbox_bin}" "${supervisor_rootfs}/openshell-sandbox" tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-sandbox +chmod 0644 "${supervisor_archive}" +test_artifacts=() child_pid= runtime_log= keep=0 @@ -321,6 +401,65 @@ start_child() { child_pid=$! } +build_e2e_test_artifacts() { + local build_log="${run_dir}/e2e-test-build.jsonl" + local artifacts_file="${run_dir}/e2e-test-artifacts.txt" + local build_args=( + mise x -- cargo zigbuild + --manifest-path e2e/rust/Cargo.toml + --features "${e2e_features}" + --target "${linux_gateway_zig_target}" + --message-format=json + ) + if [ -n "${suite_name}" ]; then + build_args+=(--test "${suite_name}") + else + build_args+=(--tests) + fi + + echo "==> Prebuilding E2E test artifacts for guest execution (${linux_gateway_rust_target})" + if ! ( + eval "$( + "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ + "${linux_gateway_zig_target}" \ + "${linux_gateway_zig_target}" \ + "${target_dir}/zig-gnu-wrapper/e2e-tests" + )" + "${build_args[@]}" + ) >"${build_log}"; then + echo "=== E2E test artifact build output ===" >&2 + cat "${build_log}" >&2 + echo "=== end E2E test artifact build output ===" >&2 + return 1 + fi + python3 - "${build_log}" >"${artifacts_file}" <<'PY' +import json +import sys + +for line in open(sys.argv[1], encoding="utf-8"): + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if message.get("reason") != "compiler-artifact": + continue + target = message.get("target") or {} + if "test" not in (target.get("kind") or []): + continue + executable = message.get("executable") + if executable: + print(executable) +PY + while IFS= read -r artifact; do + if [ -n "${artifact}" ]; then + test_artifacts+=("${artifact}") + fi + done <"${artifacts_file}" + if [ "${#test_artifacts[@]}" -eq 0 ]; then + die "cargo did not report any E2E test executables" + fi +} + # Invoked by the EXIT trap through cleanup. # shellcheck disable=SC2329 stop_child() { @@ -367,6 +506,10 @@ trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM +if [ "${tests_in_vm}" -eq 1 ]; then + build_e2e_test_artifacts +fi + jwt_source_dir="${run_dir}/gateway-jwt" host_runtime_dir= if [ "${mode}" = host ]; then @@ -390,7 +533,7 @@ gateway_name="openshell-e2e-${mode}-${host_port}" gateway_endpoint="http://127.0.0.1:${host_port}" export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" export OPENSHELL_GATEWAY="${gateway_name}" -export OPENSHELL_BIN="${host_cli_bin}" +export OPENSHELL_BIN="${cli_bin}" if [ "${mode}" = host ]; then case "${gateway_driver}" in @@ -424,6 +567,23 @@ else guest_launcher="${run_dir}/launch-gateway.sh" guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway guest_supervisor_archive_path=/home/openshell/.cache/openshell-e2e/supervisor.tar + guest_test_artifact_dir=/home/openshell/.cache/openshell-e2e/tests + guest_test_manifest="${run_dir}/test-artifacts.txt" + guest_test_manifest_path=/home/openshell/.cache/openshell-e2e/test-artifacts.txt + if [ "${tests_in_vm}" -eq 1 ]; then + : >"${guest_test_manifest}" + for artifact in "${test_artifacts[@]}"; do + printf '%s/%s\n' "${guest_test_artifact_dir}" "${artifact##*/}" >>"${guest_test_manifest}" + done + chmod 0644 "${guest_test_manifest}" + fi + guest_e2e_network_name="$(mise x -- python3 - "${gateway_config}" <<'PY' +import sys, tomllib + +config = tomllib.load(open(sys.argv[1], "rb")) +print(config.get("openshell", {}).get("drivers", {}).get("podman", {}).get("network_name", "openshell-e2e")) +PY +)" config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" @@ -475,13 +635,178 @@ podman) esac report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" cd /home/openshell + +if [ '${tests_in_vm}' = 1 ]; then + gateway_log=\${state_root}/gateway.log + gateway_pid_file=\${state_root}/gateway.pid + gateway_args_file=\${state_root}/gateway.args + spiffe_root=\${state_root}/spiffe + mkdir -p "\${spiffe_root}" "${guest_test_artifact_dir}" + + toml_string() { + python3 - "\$1" <<'PY' +import json +import sys + +print(json.dumps(sys.argv[1])) +PY + } + + pick_free_port() { + python3 - <<'PY' +import socket + +sock = socket.socket() +sock.bind(("0.0.0.0", 0)) +print(sock.getsockname()[1]) +sock.close() +PY + } + + insert_podman_config_key() { + local key=\$1 + local value=\$2 + + python3 - "\${config_path}" "\${key}" "\${value}" <<'PY' +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +key = sys.argv[2] +value = sys.argv[3] +section = "[openshell.drivers.podman]" +lines = path.read_text(encoding="utf-8").splitlines() +try: + start = next(index for index, line in enumerate(lines) if line.strip() == section) +except StopIteration: + raise SystemExit(f"{section} not found in {path}") +end = len(lines) +for index in range(start + 1, len(lines)): + if lines[index].lstrip().startswith("["): + end = index + break +for line in lines[start + 1:end]: + if line.split("=", 1)[0].strip() == key: + raise SystemExit(0) +lines.insert(end, f"{key} = {value}") +path.write_text("\\n".join(lines) + "\\n", encoding="utf-8") +PY + } + + write_gateway_args_file() { + : >"\${gateway_args_file}" + for arg in "\$@"; do + printf '%s\0' "\${arg}" >>"\${gateway_args_file}" + done + } + + stop_gateway() { + local gateway_pid= + if [ -f "\${gateway_pid_file}" ]; then + gateway_pid=\$(cat "\${gateway_pid_file}" 2>/dev/null || true) + fi + if [ -n "\${gateway_pid}" ] && kill -0 "\${gateway_pid}" 2>/dev/null; then + kill "\${gateway_pid}" 2>/dev/null || true + for _ in \$(seq 1 60); do + kill -0 "\${gateway_pid}" 2>/dev/null || break + sleep 0.5 + done + kill -KILL "\${gateway_pid}" 2>/dev/null || true + wait "\${gateway_pid}" 2>/dev/null || true + fi + rm -f "\${gateway_pid_file}" 2>/dev/null || true + } + + cleanup_guest_tests() { + local status=\$? + trap - EXIT INT TERM + stop_gateway + if [ "\${status}" -ne 0 ] && [ -f "\${gateway_log}" ]; then + echo "=== guest gateway log ===" >&2 + cat "\${gateway_log}" >&2 + echo "=== end guest gateway log ===" >&2 + fi + exit "\${status}" + } + trap cleanup_guest_tests EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + export OPENSHELL_BIN=/usr/local/bin/openshell + export OPENSHELL_GATEWAY_ENDPOINT=http://127.0.0.1:${guest_port} + export OPENSHELL_GATEWAY=openshell-e2e-vm-${guest_port} + export OPENSHELL_PROVISION_TIMEOUT=\${OPENSHELL_PROVISION_TIMEOUT:-300} + export OPENSHELL_E2E_TESTS_IN_VM=1 + if [ '${gateway_driver}' = podman ]; then + export CONTAINER_ENGINE=podman + export OPENSHELL_E2E_DRIVER=podman + export OPENSHELL_E2E_NETWORK_NAME='${guest_e2e_network_name}' + export OPENSHELL_E2E_SANDBOX_NAMESPACE='${guest_e2e_network_name}' + export XDG_RUNTIME_DIR="\${XDG_RUNTIME_DIR:-/run/user/\$(id -u)}" + export OPENSHELL_PODMAN_SOCKET="\${XDG_RUNTIME_DIR}/podman/podman.sock" + export CONTAINER_HOST="unix://\${OPENSHELL_PODMAN_SOCKET}" + export OPENSHELL_E2E_CONTAINER_ENGINE_UNSET_XDG_CONFIG_HOME=1 + insert_podman_config_key socket_path "\$(toml_string "\${OPENSHELL_PODMAN_SOCKET}")" + insert_podman_config_key enable_bind_mounts true + + provider_spiffe_port=\$(pick_free_port) + export OPENSHELL_E2E_GATEWAY_SPIFFE_SOCKET="\${spiffe_root}/gateway.sock" + export OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET="\${OPENSHELL_E2E_GATEWAY_SPIFFE_SOCKET}" + export OPENSHELL_E2E_PROVIDER_SPIFFE_LISTEN="0.0.0.0:\${provider_spiffe_port}" + export OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET="tcp:169.254.1.2:\${provider_spiffe_port}" + insert_podman_config_key provider_spiffe_workload_api_socket "\$(toml_string "\${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" + fi + + gateway_args=( + --config "\${config_path}" + --bind-address 127.0.0.1 + --port ${guest_port} + --disable-tls + ) + write_gateway_args_file "\${gateway_args[@]}" + export OPENSHELL_E2E_GATEWAY_BIN=/usr/local/bin/openshell-gateway + export OPENSHELL_E2E_GATEWAY_ARGS_FILE="\${gateway_args_file}" + export OPENSHELL_E2E_GATEWAY_LOG="\${gateway_log}" + export OPENSHELL_E2E_GATEWAY_PID_FILE="\${gateway_pid_file}" + + /usr/local/bin/openshell-gateway "\${gateway_args[@]}" >"\${gateway_log}" 2>&1 & + printf '%s\n' "\$!" >"\${gateway_pid_file}" + + echo "==> Waiting for guest gateway readiness" + gateway_ready=0 + for _ in \$(seq 1 "${gateway_ready_timeout}"); do + if ! kill -0 "\$(cat "\${gateway_pid_file}")" 2>/dev/null; then + echo "ERROR: guest gateway exited before becoming ready" >&2 + exit 1 + fi + if NO_COLOR=1 /usr/local/bin/openshell status >/tmp/openshell-e2e-status.log 2>&1 && + grep -q "Connected" /tmp/openshell-e2e-status.log; then + gateway_ready=1 + break + fi + sleep 1 + done + if [ "\${gateway_ready}" -ne 1 ]; then + echo "ERROR: guest gateway did not become ready" >&2 + cat /tmp/openshell-e2e-status.log >&2 || true + exit 1 + fi + + while IFS= read -r test_bin <&3; do + [ -n "\${test_bin}" ] || continue + echo "==> Running guest E2E artifact: \${test_bin##*/}" + "\${test_bin}" --nocapture Running prebuilt E2E test artifacts inside ${vm} test guest" + "${vm_args[@]}" + exit $? + fi + echo "==> Starting ${vm} test guest gateway at ${gateway_endpoint}" start_child "${ROOT}" "${runtime_log}" "${vm_args[@]}" fi @@ -588,10 +926,17 @@ wait_for_gateway() { wait_for_gateway -echo "==> Running E2E suite: ${suite_name}" +test_args=( + cargo test + --manifest-path e2e/rust/Cargo.toml + --features "${e2e_features}" +) +echo "==> Running E2E features: ${e2e_features}" +if [ -n "${suite_name}" ]; then + echo "==> Running E2E suite: ${suite_name}" + test_args+=(--test "${suite_name}") +fi +test_args+=(-- --nocapture) + cd "${ROOT}" -cargo test \ - --manifest-path e2e/rust/Cargo.toml \ - --features e2e \ - --test "${suite_name}" \ - -- --nocapture +"${test_args[@]}" diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs index 7a1e12923a..4cded2354b 100644 --- a/e2e/rust/tests/live_policy_update.rs +++ b/e2e/rust/tests/live_policy_update.rs @@ -56,6 +56,11 @@ ENV OPENSHELL_POLICY_POLL_INTERVAL_SECS=1 CMD ["sleep", "infinity"] "#; +const SPARSE_POLICY: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../examples/policy-advisor/sandbox-policy.yaml" +)); + // --------------------------------------------------------------------------- // Policy YAML builders // --------------------------------------------------------------------------- @@ -146,6 +151,15 @@ landlock: Ok(file) } +fn write_sparse_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; + file.write_all(SPARSE_POLICY.as_bytes()) + .map_err(|e| format!("write temp policy file: {e}"))?; + file.flush() + .map_err(|e| format!("flush temp policy file: {e}"))?; + Ok(file) +} + #[cfg(feature = "e2e-docker")] fn write_local_override_image() -> Result { let dir = tempfile::tempdir().map_err(|e| format!("create image context: {e}"))?; @@ -521,18 +535,18 @@ async fn live_policy_update_from_empty_network_policies() { /// no revision remaining `Pending` once the acknowledgement lands. #[tokio::test] async fn initial_sparse_policy_is_acknowledged_as_loaded() { - // Repo-relative path to the sparse network-only policy fixture. - let sparse_policy = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../examples/policy-advisor/sandbox-policy.yaml" - ); + let sparse_policy = write_sparse_policy().expect("write sparse policy fixture"); + let sparse_policy_path = sparse_policy + .path() + .to_str() + .expect("sparse policy path is not UTF-8"); let mut guard = SandboxGuard::create_keep_with_args( &[ "--name", "e2e-sparse-enrich", "--policy", - sparse_policy, + sparse_policy_path, "--no-tty", ], &["sh", "-c", "echo Ready && sleep infinity"], diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 1027654641..aef5a39d87 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -141,6 +141,13 @@ async fn rootless_podman_musl_getaddrinfo_uses_udp_policy_dns() { if !is_e2e_driver("podman") { return; } + if std::env::var_os("OPENSHELL_E2E_TESTS_IN_VM").is_some() { + eprintln!( + "skipping musl DNS probe test in guest prebuilt-artifact mode; \ + restore with a prebuilt probe artifact tracked in #3009" + ); + return; + } let probe = MuslDnsProbe::build().expect("build static musl DNS probe"); let fixture = SupportContainer::start_python( diff --git a/e2e/support/capbset-probe.c b/e2e/support/capbset-probe.c deleted file mode 100644 index bc93d766db..0000000000 --- a/e2e/support/capbset-probe.c +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Verify the capability-bounding-set condition behind issue #2069 from -// inside a rootless Podman container. - -#include -#include -#include -#include -#include -#include - -static unsigned long long status_capability(const char *field) { - FILE *status = fopen("/proc/self/status", "r"); - if (status == NULL) { - perror("fopen(/proc/self/status)"); - exit(EXIT_FAILURE); - } - - char line[256]; - unsigned long long value = 0; - int found = 0; - while (fgets(line, sizeof(line), status) != NULL) { - char name[32]; - unsigned long long candidate; - if (sscanf(line, "%31[^:]:%llx", name, &candidate) == 2 && - strcmp(name, field) == 0) { - value = candidate; - found = 1; - break; - } - } - fclose(status); - - if (!found) { - fprintf(stderr, "missing %s in /proc/self/status\n", field); - exit(EXIT_FAILURE); - } - return value; -} - -static void print_apparmor_profile(void) { - FILE *profile = fopen("/proc/self/attr/current", "r"); - if (profile == NULL) { - perror("fopen(/proc/self/attr/current)"); - return; - } - - char line[256]; - if (fgets(line, sizeof(line), profile) != NULL) { - printf("apparmor_profile=%s", line); - if (strchr(line, '\n') == NULL) { - putchar('\n'); - } - } - fclose(profile); -} - -int main(int argc, char **argv) { - if (argc != 1) { - fprintf(stderr, "usage: %s\n", argv[0]); - return EXIT_FAILURE; - } - - const unsigned long long setpcap_mask = 1ULL << CAP_SETPCAP; - const unsigned long long cap_bnd_before = status_capability("CapBnd"); - const unsigned long long cap_eff_before = status_capability("CapEff"); - const int setpcap_before = prctl(PR_CAPBSET_READ, CAP_SETPCAP, 0, 0, 0); - if (setpcap_before == -1) { - perror("prctl(PR_CAPBSET_READ) before drop"); - return EXIT_FAILURE; - } - - print_apparmor_profile(); - printf("cap_bnd_before=%016llx\n", cap_bnd_before); - printf("cap_eff_before=%016llx\n", cap_eff_before); - printf("setpcap_bounding_before=%d\n", setpcap_before); - - if (cap_bnd_before == 0 || (cap_bnd_before & setpcap_mask) == 0 || - (cap_eff_before & setpcap_mask) == 0 || setpcap_before != 1) { - fprintf(stderr, "CAP_SETPCAP must be effective and present in a non-empty bounding set\n"); - return EXIT_FAILURE; - } - - errno = 0; - const int drop_result = prctl(PR_CAPBSET_DROP, CAP_SETPCAP, 0, 0, 0); - const int drop_errno = errno; - const unsigned long long cap_bnd_after = status_capability("CapBnd"); - const int setpcap_after = prctl(PR_CAPBSET_READ, CAP_SETPCAP, 0, 0, 0); - - printf("drop_result=%d\n", drop_result); - printf("drop_errno=%d (%s)\n", drop_errno, strerror(drop_errno)); - printf("cap_bnd_after=%016llx\n", cap_bnd_after); - printf("setpcap_bounding_after=%d\n", setpcap_after); - - if (drop_result == 0) { - if (setpcap_after != 0 || (cap_bnd_after & setpcap_mask) != 0) { - fprintf(stderr, "CAP_SETPCAP remained in the bounding set after a successful drop\n"); - return EXIT_FAILURE; - } - } else if (drop_errno == EPERM) { - if (setpcap_after != 1 || (cap_bnd_after & setpcap_mask) == 0) { - fprintf(stderr, "CAP_SETPCAP changed in the bounding set after EPERM\n"); - return EXIT_FAILURE; - } - } else { - fprintf(stderr, "unexpected PR_CAPBSET_DROP result\n"); - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index b608cb19fe..a65baf02a3 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -33,12 +33,14 @@ nix/test-guest/ ├── cache-lib.sh ├── cache-seal.sh ├── distros/ -│ ├── ubuntu.nix +│ ├── ubuntu-24-04.nix +│ ├── ubuntu-26-04.nix │ ├── centos.nix │ ├── fedora.nix │ └── rocky.nix └── configuration/ ├── docker.yml + ├── podman-rootless.yml ├── podman.yml └── selinux.yml ``` @@ -56,12 +58,13 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues ## Supported configurations -| Distro | Docker | Podman | SELinux | Package format | -| --- | --- | --- | --- | --- | -| Ubuntu 24.04 | Yes | Yes | No | `.deb` | -| CentOS Stream 10 | No | Yes | Yes | `.rpm` | -| Fedora 44 | No | Yes | Yes | `.rpm` | -| Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | +| Distro | Docker | Podman | Rootless Podman | SELinux | Package format | +| --- | --- | --- | --- | --- | --- | +| Ubuntu 24.04 | Yes | Yes | No | No | `.deb` | +| Ubuntu 26.04 | Yes | Yes | Yes | No | `.deb` | +| CentOS Stream 10 | No | Yes | No | Yes | `.rpm` | +| Fedora 44 | No | Yes | No | Yes | `.rpm` | +| Rocky Linux 9 | Yes | Yes | No | Yes | `.rpm` | The `snapd` configuration is available for Ubuntu and prepares snapd for local Snap lifecycle experiments. It does not install Docker, because the Snap @@ -70,8 +73,8 @@ interface rather than the host-package Docker configuration. The Ubuntu 24.04 Podman configuration is available for runtime and packaging checks, but its Podman 4 release does not provide the `pasta` rootless network -helper required by OpenShell sandbox callbacks. OpenShell Podman E2E runs use -the Fedora guest, which provides Podman 5 and `pasta`. +helper required by OpenShell sandbox callbacks. Rootless Podman E2E uses the +Ubuntu 26.04 guest with `--with podman-rootless`. List the available distros and configurations: @@ -84,13 +87,13 @@ nix run .#test-guest -- --list Boot a base Ubuntu VM: ```shell -nix run .#test-guest -- --distro ubuntu +nix run .#test-guest -- --distro ubuntu-24-04 ``` Apply the Docker configuration before opening the SSH session: ```shell -nix run .#test-guest -- --distro ubuntu --with docker +nix run .#test-guest -- --distro ubuntu-24-04 --with docker ``` Other combinations use the same interface: @@ -99,13 +102,14 @@ Other combinations use the same interface: nix run .#test-guest -- --distro rocky --with docker nix run .#test-guest -- --distro centos --with podman nix run .#test-guest -- --distro fedora --with podman +nix run .#test-guest -- --distro ubuntu-26-04 --with podman-rootless ``` Configurations are repeatable: ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --with podman ``` @@ -138,7 +142,7 @@ The `test-guest-cache` app ensures a prepared disk exists for one exact distro, ```shell nix run .#test-guest-cache -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker ``` @@ -147,7 +151,7 @@ backing cache: ```shell nix run .#test-guest-cache -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --repository ghcr.io/nvidia/openshell/test-guest-cache \ --digest sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef @@ -157,7 +161,7 @@ The command never publishes implicitly. Add `--push` after authenticating ORAS t ```shell nix run .#test-guest-cache -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --repository ghcr.io/nvidia/openshell/test-guest-cache \ --push @@ -184,7 +188,7 @@ The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-g Cache command options: ```text ---distro NAME Base distro: ubuntu, centos, fedora, or rocky +--distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --repository REF OCI repository without a tag --digest DIGEST Trusted OCI manifest digest required for pulls @@ -209,7 +213,7 @@ Install the package in an Ubuntu VM and run a command: ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --install artifacts/openshell_0.0.0-local_arm64.deb \ -- openshell --version @@ -226,7 +230,7 @@ guest file preserves the source's ordinary permission bits: ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --copy ./openshell:/usr/local/bin/openshell \ -- openshell --version ``` @@ -241,7 +245,7 @@ gateway. On each failure it prints snapd and gateway journals. ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with snapd \ --keep \ --copy ./openshell_*.snap:/tmp/openshell.snap \ @@ -259,7 +263,7 @@ The destination must be an absolute guest path. Copied files are installed with ## Runner options ```text ---distro NAME Base distro: ubuntu, centos, fedora, or rocky +--distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy a regular file into the guest, preserving its host mode; diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index 1103b3c289..35e3ac441c 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -12,8 +12,8 @@ Usage: nix run .#test-guest-cache -- --distro DISTRO [OPTIONS] Options: - --distro NAME Base distro: ubuntu, centos, fedora, or rocky - --with NAME Apply a configuration; repeatable (docker, podman, selinux) + --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, podman-rootless, selinux) --repository REF OCI repository without a tag --digest DIGEST Trusted OCI manifest digest required for pulls --cache-dir PATH Override the local prepared-disk cache directory diff --git a/nix/test-guest/configuration/podman-rootless.yml b/nix/test-guest/configuration/podman-rootless.yml new file mode 100644 index 0000000000..dc99934f74 --- /dev/null +++ b/nix/test-guest/configuration/podman-rootless.yml @@ -0,0 +1,89 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure rootless Podman in a disposable test guest. + +- name: Configure rootless Podman + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate rootless Podman support + ansible.builtin.assert: + that: + - ansible_facts.distribution == "Ubuntu" + - ansible_facts.distribution_version is version("26.04", ">=") + fail_msg: >- + Rootless Podman requires Ubuntu 26.04 or newer, not + {{ ansible_facts.distribution }} {{ ansible_facts.distribution_version }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + + - name: Install Ubuntu rootless Podman + ansible.builtin.apt: + name: + - apparmor + - fuse-overlayfs + - passt + - podman + - uidmap + state: present + + - name: Allow pasta to receive Podman stop signals + ansible.builtin.lineinfile: + path: /etc/apparmor.d/usr.bin.pasta + insertafter: "^ include $" + line: " signal (receive) peer=podman," + state: present + register: pasta_apparmor_profile + + - name: Reload pasta AppArmor profile + ansible.builtin.command: + argv: + - apparmor_parser + - --replace + - /etc/apparmor.d/usr.bin.pasta + when: pasta_apparmor_profile.changed + changed_when: pasta_apparmor_profile.changed + + - name: Enable the rootless Podman API socket + ansible.builtin.systemd_service: + name: podman.socket + scope: user + enabled: true + state: started + become: false + + - name: Verify rootless Podman + ansible.builtin.command: + cmd: podman info + become: false + changed_when: false + + - name: Verify rootless Podman mode + ansible.builtin.command: + argv: + - podman + - info + - --format + - "{% raw %}{{.Host.Security.Rootless}}{% endraw %}" + become: false + changed_when: false + register: podman_rootless + failed_when: podman_rootless.stdout != "true" + + - name: Verify rootless Podman uses pasta + ansible.builtin.command: + argv: + - podman + - info + - --format + - "{% raw %}{{.Host.RootlessNetworkCmd}}{% endraw %}" + become: false + changed_when: false + register: podman_rootless_network + failed_when: podman_rootless_network.stdout != "pasta" diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index 753f266ab4..b2249e1f4e 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -18,7 +18,8 @@ let if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; distros = { - ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + ubuntu-24-04 = import ./distros/ubuntu-24-04.nix { inherit pkgs architecture; }; + ubuntu-26-04 = import ./distros/ubuntu-26-04.nix { inherit pkgs architecture; }; centos = import ./distros/centos.nix { inherit pkgs architecture; }; fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; @@ -27,6 +28,7 @@ let configurations = { docker = ./configuration/docker.yml; podman = ./configuration/podman.yml; + podman-rootless = ./configuration/podman-rootless.yml; selinux = ./configuration/selinux.yml; snapd = ./configuration/snapd.yml; }; diff --git a/nix/test-guest/distros/ubuntu.nix b/nix/test-guest/distros/ubuntu-24-04.nix similarity index 100% rename from nix/test-guest/distros/ubuntu.nix rename to nix/test-guest/distros/ubuntu-24-04.nix diff --git a/nix/test-guest/distros/ubuntu-26-04.nix b/nix/test-guest/distros/ubuntu-26-04.nix new file mode 100644 index 0000000000..7f37fec138 --- /dev/null +++ b/nix/test-guest/distros/ubuntu-26-04.nix @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64"; + imageUrl = "https://cloud-images.ubuntu.com/releases/releases/26.04/release/ubuntu-26.04-server-cloudimg-${imageArchitecture}.img"; + imageHash = + if architecture == "aarch64" then + "sha256-PhE/3UHznhNyk3UXO7KueT+H3G20KU5SUf8kdpcXiLo=" + else + "sha256-gZa+nXlYBZy1bGx1yA/fbO6KiIW8FJ6nkdfbHH75MDU="; +in +{ + osId = "ubuntu"; + osVersion = "26.04"; + packageFamily = "deb"; + inherit imageUrl imageHash; + image = pkgs.fetchurl { + name = "ubuntu-26.04-server-cloudimg-${imageArchitecture}.img"; + url = imageUrl; + hash = imageHash; + }; +} diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 6be58eb235..9e5d19baef 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -12,8 +12,8 @@ Usage: nix run .#test-guest -- --distro DISTRO [OPTIONS] [-- COMMAND...] Options: - --distro NAME Base distro: ubuntu, centos, fedora, or rocky - --with NAME Apply a configuration; repeatable (docker, podman, selinux, snapd) + --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, podman-rootless, selinux, snapd) --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy a regular file to an absolute guest path, preserving its host mode; repeatable @@ -49,7 +49,7 @@ preserved_file_mode() { local source_mode if [ "$(uname -s)" = Darwin ]; then - if ! source_mode=$(stat -f '%Lp' "${source_path}"); then + if ! source_mode=$(/usr/bin/stat -f '%Lp' "${source_path}"); then echo "could not determine mode for --copy source: ${source_path}" >&2 return 1 fi diff --git a/tasks/test.toml b/tasks/test.toml index 29e06b826a..49e8df7f1a 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -136,6 +136,10 @@ run = [ "CONTAINER_RUNTIME=docker e2e/with-keycloak.sh env OPENSHELL_E2E_OIDC_GATEWAY=1 e2e/with-docker-gateway.sh uv run pytest -m 'not gpu' e2e/python/oidc", ] +["e2e:podman:rootless"] +description = "Run Rust Podman e2e inside a rootless Ubuntu 26.04 Nix test guest" +run = "e2e/run.sh --vm ubuntu-26-04 --with podman-rootless --tests-in-vm --gateway-config e2e/configs/gateway/podman.toml --features e2e-podman" + ["e2e:podman:gpu"] description = "Run GPU e2e against a standalone gateway with the Podman compute driver" env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" }