From b1301c879347e14cff7bdb113b7296dfdd85e520 Mon Sep 17 00:00:00 2001 From: tyler Date: Mon, 14 Sep 2026 21:03:44 -0600 Subject: [PATCH] feat: run subscription agents as private cloud workers --- .github/published-images.json | 1 + .github/workflows/aws-deploy.yml | 117 ++++ .github/workflows/ci.yml | 16 +- .github/workflows/publish-release.yml | 28 +- .github/workflows/subscription-gateway.yml | 60 ++ .gitignore | 1 + agent-subscription-gateway/Dockerfile | 40 ++ agent-subscription-gateway/bun.lock | 24 + agent-subscription-gateway/package.json | 14 + .../src/drivers/agent-driver.ts | 45 ++ .../src/drivers/claude/claude-driver.ts | 466 ++++++++++++++ .../src/drivers/codex/codex-driver.ts | 581 ++++++++++++++++++ .../src/drivers/create-driver.ts | 11 + .../src/drivers/grok/grok-driver.ts | 566 +++++++++++++++++ agent-subscription-gateway/src/main.ts | 79 +++ .../src/observability/run-counts.ts | 26 + .../src/observability/run-logger.ts | 15 + .../src/observability/status.ts | 28 + agent-subscription-gateway/src/runs/queue.ts | 272 ++++++++ .../src/runs/run-service.ts | 252 ++++++++ .../src/runs/session-input.ts | 84 +++ agent-subscription-gateway/src/server/app.ts | 99 +++ .../src/server/index.ts | 24 + .../src/server/input.ts | 37 ++ .../src/server/run-agent.ts | 205 ++++++ agent-subscription-gateway/src/server/sse.ts | 11 + .../src/storage/run-store.ts | 447 ++++++++++++++ .../src/workspaces/cleanup.ts | 18 + .../src/workspaces/workspace-manager.ts | 255 ++++++++ .../tests/ag-ui.test.ts | 183 ++++++ agent-subscription-gateway/tests/auth.test.ts | 56 ++ .../tests/claude-driver.test.ts | 355 +++++++++++ .../tests/codex-driver.test.ts | 258 ++++++++ .../claude-stream/permission-denied.jsonl | 2 + .../fixtures/claude-stream/rate-limit.jsonl | 3 + .../fixtures/claude-stream/success.jsonl | 6 + .../codex-app-server/fake-app-server.ts | 147 +++++ .../tests/fixtures/grok-acp/fake-agent.ts | 107 ++++ .../tests/grok-driver.test.ts | 262 ++++++++ .../tests/health.test.ts | 82 +++ .../tests/queue.test.ts | 174 ++++++ .../tests/recovery.test.ts | 84 +++ .../tests/sessions.test.ts | 232 +++++++ .../tests/workspaces.test.ts | 171 ++++++ agent-subscription-gateway/tsconfig.json | 7 + biome.json | 1 + deploy/aws/README.md | 19 + deploy/aws/bin/deploy-via-ssm | 142 +++++ deploy/aws/bin/ecr-login | 19 + deploy/aws/bin/install-host | 102 +++ deploy/aws/bin/publish-health-metrics | 53 ++ deploy/aws/bin/render-secret-env | 38 ++ deploy/aws/bin/worker-firewall | 8 + deploy/aws/caddy/Caddyfile | 11 + deploy/aws/claude-managed-settings.json | 57 ++ deploy/aws/codex-config.toml | 23 + deploy/aws/codex-requirements.toml | 5 + deploy/aws/compose.control.yml | 154 +++++ deploy/aws/compose.worker.yml | 154 +++++ deploy/aws/control-secret.example.json | 26 + deploy/aws/grok-requirements.toml | 64 ++ .../openbot-attachment-cleanup.service | 9 + .../systemd/openbot-attachment-cleanup.timer | 11 + .../aws/systemd/openbot-control-env.service | 12 + deploy/aws/systemd/openbot-control.service | 21 + .../systemd/openbot-database-backup.service | 9 + .../aws/systemd/openbot-database-backup.timer | 11 + .../systemd/openbot-health-metrics.service | 9 + .../aws/systemd/openbot-health-metrics.timer | 11 + deploy/aws/systemd/openbot-routines.service | 9 + deploy/aws/systemd/openbot-routines.timer | 11 + deploy/aws/systemd/openbot-worker-env.service | 12 + .../systemd/openbot-worker-firewall.service | 13 + deploy/aws/systemd/openbot-worker.service | 19 + deploy/aws/worker-secret.example.json | 11 + docs/configuration.md | 34 +- docs/coworkers.md | 29 + docs/deployment.md | 2 + docs/runbooks/aws-deployment.md | 77 +++ docs/runbooks/provider-sign-in.md | 129 ++++ docs/runbooks/worker-operations.md | 49 ++ examples/fintech/agents.yaml | 27 + .../environments/personal/.terraform.lock.hcl | 25 + infra/aws/environments/personal/main.tf | 103 ++++ infra/aws/environments/personal/operations.tf | 251 ++++++++ infra/aws/environments/personal/outputs.tf | 46 ++ .../personal/terraform.tfvars.example | 13 + infra/aws/environments/personal/variables.tf | 83 +++ infra/aws/environments/personal/versions.tf | 10 + .../modules/control-host/bootstrap.sh.tftpl | 26 + infra/aws/modules/control-host/main.tf | 145 +++++ infra/aws/modules/control-host/outputs.tf | 19 + infra/aws/modules/control-host/variables.tf | 45 ++ infra/aws/modules/network/main.tf | 106 ++++ infra/aws/modules/network/outputs.tf | 27 + infra/aws/modules/network/variables.tf | 25 + infra/aws/modules/registry/main.tf | 34 + infra/aws/modules/registry/outputs.tf | 7 + infra/aws/modules/registry/variables.tf | 10 + .../modules/worker-host/bootstrap.sh.tftpl | 37 ++ infra/aws/modules/worker-host/main.tf | 181 ++++++ infra/aws/modules/worker-host/outputs.tf | 23 + infra/aws/modules/worker-host/variables.tf | 50 ++ infra/aws/versions.tf | 10 + package.json | 6 +- server/src/agents/runtime-agents.ts | 25 +- server/src/auth/index.ts | 34 + server/src/config.ts | 69 ++- server/tests/agent-endpoint.test.ts | 22 + server/tests/auth-owner-allowlist.test.ts | 78 +++ server/tests/config.test.ts | 99 +++ server/tests/picked-harness-auth.test.ts | 56 +- server/tests/support/environment.ts | 1 + server/tests/tenant-package.test.ts | 44 ++ tests/aws-compose.test.ts | 250 ++++++++ tests/aws-infra.test.ts | 141 +++++ tests/release-config.test.ts | 48 ++ tests/runbook-contract.test.ts | 51 ++ 118 files changed, 9499 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/aws-deploy.yml create mode 100644 .github/workflows/subscription-gateway.yml create mode 100644 agent-subscription-gateway/Dockerfile create mode 100644 agent-subscription-gateway/bun.lock create mode 100644 agent-subscription-gateway/package.json create mode 100644 agent-subscription-gateway/src/drivers/agent-driver.ts create mode 100644 agent-subscription-gateway/src/drivers/claude/claude-driver.ts create mode 100644 agent-subscription-gateway/src/drivers/codex/codex-driver.ts create mode 100644 agent-subscription-gateway/src/drivers/create-driver.ts create mode 100644 agent-subscription-gateway/src/drivers/grok/grok-driver.ts create mode 100644 agent-subscription-gateway/src/main.ts create mode 100644 agent-subscription-gateway/src/observability/run-counts.ts create mode 100644 agent-subscription-gateway/src/observability/run-logger.ts create mode 100644 agent-subscription-gateway/src/observability/status.ts create mode 100644 agent-subscription-gateway/src/runs/queue.ts create mode 100644 agent-subscription-gateway/src/runs/run-service.ts create mode 100644 agent-subscription-gateway/src/runs/session-input.ts create mode 100644 agent-subscription-gateway/src/server/app.ts create mode 100644 agent-subscription-gateway/src/server/index.ts create mode 100644 agent-subscription-gateway/src/server/input.ts create mode 100644 agent-subscription-gateway/src/server/run-agent.ts create mode 100644 agent-subscription-gateway/src/server/sse.ts create mode 100644 agent-subscription-gateway/src/storage/run-store.ts create mode 100644 agent-subscription-gateway/src/workspaces/cleanup.ts create mode 100644 agent-subscription-gateway/src/workspaces/workspace-manager.ts create mode 100644 agent-subscription-gateway/tests/ag-ui.test.ts create mode 100644 agent-subscription-gateway/tests/auth.test.ts create mode 100644 agent-subscription-gateway/tests/claude-driver.test.ts create mode 100644 agent-subscription-gateway/tests/codex-driver.test.ts create mode 100644 agent-subscription-gateway/tests/fixtures/claude-stream/permission-denied.jsonl create mode 100644 agent-subscription-gateway/tests/fixtures/claude-stream/rate-limit.jsonl create mode 100644 agent-subscription-gateway/tests/fixtures/claude-stream/success.jsonl create mode 100644 agent-subscription-gateway/tests/fixtures/codex-app-server/fake-app-server.ts create mode 100644 agent-subscription-gateway/tests/fixtures/grok-acp/fake-agent.ts create mode 100644 agent-subscription-gateway/tests/grok-driver.test.ts create mode 100644 agent-subscription-gateway/tests/health.test.ts create mode 100644 agent-subscription-gateway/tests/queue.test.ts create mode 100644 agent-subscription-gateway/tests/recovery.test.ts create mode 100644 agent-subscription-gateway/tests/sessions.test.ts create mode 100644 agent-subscription-gateway/tests/workspaces.test.ts create mode 100644 agent-subscription-gateway/tsconfig.json create mode 100644 deploy/aws/README.md create mode 100755 deploy/aws/bin/deploy-via-ssm create mode 100755 deploy/aws/bin/ecr-login create mode 100755 deploy/aws/bin/install-host create mode 100755 deploy/aws/bin/publish-health-metrics create mode 100755 deploy/aws/bin/render-secret-env create mode 100755 deploy/aws/bin/worker-firewall create mode 100644 deploy/aws/caddy/Caddyfile create mode 100644 deploy/aws/claude-managed-settings.json create mode 100644 deploy/aws/codex-config.toml create mode 100644 deploy/aws/codex-requirements.toml create mode 100644 deploy/aws/compose.control.yml create mode 100644 deploy/aws/compose.worker.yml create mode 100644 deploy/aws/control-secret.example.json create mode 100644 deploy/aws/grok-requirements.toml create mode 100644 deploy/aws/systemd/openbot-attachment-cleanup.service create mode 100644 deploy/aws/systemd/openbot-attachment-cleanup.timer create mode 100644 deploy/aws/systemd/openbot-control-env.service create mode 100644 deploy/aws/systemd/openbot-control.service create mode 100644 deploy/aws/systemd/openbot-database-backup.service create mode 100644 deploy/aws/systemd/openbot-database-backup.timer create mode 100644 deploy/aws/systemd/openbot-health-metrics.service create mode 100644 deploy/aws/systemd/openbot-health-metrics.timer create mode 100644 deploy/aws/systemd/openbot-routines.service create mode 100644 deploy/aws/systemd/openbot-routines.timer create mode 100644 deploy/aws/systemd/openbot-worker-env.service create mode 100644 deploy/aws/systemd/openbot-worker-firewall.service create mode 100644 deploy/aws/systemd/openbot-worker.service create mode 100644 deploy/aws/worker-secret.example.json create mode 100644 docs/runbooks/aws-deployment.md create mode 100644 docs/runbooks/provider-sign-in.md create mode 100644 docs/runbooks/worker-operations.md create mode 100644 infra/aws/environments/personal/.terraform.lock.hcl create mode 100644 infra/aws/environments/personal/main.tf create mode 100644 infra/aws/environments/personal/operations.tf create mode 100644 infra/aws/environments/personal/outputs.tf create mode 100644 infra/aws/environments/personal/terraform.tfvars.example create mode 100644 infra/aws/environments/personal/variables.tf create mode 100644 infra/aws/environments/personal/versions.tf create mode 100644 infra/aws/modules/control-host/bootstrap.sh.tftpl create mode 100644 infra/aws/modules/control-host/main.tf create mode 100644 infra/aws/modules/control-host/outputs.tf create mode 100644 infra/aws/modules/control-host/variables.tf create mode 100644 infra/aws/modules/network/main.tf create mode 100644 infra/aws/modules/network/outputs.tf create mode 100644 infra/aws/modules/network/variables.tf create mode 100644 infra/aws/modules/registry/main.tf create mode 100644 infra/aws/modules/registry/outputs.tf create mode 100644 infra/aws/modules/registry/variables.tf create mode 100644 infra/aws/modules/worker-host/bootstrap.sh.tftpl create mode 100644 infra/aws/modules/worker-host/main.tf create mode 100644 infra/aws/modules/worker-host/outputs.tf create mode 100644 infra/aws/modules/worker-host/variables.tf create mode 100644 infra/aws/versions.tf create mode 100644 server/tests/auth-owner-allowlist.test.ts create mode 100644 tests/aws-compose.test.ts create mode 100644 tests/aws-infra.test.ts create mode 100644 tests/release-config.test.ts create mode 100644 tests/runbook-contract.test.ts diff --git a/.github/published-images.json b/.github/published-images.json index d460e1f33..bfbdf7134 100644 --- a/.github/published-images.json +++ b/.github/published-images.json @@ -1,5 +1,6 @@ [ "agent-computer", + "agent-subscription-gateway", "supervisor", "agent-bot", "agent-langgraph", diff --git a/.github/workflows/aws-deploy.yml b/.github/workflows/aws-deploy.yml new file mode 100644 index 000000000..88e5e6566 --- /dev/null +++ b/.github/workflows/aws-deploy.yml @@ -0,0 +1,117 @@ +name: Deploy personal AWS stack + +on: + workflow_dispatch: + inputs: + target: + description: Hosts to update + required: true + default: all + type: choice + options: [all, worker, control] + +permissions: + contents: read + id-token: write + +concurrency: + group: openbot-personal-aws + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: production + env: + AWS_REGION: ${{ vars.AWS_REGION }} + CONTROL_REPOSITORY: ${{ vars.AWS_OPENBOT_ECR_REPOSITORY_URL }} + GATEWAY_REPOSITORY: ${{ vars.AWS_GATEWAY_ECR_REPOSITORY_URL }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check protected deployment settings + env: + ROLE_ARN: ${{ vars.AWS_ROLE_ARN }} + CONTROL_INSTANCE: ${{ vars.AWS_CONTROL_INSTANCE_ID }} + WORKER_INSTANCE: ${{ vars.AWS_WORKER_INSTANCE_ID }} + CONTROL_SECRET: ${{ vars.AWS_CONTROL_SECRET_ARN }} + WORKER_SECRET: ${{ vars.AWS_WORKER_SECRET_ARN }} + run: | + set -euo pipefail + for name in AWS_REGION ROLE_ARN CONTROL_REPOSITORY GATEWAY_REPOSITORY CONTROL_INSTANCE WORKER_INSTANCE CONTROL_SECRET WORKER_SECRET; do + test -n "${!name:-}" || { echo "::error::$name is not set in the production environment"; exit 1; } + done + test "${CONTROL_REPOSITORY%%/*}" = "${GATEWAY_REPOSITORY%%/*}" + - uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 + with: + role-to-assume: ${{ vars.AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + - uses: aws-actions/amazon-ecr-login@4625ce35226a7557230889aae2f52eb50ec3dcda # v2.0.1 + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - id: control-image + name: Build and push OpenBot + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ env.CONTROL_REPOSITORY }}:${{ github.sha }} + cache-from: type=gha,scope=aws-openbot-amd64 + cache-to: type=gha,mode=max,scope=aws-openbot-amd64 + provenance: true + sbom: true + - id: gateway-image + name: Build and push subscription gateway + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: agent-subscription-gateway/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ env.GATEWAY_REPOSITORY }}:${{ github.sha }} + cache-from: type=gha,scope=aws-gateway-amd64 + cache-to: type=gha,mode=max,scope=aws-gateway-amd64 + provenance: true + sbom: true + - uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 + - name: Sign and scan immutable images + env: + CONTROL_DIGEST: ${{ steps.control-image.outputs.digest }} + GATEWAY_DIGEST: ${{ steps.gateway-image.outputs.digest }} + run: | + set -euo pipefail + for ref in "$CONTROL_REPOSITORY@$CONTROL_DIGEST" "$GATEWAY_REPOSITORY@$GATEWAY_DIGEST"; do + cosign sign --yes "$ref" + docker run --rm \ + -v "$HOME/.docker:/root/.docker:ro" \ + aquasec/trivy@sha256:e2b22eac59c02003d8749f5b8d9bd073b62e30fefaef5b7c8371204e0a4b0c08 \ + image --exit-code 1 --ignore-unfixed --severity CRITICAL "$ref" + done + - name: Pack the host files + run: tar -czf "$RUNNER_TEMP/openbot-aws-deploy.tgz" deploy/aws + - name: Deploy worker first + if: inputs.target == 'all' || inputs.target == 'worker' + env: + DEPLOY_BUNDLE: ${{ runner.temp }}/openbot-aws-deploy.tgz + WORKER_INSTANCE: ${{ vars.AWS_WORKER_INSTANCE_ID }} + WORKER_SECRET: ${{ vars.AWS_WORKER_SECRET_ARN }} + GATEWAY_DIGEST: ${{ steps.gateway-image.outputs.digest }} + run: | + deploy/aws/bin/deploy-via-ssm \ + worker "$WORKER_INSTANCE" "$WORKER_SECRET" "$AWS_REGION" \ + "${GATEWAY_REPOSITORY%%/*}" GATEWAY_IMAGE \ + "$GATEWAY_REPOSITORY@$GATEWAY_DIGEST" + - name: Deploy control + if: inputs.target == 'all' || inputs.target == 'control' + env: + DEPLOY_BUNDLE: ${{ runner.temp }}/openbot-aws-deploy.tgz + CONTROL_INSTANCE: ${{ vars.AWS_CONTROL_INSTANCE_ID }} + CONTROL_SECRET: ${{ vars.AWS_CONTROL_SECRET_ARN }} + CONTROL_DIGEST: ${{ steps.control-image.outputs.digest }} + run: | + deploy/aws/bin/deploy-via-ssm \ + control "$CONTROL_INSTANCE" "$CONTROL_SECRET" "$AWS_REGION" \ + "${CONTROL_REPOSITORY%%/*}" OPENBOT_IMAGE \ + "$CONTROL_REPOSITORY@$CONTROL_DIGEST" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed5bcf759..09cbe539f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,11 +50,12 @@ jobs: exit 1 fi - # The two packages that are deployables in their own right rather than root workspaces. Root + # The three packages that are deployables in their own right rather than root workspaces. Root # `typecheck` is `bun run --filter '*' typecheck`, and `--filter '*'` enumerates `workspaces`, which # is app, server and worker. So both of these ship a `typecheck` script that nothing has ever run: # `agent-computer` holds the only `spawn` in the deployment and `supervisor` is the only thing - # holding a Docker socket, which is a poor pair to leave untyped. Both are clean today, so this + # holding a Docker socket, and the subscription gateway holds provider processes. They are clean + # today, so this # changes nothing about main and only stops the next change being the first one a type checker sees. # # Not by adding them to `workspaces`: each is built from its own lockfile and the Dockerfile depends @@ -68,7 +69,7 @@ jobs: # One red package must not hide whether the other is red too. fail-fast: false matrix: - package: [agent-computer, supervisor] + package: [agent-computer, supervisor, agent-subscription-gateway] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -86,6 +87,9 @@ jobs: working-directory: ${{ matrix.package }} - run: bun run typecheck working-directory: ${{ matrix.package }} + - run: bun run test + if: matrix.package == 'agent-subscription-gateway' + working-directory: ${{ matrix.package }} chart: name: chart (${{ matrix.target }}) @@ -240,7 +244,7 @@ jobs: working-directory: desktop # Not the db:migrate script: that one loads ../.env, which does not exist in CI. DATABASE_URL # comes from the job env instead, which drizzle.config.ts already reads. - - run: bunx drizzle-kit migrate --config=drizzle.config.ts + - run: bun x drizzle-kit migrate --config=drizzle.config.ts working-directory: server # A passing job must include the expected test floor. Import-time failures can otherwise skip # files before their tests are registered. @@ -317,7 +321,7 @@ jobs: bun-version: 1.3.14 - run: bun install --frozen-lockfile # Collisions and gaps between the migration files themselves. - - run: bunx drizzle-kit check --config=drizzle.config.ts + - run: bun x drizzle-kit check --config=drizzle.config.ts working-directory: server # And the other direction: a schema change with no migration written for it. `generate` writes a # file when it finds one, so the tree being dirty afterwards is the failure. @@ -325,7 +329,7 @@ jobs: working-directory: server run: | set -euo pipefail - bunx drizzle-kit generate --config=drizzle.config.ts --name=ci_drift_probe + bun x drizzle-kit generate --config=drizzle.config.ts --name=ci_drift_probe if [ -n "$(git status --porcelain drizzle)" ]; then echo "::error::The schema has changed without a migration. Run drizzle-kit generate and commit it." git status --porcelain drizzle diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index df0057db1..0b0d9e8b5 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -79,6 +79,7 @@ jobs: runs-on: ubuntu-latest outputs: components: ${{ steps.components.outputs.components }} + image_prefix: ${{ steps.registry.outputs.image_prefix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -101,13 +102,16 @@ jobs: # set that is tested and the set that is published cannot drift apart. - id: components run: echo "components=$(jq -c . .github/published-images.json)" >> "$GITHUB_OUTPUT" + - id: registry + name: Select this fork's package namespace + run: echo "image_prefix=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/openbot" >> "$GITHUB_OUTPUT" # The same checks CI runs, against the commit being published. This is the gate: nothing is built # or tagged unless they pass here, on this exact tree. checks: needs: [metadata, verify] if: needs.metadata.outputs.is_release == 'true' - uses: $/.github/workflows/ci.yml + uses: ./.github/workflows/ci.yml permissions: contents: read @@ -146,20 +150,20 @@ jobs: # The version, the commit, and a moving latest. The version tag is the one to deploy; the # commit tag is how you find out what a running image actually contains. tags: | - ghcr.io/copilotkit/openbot:${{ needs.metadata.outputs.version }} - ghcr.io/copilotkit/openbot:${{ github.sha }} - ghcr.io/copilotkit/openbot:latest + ${{ needs.verify.outputs.image_prefix }}:${{ needs.metadata.outputs.version }} + ${{ needs.verify.outputs.image_prefix }}:${{ github.sha }} + ${{ needs.verify.outputs.image_prefix }}:latest cache-from: type=gha cache-to: type=gha,mode=max provenance: true sbom: true # BuildKit's own attestations above travel inside the image. This one is the record GitHub - # holds, and it is what `gh attestation verify oci://ghcr.io/copilotkit/openbot:vX.Y.Z - # -R CopilotKit/OpenBot` checks before anybody deploys it. Bound to the digest, never a tag, + # holds, and it is what `gh attestation verify oci:///openbot:vX.Y.Z` checks before + # anybody deploys it. Bound to the digest, never a tag, # because a tag can be moved to point at something else afterwards. - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - subject-name: ghcr.io/copilotkit/openbot + subject-name: ${{ needs.verify.outputs.image_prefix }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true @@ -226,7 +230,7 @@ jobs: # `oci-mediatypes=true` because zstd layers have no Docker-schema2 media type to be # described by. It is buildx's default for this exporter and is stated rather than # assumed, since the push fails without it and the reason would not be obvious. - outputs: type=image,name=ghcr.io/copilotkit/openbot-${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true,compression=zstd,force-compression=true + outputs: type=image,name=${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true,compression=zstd,force-compression=true # Scoped per image and per architecture. One shared scope would have ten builds # overwriting each other's cache and none of them reading their own. cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.platform.arch }} @@ -292,7 +296,7 @@ jobs: - id: merge name: Write the tags over both architectures env: - REPOSITORY: ghcr.io/copilotkit/openbot-${{ matrix.image }} + REPOSITORY: ${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }} VERSION: ${{ needs.metadata.outputs.version }} COMMIT: ${{ github.sha }} run: | @@ -319,7 +323,7 @@ jobs: echo "digest=$digest" >> "$GITHUB_OUTPUT" - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - subject-name: ghcr.io/copilotkit/openbot-${{ matrix.image }} + subject-name: ${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }} subject-digest: ${{ steps.merge.outputs.digest }} push-to-registry: true # For the same reason the digests above travel as files: a matrix cannot hand a value to a @@ -327,7 +331,7 @@ jobs: - name: Record the manifest env: NAME: ${{ matrix.image }} - REPOSITORY: ghcr.io/copilotkit/openbot-${{ matrix.image }} + REPOSITORY: ${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }} DIGEST: ${{ steps.merge.outputs.digest }} run: | set -euo pipefail @@ -396,7 +400,7 @@ jobs: --arg version "$VERSION" \ --arg digest "$DIGEST" \ --arg commit "$COMMIT" \ - --arg repository "ghcr.io/copilotkit/openbot" \ + --arg repository "${{ needs.verify.outputs.image_prefix }}" \ --argjson components "$components" \ '{ version: $version, diff --git a/.github/workflows/subscription-gateway.yml b/.github/workflows/subscription-gateway.yml new file mode 100644 index 000000000..098e32a3d --- /dev/null +++ b/.github/workflows/subscription-gateway.yml @@ -0,0 +1,60 @@ +name: Subscription gateway + +on: + pull_request: + paths: + - agent-subscription-gateway/** + - deploy/aws/** + - shared/** + - .github/workflows/subscription-gateway.yml + push: + branches: [main] + paths: + - agent-subscription-gateway/** + - deploy/aws/** + - shared/** + - .github/workflows/subscription-gateway.yml + workflow_call: + +permissions: + contents: read + +concurrency: + group: subscription-gateway-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.14 + - run: bun install --frozen-lockfile + working-directory: agent-subscription-gateway + - run: bun run typecheck + working-directory: agent-subscription-gateway + - run: bun run test + working-directory: agent-subscription-gateway + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - name: Build the AWS architecture + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: agent-subscription-gateway/Dockerfile + platforms: linux/amd64 + load: true + tags: openbot-subscription-gateway:ci + cache-from: type=gha,scope=subscription-gateway-amd64 + cache-to: type=gha,mode=max,scope=subscription-gateway-amd64 + - name: Scan the image + run: >- + docker run --rm + -v /var/run/docker.sock:/var/run/docker.sock + aquasec/trivy@sha256:e2b22eac59c02003d8749f5b8d9bd073b62e30fefaef5b7c8371204e0a4b0c08 + image --exit-code 1 --ignore-unfixed --severity CRITICAL + openbot-subscription-gateway:ci diff --git a/.gitignore b/.gitignore index d6aa06ad1..4497f5a25 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ egress.env # named node_modules is not a directory — so a sandbox that links its dependencies at the repo root # leaves three links git will happily commit. That happened. node_modules +.terraform/ **/dist/ app/src/lib/generated/application-config.ts .logs/ diff --git a/agent-subscription-gateway/Dockerfile b/agent-subscription-gateway/Dockerfile new file mode 100644 index 000000000..f9f7c8542 --- /dev/null +++ b/agent-subscription-gateway/Dockerfile @@ -0,0 +1,40 @@ +FROM node:24.18.1-bookworm-slim@sha256:235600a8101ab264e117b1768e925532262668dc9b581ef1dd7d96ced463b8e7 AS node-toolchain +FROM oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS bun-toolchain + +FROM node-toolchain AS runtime + +ARG CODEX_VERSION=0.154.0 +ARG CLAUDE_VERSION=2.1.271 +ARG GROK_VERSION=1.0.30 +ARG NPM_VERSION=12.0.2 +ENV CODEX_VERSION=${CODEX_VERSION} +ENV CLAUDE_VERSION=${CLAUDE_VERSION} +ENV GROK_VERSION=${GROK_VERSION} +ENV HOME=/home/gateway + +COPY --from=bun-toolchain /usr/local/bin/bun /usr/local/bin/bun +RUN ln -s bun /usr/local/bin/bunx \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends ca-certificates git bubblewrap socat \ + && npm install --global "npm@${NPM_VERSION}" \ + && npm install --global --allow-scripts=@anthropic-ai/claude-code,@xai-official/grok "@openai/codex@${CODEX_VERSION}" "@anthropic-ai/claude-code@${CLAUDE_VERSION}" "@xai-official/grok@${GROK_VERSION}" \ + && npm cache clean --force \ + && rm -rf /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 10001 --shell /usr/sbin/nologin gateway \ + && mkdir -p /app /workspaces /var/lib/openbot-gateway /home/gateway/.codex /home/gateway/.grok /etc/codex /etc/claude-code /etc/grok \ + && chmod 0755 /etc/codex /etc/claude-code /etc/grok \ + && chown -R 10001:10001 /app /workspaces /var/lib/openbot-gateway /home/gateway + +WORKDIR /app +COPY --chown=10001:10001 agent-subscription-gateway/package.json agent-subscription-gateway/package.json +COPY --chown=10001:10001 agent-subscription-gateway/src agent-subscription-gateway/src +COPY --chown=10001:10001 shared shared +COPY --chmod=0444 deploy/aws/codex-config.toml /etc/codex/config.toml +COPY --chmod=0444 deploy/aws/codex-requirements.toml /etc/codex/requirements.toml +COPY --chmod=0444 deploy/aws/claude-managed-settings.json /etc/claude-code/managed-settings.json +COPY --chmod=0444 deploy/aws/grok-requirements.toml /etc/grok/requirements.toml + +USER 10001:10001 +CMD ["bun", "agent-subscription-gateway/src/main.ts"] diff --git a/agent-subscription-gateway/bun.lock b/agent-subscription-gateway/bun.lock new file mode 100644 index 000000000..f244428c6 --- /dev/null +++ b/agent-subscription-gateway/bun.lock @@ -0,0 +1,24 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@openbot/agent-subscription-gateway", + "devDependencies": { + "@types/bun": "1.3.11", + "typescript": "5.9.3", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], + + "@types/node": ["@types/node@26.5.1", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-CzNm2FezW4VR/LjG6yUdiEgLE/rAQ9Slj5gCu/C2VrdcW7I0ahNZ8DRbHT7zOZ6r3ONgd/bsQIeSaoDGrd1C6g=="], + + "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + } +} diff --git a/agent-subscription-gateway/package.json b/agent-subscription-gateway/package.json new file mode 100644 index 000000000..6a8e4ea1d --- /dev/null +++ b/agent-subscription-gateway/package.json @@ -0,0 +1,14 @@ +{ + "name": "@openbot/agent-subscription-gateway", + "license": "MIT", + "private": true, + "type": "module", + "scripts": { + "test": "bun test tests", + "typecheck": "bun x tsc --noEmit" + }, + "devDependencies": { + "@types/bun": "1.3.11", + "typescript": "5.9.3" + } +} diff --git a/agent-subscription-gateway/src/drivers/agent-driver.ts b/agent-subscription-gateway/src/drivers/agent-driver.ts new file mode 100644 index 000000000..9ec20a0bd --- /dev/null +++ b/agent-subscription-gateway/src/drivers/agent-driver.ts @@ -0,0 +1,45 @@ +export interface DriverRun { + threadId: string; + runId: string; + messages: readonly unknown[]; + context: readonly unknown[]; + state: Record; + forwardedProps: Record; + workspacePath?: string; +} + +export interface DriverResumeRun extends DriverRun { + sessionId: string; +} + +export interface DriverRunContext { + signal: AbortSignal; +} + +export type DriverEvent = + | { type: "text"; delta: string } + | { type: "session"; sessionId: string } + | { + type: "activity"; + message: string; + data?: Record; + }; + +export interface AgentDriver { + readonly provider: string; + readonly version: string; + isAuthReady(): Promise; + start(run: DriverRun, context: DriverRunContext): AsyncIterable; + resume( + run: DriverResumeRun, + context: DriverRunContext, + ): AsyncIterable; + cancel(runId: string): Promise; +} + +export class SafeDriverError extends Error { + constructor(readonly publicMessage: string) { + super(publicMessage); + this.name = "SafeDriverError"; + } +} diff --git a/agent-subscription-gateway/src/drivers/claude/claude-driver.ts b/agent-subscription-gateway/src/drivers/claude/claude-driver.ts new file mode 100644 index 000000000..c615bad73 --- /dev/null +++ b/agent-subscription-gateway/src/drivers/claude/claude-driver.ts @@ -0,0 +1,466 @@ +import { isAbsolute } from "node:path"; +import { constants } from "node:os"; +import { + type AgentDriver, + type DriverEvent, + type DriverResumeRun, + type DriverRun, + type DriverRunContext, + SafeDriverError, +} from "../agent-driver"; + +interface JsonObject { + [key: string]: unknown; +} + +export interface ClaudeChildProcess { + stdout: ReadableStream; + stderr: ReadableStream; + exited: Promise; + kill(signal?: number): unknown; +} + +export interface ClaudeSpawnOptions { + command: string[]; + cwd: string; + env: Record; +} + +export type ClaudeProcessFactory = ( + options: ClaudeSpawnOptions, +) => ClaudeChildProcess; + +interface ActiveRun { + child: ClaudeChildProcess; + cancelled: boolean; + stopPromise?: Promise; +} + +const SAFE_ENVIRONMENT_NAMES = new Set([ + "PATH", + "HOME", + "LANG", + "LANGUAGE", + "LC_ALL", + "TZ", + "TMPDIR", + "TMP", + "TEMP", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "COLORTERM", + "CLAUDE_CONFIG_DIR", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE", + "SSL_CERT_DIR", +]); + +function isObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function safeEnvironment( + source: Record, +): Record { + const environment: Record = {}; + for (const [name, value] of Object.entries(source)) { + if (value !== undefined && SAFE_ENVIRONMENT_NAMES.has(name)) { + environment[name] = value; + } + } + environment.CLAUDE_CODE_DISABLE_AUTO_UPDATER = "1"; + environment.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB = "1"; + return environment; +} + +async function* jsonLines( + stream: ReadableStream, +): AsyncGenerator { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let buffered = ""; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + buffered += decoder.decode(chunk.value, { stream: true }); + while (true) { + const newline = buffered.indexOf("\n"); + if (newline < 0) break; + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + if (line) { + const value: unknown = JSON.parse(line); + if (!isObject(value)) throw new Error("Invalid stream record."); + yield value; + } + } + } + buffered += decoder.decode(); + const finalLine = buffered.trim(); + if (finalLine) { + const value: unknown = JSON.parse(finalLine); + if (!isObject(value)) throw new Error("Invalid stream record."); + yield value; + } + } finally { + reader.releaseLock(); + } +} + +async function drain(stream: ReadableStream): Promise { + try { + for await (const _chunk of stream) { + } + } catch {} +} + +function defaultProcessFactory( + options: ClaudeSpawnOptions, +): ClaudeChildProcess { + const child = Bun.spawn(options.command, { + cwd: options.cwd, + env: options.env, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + return { + stdout: child.stdout, + stderr: child.stderr, + exited: child.exited, + kill: (signal) => child.kill(signal), + }; +} + +function requiredWorkspace(run: DriverRun): string { + if (!run.workspacePath || !isAbsolute(run.workspacePath)) { + throw new SafeDriverError("The Claude workspace is not available."); + } + return run.workspacePath; +} + +function transcript(run: DriverRun): string { + const parts: string[] = []; + for (const raw of run.messages) { + if (!isObject(raw)) continue; + const { id, role, content } = raw; + if ( + typeof id !== "string" || + typeof role !== "string" || + typeof content !== "string" + ) { + continue; + } + parts.push(`[${role} ${id}]\n${content}`); + } + if (parts.length === 0) { + throw new SafeDriverError("The Claude run has no transcript input."); + } + return `OpenBot transcript:\n\n${parts.join("\n\n")}`; +} + +function toolName(value: unknown): string { + if (typeof value !== "string") return "Native tool"; + const known = new Set([ + "Agent", + "Bash", + "Edit", + "Glob", + "Grep", + "NotebookEdit", + "Read", + "Task", + "TodoWrite", + "WebFetch", + "WebSearch", + "Write", + ]); + return known.has(value) ? value : "Native tool"; +} + +function assistantActivity(record: JsonObject): DriverEvent | undefined { + const message = isObject(record.message) ? record.message : undefined; + const content = + message && Array.isArray(message.content) ? message.content : []; + const use = content.find( + (item): item is JsonObject => isObject(item) && item.type === "tool_use", + ); + if (!use) return undefined; + const name = toolName(use.name); + return { + type: "activity", + message: `${name} started`, + data: { nativeTool: name }, + }; +} + +function toolResultActivity(record: JsonObject): DriverEvent | undefined { + const message = isObject(record.message) ? record.message : undefined; + const content = + message && Array.isArray(message.content) ? message.content : []; + const result = content.find( + (item): item is JsonObject => isObject(item) && item.type === "tool_result", + ); + if (!result) return undefined; + return { + type: "activity", + message: + result.is_error === true ? "Native tool failed" : "Native tool completed", + }; +} + +async function exitedWithin( + child: ClaudeChildProcess, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }); + const exited = child.exited.then( + () => true, + () => true, + ); + const result = await Promise.race([exited, timedOut]); + if (timer) clearTimeout(timer); + return result; +} + +export interface ClaudeDriverOptions { + processFactory?: ClaudeProcessFactory; + binary?: string; + version?: string; + environment?: Record; + interruptTimeoutMs?: number; + terminateTimeoutMs?: number; + authTimeoutMs?: number; +} + +export class ClaudeDriver implements AgentDriver { + readonly provider = "claude"; + readonly version: string; + private readonly processFactory: ClaudeProcessFactory; + private readonly binary: string; + private readonly environment: Record; + private readonly interruptTimeoutMs: number; + private readonly terminateTimeoutMs: number; + private readonly authTimeoutMs: number; + private readonly active = new Map(); + + constructor(options: ClaudeDriverOptions = {}) { + this.processFactory = options.processFactory ?? defaultProcessFactory; + this.binary = options.binary ?? "claude"; + this.version = options.version ?? process.env.CLAUDE_VERSION ?? "unknown"; + this.environment = safeEnvironment(options.environment ?? process.env); + this.interruptTimeoutMs = options.interruptTimeoutMs ?? 3_000; + this.terminateTimeoutMs = options.terminateTimeoutMs ?? 2_000; + this.authTimeoutMs = options.authTimeoutMs ?? 5_000; + } + + async isAuthReady(): Promise { + let child: ClaudeChildProcess; + try { + child = this.processFactory({ + command: [this.binary, "auth", "status"], + cwd: process.cwd(), + env: this.environment, + }); + } catch { + return false; + } + void drain(child.stdout); + void drain(child.stderr); + if (!(await exitedWithin(child, this.authTimeoutMs))) { + try { + child.kill(constants.signals.SIGKILL); + } catch {} + return false; + } + return (await child.exited) === 0; + } + + start(run: DriverRun, context: DriverRunContext): AsyncIterable { + return this.execute(run, context); + } + + resume( + run: DriverResumeRun, + context: DriverRunContext, + ): AsyncIterable { + return this.execute(run, context, run.sessionId); + } + + async cancel(runId: string): Promise { + const active = this.active.get(runId); + if (!active) return; + active.cancelled = true; + await this.stopProcess(active); + } + + private stopProcess(active: ActiveRun): Promise { + if (active.stopPromise) return active.stopPromise; + active.stopPromise = (async () => { + try { + active.child.kill(constants.signals.SIGINT); + } catch { + return; + } + if (await exitedWithin(active.child, this.interruptTimeoutMs)) return; + try { + active.child.kill(constants.signals.SIGTERM); + } catch { + return; + } + if (await exitedWithin(active.child, this.terminateTimeoutMs)) return; + try { + active.child.kill(constants.signals.SIGKILL); + } catch {} + await exitedWithin(active.child, this.terminateTimeoutMs); + })(); + return active.stopPromise; + } + + private async *execute( + run: DriverRun, + context: DriverRunContext, + sessionId?: string, + ): AsyncGenerator { + const workspace = requiredWorkspace(run); + if (this.active.has(run.runId)) { + throw new SafeDriverError("The Claude run is already active."); + } + const command = [ + this.binary, + "-p", + transcript(run), + "--output-format", + "stream-json", + "--verbose", + "--include-partial-messages", + "--permission-mode", + "dontAsk", + ...(sessionId ? ["--resume", sessionId] : []), + ]; + let child: ClaudeChildProcess; + try { + child = this.processFactory({ + command, + cwd: workspace, + env: this.environment, + }); + } catch { + throw new SafeDriverError("The Claude CLI could not start."); + } + const active: ActiveRun = { child, cancelled: false }; + this.active.set(run.runId, active); + void drain(child.stderr); + const onAbort = () => void this.cancel(run.runId); + context.signal.addEventListener("abort", onAbort, { once: true }); + let terminal = false; + let initialized = false; + + try { + try { + for await (const record of jsonLines(child.stdout)) { + if (record.type === "system" && record.subtype === "init") { + initialized = true; + if (!sessionId) { + if (typeof record.session_id !== "string" || !record.session_id) { + throw new SafeDriverError( + "The Claude CLI returned invalid data.", + ); + } + yield { type: "session", sessionId: record.session_id }; + } + continue; + } + if (record.type === "stream_event") { + const event = isObject(record.event) ? record.event : undefined; + const delta = + event && isObject(event.delta) ? event.delta : undefined; + if ( + event?.type === "content_block_delta" && + delta?.type === "text_delta" && + typeof delta.text === "string" && + delta.text + ) { + yield { type: "text", delta: delta.text }; + } + continue; + } + if (record.type === "assistant") { + const activity = assistantActivity(record); + if (activity) yield activity; + continue; + } + if (record.type === "user") { + const activity = toolResultActivity(record); + if (activity) yield activity; + continue; + } + if (record.type === "system" && record.subtype === "api_retry") { + const isRateLimit = + record.error === "rate_limit" || record.error_status === 429; + yield { + type: "activity", + message: isRateLimit + ? "Claude is retrying after a rate limit" + : "Claude is retrying a provider request", + data: { + ...(typeof record.attempt === "number" + ? { attempt: record.attempt } + : {}), + ...(typeof record.max_retries === "number" + ? { maxRetries: record.max_retries } + : {}), + }, + }; + continue; + } + if ( + record.type === "system" && + (record.subtype === "hook_started" || + record.subtype === "permission_denial") + ) { + throw new SafeDriverError( + "The Claude run requested permission beyond worker policy.", + ); + } + if (record.type === "result") { + terminal = true; + if ( + Array.isArray(record.permission_denials) && + record.permission_denials.length > 0 + ) { + throw new SafeDriverError( + "The Claude run requested permission beyond worker policy.", + ); + } + if (record.is_error === true || record.subtype !== "success") { + throw new SafeDriverError("The Claude run failed."); + } + if (!initialized) { + throw new SafeDriverError( + "The Claude CLI returned invalid data.", + ); + } + return; + } + } + } catch (error) { + if (error instanceof SafeDriverError) throw error; + if (active.cancelled || context.signal.aborted) return; + throw new SafeDriverError("The Claude CLI sent invalid data."); + } + if (active.cancelled || context.signal.aborted) return; + if (!terminal) throw new SafeDriverError("The Claude CLI stopped early."); + } finally { + context.signal.removeEventListener("abort", onAbort); + if (this.active.get(run.runId) === active) this.active.delete(run.runId); + if (!terminal && !active.cancelled) await this.stopProcess(active); + } + } +} diff --git a/agent-subscription-gateway/src/drivers/codex/codex-driver.ts b/agent-subscription-gateway/src/drivers/codex/codex-driver.ts new file mode 100644 index 000000000..7759f5799 --- /dev/null +++ b/agent-subscription-gateway/src/drivers/codex/codex-driver.ts @@ -0,0 +1,581 @@ +import { isAbsolute } from "node:path"; +import { + type AgentDriver, + type DriverEvent, + type DriverResumeRun, + type DriverRun, + type DriverRunContext, + SafeDriverError, +} from "../agent-driver"; + +interface WritableInput { + write(chunk: string | Uint8Array): number | Promise; + flush?(): number | Promise; + end(): unknown; +} + +export interface CodexChildProcess { + stdin: WritableInput; + stdout: ReadableStream; + stderr: ReadableStream; + exited: Promise; + kill(exitCode?: number): unknown; +} + +export interface CodexSpawnOptions { + command: string[]; + cwd: string; + env: Record; +} + +export type CodexProcessFactory = ( + options: CodexSpawnOptions, +) => CodexChildProcess; + +interface JsonObject { + [key: string]: unknown; +} + +type RpcId = string | number; + +interface PendingRequest { + resolve(value: unknown): void; + reject(error: unknown): void; +} + +interface ActiveRun { + connection: CodexConnection; + threadId?: string; + turnId?: string; +} + +const SAFE_ENVIRONMENT_NAMES = new Set([ + "PATH", + "HOME", + "LANG", + "LANGUAGE", + "LC_ALL", + "TZ", + "TMPDIR", + "TMP", + "TEMP", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "COLORTERM", + "CODEX_HOME", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE", + "SSL_CERT_DIR", +]); + +function safeEnvironment( + source: Record, +): Record { + const environment: Record = {}; + for (const [name, value] of Object.entries(source)) { + if (value !== undefined && SAFE_ENVIRONMENT_NAMES.has(name)) { + environment[name] = value; + } + } + environment.GIT_TERMINAL_PROMPT = "0"; + return environment; +} + +class AsyncQueue implements AsyncIterable { + private readonly values: T[] = []; + private readonly waiters: Array<{ + resolve(value: IteratorResult): void; + reject(error: unknown): void; + }> = []; + private failure?: unknown; + private ended = false; + + push(value: T): void { + if (this.ended || this.failure) return; + const waiter = this.waiters.shift(); + if (waiter) waiter.resolve({ value, done: false }); + else this.values.push(value); + } + + fail(error: unknown): void { + if (this.ended || this.failure) return; + this.failure = error; + for (const waiter of this.waiters.splice(0)) waiter.reject(error); + } + + end(): void { + if (this.ended) return; + this.ended = true; + for (const waiter of this.waiters.splice(0)) { + waiter.resolve({ value: undefined, done: true }); + } + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: () => { + const value = this.values.shift(); + if (value !== undefined) return Promise.resolve({ value, done: false }); + if (this.failure) return Promise.reject(this.failure); + if (this.ended) + return Promise.resolve({ value: undefined, done: true }); + return new Promise>((resolve, reject) => { + this.waiters.push({ resolve, reject }); + }); + }, + }; + } +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function* jsonLines( + stream: ReadableStream, +): AsyncGenerator { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let buffered = ""; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + buffered += decoder.decode(chunk.value, { stream: true }); + while (true) { + const newline = buffered.indexOf("\n"); + if (newline < 0) break; + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + if (line) yield JSON.parse(line); + } + } + buffered += decoder.decode(); + const finalLine = buffered.trim(); + if (finalLine) yield JSON.parse(finalLine); + } finally { + reader.releaseLock(); + } +} + +async function drain(stream: ReadableStream): Promise { + try { + for await (const _chunk of stream) { + } + } catch {} +} + +class CodexConnection { + private readonly notifications = new AsyncQueue(); + private readonly pending = new Map(); + private nextId = 1; + private closed = false; + private shutdownPromise?: Promise; + + constructor(private readonly process: CodexChildProcess) { + void drain(process.stderr); + void this.read(); + } + + request(method: string, params: JsonObject): Promise { + if (this.closed) { + return Promise.reject( + new SafeDriverError("The Codex app server stopped."), + ); + } + const id = this.nextId++; + const response = new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + }); + this.send({ id, method, params }); + return response; + } + + notify(method: string, params: JsonObject): void { + this.send({ method, params }); + } + + events(): AsyncIterable { + return this.notifications; + } + + shutdown(): Promise { + if (this.shutdownPromise) return this.shutdownPromise; + this.closed = true; + this.stop(new SafeDriverError("The Codex app server stopped.")); + this.shutdownPromise = (async () => { + try { + this.process.stdin.end(); + } catch {} + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), 500); + }); + const exitCode = await Promise.race([this.process.exited, timedOut]); + if (timer) clearTimeout(timer); + if (exitCode === undefined) { + try { + this.process.kill(); + } catch {} + } + })(); + return this.shutdownPromise; + } + + private send(message: JsonObject): void { + try { + this.process.stdin.write(`${JSON.stringify(message)}\n`); + this.process.stdin.flush?.(); + } catch { + this.stop(new SafeDriverError("The Codex app server stopped.")); + } + } + + private async read(): Promise { + try { + for await (const raw of jsonLines(this.process.stdout)) { + if (!isObject(raw)) throw new Error("Invalid JSON-RPC message."); + if (typeof raw.method === "string" && Object.hasOwn(raw, "id")) { + this.send({ + id: raw.id as RpcId, + error: { + code: -32000, + message: "Interactive requests are disabled by worker policy.", + }, + }); + const error = new SafeDriverError( + "The Codex run requested interactive input and was stopped.", + ); + this.stop(error); + return; + } + if (Object.hasOwn(raw, "id")) { + const pending = this.pending.get(raw.id as RpcId); + if (!pending) continue; + this.pending.delete(raw.id as RpcId); + if (Object.hasOwn(raw, "error")) { + pending.reject( + new SafeDriverError("The Codex app server rejected a request."), + ); + } else { + pending.resolve(raw.result); + } + continue; + } + if (typeof raw.method === "string") this.notifications.push(raw); + } + if (!this.closed) { + this.stop(new SafeDriverError("The Codex app server stopped early.")); + } + } catch { + if (!this.closed) { + this.stop( + new SafeDriverError("The Codex app server sent invalid data."), + ); + } + } + } + + private stop(error: SafeDriverError): void { + for (const request of this.pending.values()) request.reject(error); + this.pending.clear(); + this.notifications.fail(error); + } +} + +function defaultProcessFactory(options: CodexSpawnOptions): CodexChildProcess { + return Bun.spawn(options.command, { + cwd: options.cwd, + env: options.env, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); +} + +function requiredWorkspace(run: DriverRun): string { + if (!run.workspacePath || !isAbsolute(run.workspacePath)) { + throw new SafeDriverError("The Codex workspace is not available."); + } + return run.workspacePath; +} + +function transcript(run: DriverRun): string { + const parts: string[] = []; + for (const raw of run.messages) { + if (!isObject(raw)) continue; + const { id, role, content } = raw; + if ( + typeof id !== "string" || + typeof role !== "string" || + typeof content !== "string" + ) { + continue; + } + parts.push(`[${role} ${id}]\n${content}`); + } + if (parts.length === 0) { + throw new SafeDriverError("The Codex run has no transcript input."); + } + return `OpenBot transcript:\n\n${parts.join("\n\n")}`; +} + +function resultObject(result: unknown): JsonObject { + if (!isObject(result)) { + throw new SafeDriverError("The Codex app server returned invalid data."); + } + return result; +} + +function nestedId(result: unknown, field: string): string { + const record = resultObject(result); + const nested = record[field]; + if (!isObject(nested) || typeof nested.id !== "string" || !nested.id) { + throw new SafeDriverError("The Codex app server returned invalid data."); + } + return nested.id; +} + +function itemActivity(message: JsonObject): DriverEvent | undefined { + const method = message.method; + const params = isObject(message.params) ? message.params : undefined; + const item = params && isObject(params.item) ? params.item : undefined; + if (method === "item/commandExecution/outputDelta") { + return { + type: "activity", + message: "Command produced output", + data: { itemType: "commandExecution" }, + }; + } + if ((method !== "item/started" && method !== "item/completed") || !item) { + return undefined; + } + const itemType = typeof item.type === "string" ? item.type : "nativeAction"; + const status = typeof item.status === "string" ? item.status : undefined; + const action = + itemType === "commandExecution" + ? "Command" + : itemType === "fileChange" + ? "File change" + : itemType === "mcpToolCall" + ? "MCP call" + : itemType === "webSearch" + ? "Web search" + : "Native action"; + return { + type: "activity", + message: `${action} ${method === "item/started" ? "started" : "completed"}`, + data: { itemType, ...(status ? { status } : {}) }, + }; +} + +function otherActivity(message: JsonObject): DriverEvent | undefined { + const method = message.method; + if (method === "turn/diff/updated") { + return { type: "activity", message: "Repository diff updated" }; + } + if (method === "turn/plan/updated") { + return { type: "activity", message: "Plan updated" }; + } + if (method === "hook/started") { + return { type: "activity", message: "Hook started" }; + } + if (method === "hook/completed") { + return { type: "activity", message: "Hook completed" }; + } + if (method === "warning" || method === "configWarning") { + return { type: "activity", message: "Codex reported a warning" }; + } + return undefined; +} + +export interface CodexDriverOptions { + processFactory?: CodexProcessFactory; + binary?: string; + version?: string; + environment?: Record; + interruptTimeoutMs?: number; + permissionProfile?: string; +} + +export class CodexDriver implements AgentDriver { + readonly provider = "codex"; + readonly version: string; + private readonly processFactory: CodexProcessFactory; + private readonly binary: string; + private readonly environment: Record; + private readonly interruptTimeoutMs: number; + private readonly permissionProfile: string; + private readonly active = new Map(); + + constructor(options: CodexDriverOptions = {}) { + this.processFactory = options.processFactory ?? defaultProcessFactory; + this.binary = options.binary ?? "codex"; + this.version = options.version ?? process.env.CODEX_VERSION ?? "unknown"; + this.environment = safeEnvironment({ + ...process.env, + ...options.environment, + }); + this.interruptTimeoutMs = options.interruptTimeoutMs ?? 3_000; + this.permissionProfile = options.permissionProfile ?? "openbot-worker"; + } + + async isAuthReady(): Promise { + let child: CodexChildProcess; + try { + child = this.processFactory({ + command: [this.binary, "login", "status"], + cwd: process.cwd(), + env: this.environment, + }); + } catch { + return false; + } + void drain(child.stdout); + void drain(child.stderr); + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), 5_000); + }); + const exitCode = await Promise.race([child.exited, timedOut]); + if (timer) clearTimeout(timer); + if (exitCode === undefined) { + try { + child.kill(); + } catch {} + return false; + } + return exitCode === 0; + } + + start(run: DriverRun, context: DriverRunContext): AsyncIterable { + return this.execute(run, context); + } + + resume( + run: DriverResumeRun, + context: DriverRunContext, + ): AsyncIterable { + return this.execute(run, context, run.sessionId); + } + + async cancel(runId: string): Promise { + const active = this.active.get(runId); + if (!active) return; + if (!active.threadId || !active.turnId) { + await active.connection.shutdown(); + return; + } + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), this.interruptTimeoutMs); + }); + try { + await Promise.race([ + active.connection.request("turn/interrupt", { + threadId: active.threadId, + turnId: active.turnId, + }), + timedOut, + ]); + } catch { + } finally { + if (timer) clearTimeout(timer); + await active.connection.shutdown(); + } + } + + private async *execute( + run: DriverRun, + context: DriverRunContext, + sessionId?: string, + ): AsyncGenerator { + const workspace = requiredWorkspace(run); + const child = this.processFactory({ + command: [this.binary, "app-server"], + cwd: workspace, + env: this.environment, + }); + const connection = new CodexConnection(child); + const active: ActiveRun = { connection }; + if (this.active.has(run.runId)) { + await connection.shutdown(); + throw new SafeDriverError("The Codex run is already active."); + } + this.active.set(run.runId, active); + const onAbort = () => void this.cancel(run.runId); + context.signal.addEventListener("abort", onAbort, { once: true }); + + try { + await connection.request("initialize", { + clientInfo: { + name: "openbot_subscription_gateway", + title: "OpenBot Subscription Gateway", + version: "0.1.0", + }, + capabilities: { experimentalApi: true }, + }); + connection.notify("initialized", {}); + + const threadResult = sessionId + ? await connection.request("thread/resume", { + threadId: sessionId, + cwd: workspace, + approvalPolicy: "never", + permissions: this.permissionProfile, + }) + : await connection.request("thread/start", { + cwd: workspace, + approvalPolicy: "never", + permissions: this.permissionProfile, + serviceName: "openbot_subscription_gateway", + }); + const threadId = nestedId(threadResult, "thread"); + active.threadId = threadId; + if (!sessionId) yield { type: "session", sessionId: threadId }; + + const turnResult = await connection.request("turn/start", { + threadId, + input: [{ type: "text", text: transcript(run) }], + cwd: workspace, + approvalPolicy: "never", + }); + active.turnId = nestedId(turnResult, "turn"); + + for await (const message of connection.events()) { + const method = message.method; + const params = isObject(message.params) ? message.params : undefined; + if (method === "item/agentMessage/delta") { + const delta = params?.delta; + if (typeof delta === "string" && delta) yield { type: "text", delta }; + continue; + } + if (method === "error") continue; + if (method === "turn/completed") { + const turn = + params && isObject(params.turn) ? params.turn : undefined; + if (!turn || typeof turn.status !== "string") { + throw new SafeDriverError( + "The Codex app server returned invalid data.", + ); + } + if (turn.status === "completed") return; + if (turn.status === "interrupted" && context.signal.aborted) return; + if (turn.status === "interrupted") { + throw new SafeDriverError("The Codex run was interrupted."); + } + throw new SafeDriverError("The Codex run failed."); + } + const activity = itemActivity(message) ?? otherActivity(message); + if (activity) yield activity; + } + throw new SafeDriverError("The Codex app server stopped early."); + } finally { + context.signal.removeEventListener("abort", onAbort); + if (this.active.get(run.runId) === active) this.active.delete(run.runId); + await connection.shutdown(); + } + } +} diff --git a/agent-subscription-gateway/src/drivers/create-driver.ts b/agent-subscription-gateway/src/drivers/create-driver.ts new file mode 100644 index 000000000..b87323465 --- /dev/null +++ b/agent-subscription-gateway/src/drivers/create-driver.ts @@ -0,0 +1,11 @@ +import type { AgentDriver } from "./agent-driver"; +import { ClaudeDriver } from "./claude/claude-driver"; +import { CodexDriver } from "./codex/codex-driver"; +import { GrokDriver } from "./grok/grok-driver"; + +export function createDriver(provider: string): AgentDriver { + if (provider === "codex") return new CodexDriver(); + if (provider === "claude") return new ClaudeDriver(); + if (provider === "grok") return new GrokDriver(); + throw new Error(`Unsupported provider: ${provider}.`); +} diff --git a/agent-subscription-gateway/src/drivers/grok/grok-driver.ts b/agent-subscription-gateway/src/drivers/grok/grok-driver.ts new file mode 100644 index 000000000..3a0b80651 --- /dev/null +++ b/agent-subscription-gateway/src/drivers/grok/grok-driver.ts @@ -0,0 +1,566 @@ +import { isAbsolute } from "node:path"; +import { + type AgentDriver, + type DriverEvent, + type DriverResumeRun, + type DriverRun, + type DriverRunContext, + SafeDriverError, +} from "../agent-driver"; + +interface JsonObject { + [key: string]: unknown; +} + +interface WritableInput { + write(chunk: string | Uint8Array): number | Promise; + flush?(): number | Promise; + end(): unknown; +} + +export interface GrokChildProcess { + stdin: WritableInput; + stdout: ReadableStream; + stderr: ReadableStream; + exited: Promise; + kill(exitCode?: number): unknown; +} + +export interface GrokSpawnOptions { + command: string[]; + cwd: string; + env: Record; +} + +export type GrokProcessFactory = ( + options: GrokSpawnOptions, +) => GrokChildProcess; + +type RpcId = string | number; + +interface PendingRequest { + resolve(value: unknown): void; + reject(error: unknown): void; + notifyCompletion: boolean; +} + +interface ActiveRun { + connection: GrokConnection; + sessionId?: string; +} + +class AsyncQueue implements AsyncIterable { + private readonly values: T[] = []; + private readonly waiters: Array<{ + resolve(value: IteratorResult): void; + reject(error: unknown): void; + }> = []; + private failure?: unknown; + private ended = false; + + push(value: T): void { + if (this.ended || this.failure) return; + const waiter = this.waiters.shift(); + if (waiter) waiter.resolve({ value, done: false }); + else this.values.push(value); + } + + fail(error: unknown): void { + if (this.ended || this.failure) return; + this.failure = error; + for (const waiter of this.waiters.splice(0)) waiter.reject(error); + } + + end(): void { + if (this.ended) return; + this.ended = true; + for (const waiter of this.waiters.splice(0)) { + waiter.resolve({ value: undefined, done: true }); + } + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: () => { + const value = this.values.shift(); + if (value !== undefined) return Promise.resolve({ value, done: false }); + if (this.failure) return Promise.reject(this.failure); + if (this.ended) + return Promise.resolve({ value: undefined, done: true }); + return new Promise>((resolve, reject) => { + this.waiters.push({ resolve, reject }); + }); + }, + }; + } +} + +function isObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function* jsonLines( + stream: ReadableStream, +): AsyncGenerator { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let buffered = ""; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + buffered += decoder.decode(chunk.value, { stream: true }); + while (true) { + const newline = buffered.indexOf("\n"); + if (newline < 0) break; + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + if (line) yield JSON.parse(line); + } + } + buffered += decoder.decode(); + const finalLine = buffered.trim(); + if (finalLine) yield JSON.parse(finalLine); + } finally { + reader.releaseLock(); + } +} + +async function drain(stream: ReadableStream): Promise { + try { + for await (const _chunk of stream) { + } + } catch {} +} + +class GrokConnection { + private readonly notifications = new AsyncQueue(); + private readonly pending = new Map(); + private nextId = 1; + private closed = false; + private shutdownPromise?: Promise; + + constructor(private readonly process: GrokChildProcess) { + void drain(process.stderr); + void this.read(); + } + + request( + method: string, + params: JsonObject, + notifyCompletion = false, + ): Promise { + if (this.closed) { + return Promise.reject(new SafeDriverError("The Grok agent stopped.")); + } + const id = this.nextId++; + const response = new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject, notifyCompletion }); + }); + this.send({ jsonrpc: "2.0", id, method, params }); + return response; + } + + notify(method: string, params: JsonObject): void { + this.send({ jsonrpc: "2.0", method, params }); + } + + events(): AsyncIterable { + return this.notifications; + } + + shutdown(): Promise { + if (this.shutdownPromise) return this.shutdownPromise; + this.closed = true; + this.stop(new SafeDriverError("The Grok agent stopped.")); + this.shutdownPromise = (async () => { + try { + this.process.stdin.end(); + } catch {} + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), 500); + }); + const exitCode = await Promise.race([this.process.exited, timedOut]); + if (timer) clearTimeout(timer); + if (exitCode === undefined) { + try { + this.process.kill(); + } catch {} + } + })(); + return this.shutdownPromise; + } + + private send(message: JsonObject): void { + try { + this.process.stdin.write(`${JSON.stringify(message)}\n`); + this.process.stdin.flush?.(); + } catch { + this.stop(new SafeDriverError("The Grok agent stopped.")); + } + } + + private async read(): Promise { + try { + for await (const raw of jsonLines(this.process.stdout)) { + if (!isObject(raw)) throw new Error("Invalid JSON-RPC message."); + if (typeof raw.method === "string" && Object.hasOwn(raw, "id")) { + this.send({ + jsonrpc: "2.0", + id: raw.id as RpcId, + error: { + code: -32000, + message: "Interactive requests are disabled by worker policy.", + }, + }); + this.stop( + new SafeDriverError( + "The Grok run requested permission beyond worker policy.", + ), + ); + return; + } + if (Object.hasOwn(raw, "id")) { + const pending = this.pending.get(raw.id as RpcId); + if (!pending) continue; + this.pending.delete(raw.id as RpcId); + if (pending.notifyCompletion) { + this.notifications.push({ + method: "$request/completed", + params: { id: raw.id as RpcId }, + }); + } + if (Object.hasOwn(raw, "error")) { + pending.reject( + new SafeDriverError("The Grok agent rejected a request."), + ); + } else { + pending.resolve(raw.result); + } + continue; + } + if (typeof raw.method === "string") this.notifications.push(raw); + } + if (!this.closed) { + this.stop(new SafeDriverError("The Grok agent stopped early.")); + } + } catch { + if (!this.closed) { + this.stop(new SafeDriverError("The Grok agent sent invalid data.")); + } + } + } + + private stop(error: SafeDriverError): void { + for (const request of this.pending.values()) request.reject(error); + this.pending.clear(); + this.notifications.fail(error); + } +} + +const SAFE_ENVIRONMENT_NAMES = new Set([ + "PATH", + "HOME", + "LANG", + "LANGUAGE", + "LC_ALL", + "TZ", + "TMPDIR", + "TMP", + "TEMP", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "COLORTERM", + "GROK_HOME", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE", + "SSL_CERT_DIR", +]); + +function safeEnvironment( + source: Record, +): Record { + const environment: Record = {}; + for (const [name, value] of Object.entries(source)) { + if (value !== undefined && SAFE_ENVIRONMENT_NAMES.has(name)) { + environment[name] = value; + } + } + environment.GROK_DISABLE_AUTOUPDATER = "1"; + environment.GIT_TERMINAL_PROMPT = "0"; + return environment; +} + +function defaultProcessFactory(options: GrokSpawnOptions): GrokChildProcess { + return Bun.spawn(options.command, { + cwd: options.cwd, + env: options.env, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); +} + +function requiredWorkspace(run: DriverRun): string { + if (!run.workspacePath || !isAbsolute(run.workspacePath)) { + throw new SafeDriverError("The Grok workspace is not available."); + } + return run.workspacePath; +} + +function transcript(run: DriverRun): string { + const parts: string[] = []; + for (const raw of run.messages) { + if (!isObject(raw)) continue; + const { id, role, content } = raw; + if ( + typeof id !== "string" || + typeof role !== "string" || + typeof content !== "string" + ) { + continue; + } + parts.push(`[${role} ${id}]\n${content}`); + } + if (parts.length === 0) { + throw new SafeDriverError("The Grok run has no transcript input."); + } + return `OpenBot transcript:\n\n${parts.join("\n\n")}`; +} + +function resultObject(result: unknown): JsonObject { + if (!isObject(result)) { + throw new SafeDriverError("The Grok agent returned invalid data."); + } + return result; +} + +function sessionIdFrom(result: unknown): string { + const sessionId = resultObject(result).sessionId; + if (typeof sessionId !== "string" || !sessionId) { + throw new SafeDriverError("The Grok agent returned invalid data."); + } + return sessionId; +} + +function eventFor(message: JsonObject): DriverEvent | undefined { + if (message.method !== "session/update") return undefined; + const params = isObject(message.params) ? message.params : undefined; + const update = params && isObject(params.update) ? params.update : undefined; + if (!update || typeof update.sessionUpdate !== "string") return undefined; + if (update.sessionUpdate === "agent_message_chunk") { + const content = isObject(update.content) ? update.content : undefined; + if ( + content?.type === "text" && + typeof content.text === "string" && + content.text + ) { + return { type: "text", delta: content.text }; + } + return undefined; + } + if (update.sessionUpdate === "tool_call") { + return { type: "activity", message: "Native tool started" }; + } + if (update.sessionUpdate === "tool_call_update") { + const status = update.status; + const safeStatus = + status === "completed" || status === "failed" || status === "in_progress" + ? status + : undefined; + return { + type: "activity", + message: + safeStatus === "completed" + ? "Native tool completed" + : safeStatus === "failed" + ? "Native tool failed" + : "Native tool updated", + ...(safeStatus ? { data: { status: safeStatus } } : {}), + }; + } + if (update.sessionUpdate === "plan") { + return { type: "activity", message: "Plan updated" }; + } + return undefined; +} + +export interface GrokDriverOptions { + processFactory?: GrokProcessFactory; + binary?: string; + version?: string; + environment?: Record; + authTimeoutMs?: number; + sandboxProfile?: string; +} + +export class GrokDriver implements AgentDriver { + readonly provider = "grok"; + readonly version: string; + private readonly processFactory: GrokProcessFactory; + private readonly binary: string; + private readonly environment: Record; + private readonly authTimeoutMs: number; + private readonly sandboxProfile: string; + private readonly active = new Map(); + + constructor(options: GrokDriverOptions = {}) { + this.processFactory = options.processFactory ?? defaultProcessFactory; + this.binary = options.binary ?? "grok"; + this.version = options.version ?? process.env.GROK_VERSION ?? "unknown"; + this.environment = safeEnvironment(options.environment ?? process.env); + this.authTimeoutMs = options.authTimeoutMs ?? 10_000; + this.sandboxProfile = options.sandboxProfile ?? "strict"; + } + + async isAuthReady(): Promise { + let child: GrokChildProcess; + try { + child = this.processFactory({ + command: [this.binary, "models"], + cwd: process.cwd(), + env: this.environment, + }); + } catch { + return false; + } + void drain(child.stdout); + void drain(child.stderr); + let timer: ReturnType | undefined; + const timedOut = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), this.authTimeoutMs); + }); + const exitCode = await Promise.race([child.exited, timedOut]); + if (timer) clearTimeout(timer); + if (exitCode === undefined) { + try { + child.kill(); + } catch {} + return false; + } + return exitCode === 0; + } + + start(run: DriverRun, context: DriverRunContext): AsyncIterable { + return this.execute(run, context); + } + + resume( + run: DriverResumeRun, + context: DriverRunContext, + ): AsyncIterable { + return this.execute(run, context, run.sessionId); + } + + async cancel(runId: string): Promise { + const active = this.active.get(runId); + if (!active) return; + if (active.sessionId) { + active.connection.notify("session/cancel", { + sessionId: active.sessionId, + }); + } + await active.connection.shutdown(); + } + + private async *execute( + run: DriverRun, + context: DriverRunContext, + existingSessionId?: string, + ): AsyncGenerator { + const workspace = requiredWorkspace(run); + if (this.active.has(run.runId)) { + throw new SafeDriverError("The Grok run is already active."); + } + let child: GrokChildProcess; + try { + child = this.processFactory({ + command: [ + this.binary, + "--no-subagents", + "--sandbox", + this.sandboxProfile, + "agent", + "--always-approve", + "--no-leader", + "stdio", + ], + cwd: workspace, + env: this.environment, + }); + } catch { + throw new SafeDriverError("The Grok agent could not start."); + } + const connection = new GrokConnection(child); + const active: ActiveRun = { connection }; + this.active.set(run.runId, active); + const onAbort = () => void this.cancel(run.runId); + context.signal.addEventListener("abort", onAbort, { once: true }); + + try { + await connection.request("initialize", { + protocolVersion: 1, + clientCapabilities: {}, + clientInfo: { + name: "openbot_subscription_gateway", + title: "OpenBot Subscription Gateway", + version: "0.1.0", + }, + }); + const sessionResult = existingSessionId + ? await connection.request("session/load", { + sessionId: existingSessionId, + cwd: workspace, + mcpServers: [], + _meta: { yoloMode: true }, + }) + : await connection.request("session/new", { + cwd: workspace, + mcpServers: [], + _meta: { yoloMode: true }, + }); + const sessionId = existingSessionId ?? sessionIdFrom(sessionResult); + active.sessionId = sessionId; + if (!existingSessionId) yield { type: "session", sessionId }; + + const terminal = connection + .request( + "session/prompt", + { + sessionId, + prompt: [{ type: "text", text: transcript(run) }], + }, + true, + ) + .then( + (result) => ({ result }), + (error: unknown) => ({ error }), + ); + for await (const message of connection.events()) { + if (message.method === "$request/completed") { + const outcome = await terminal; + if ("error" in outcome) throw outcome.error; + const result = resultObject(outcome.result); + if (typeof result.stopReason !== "string") { + throw new SafeDriverError("The Grok agent returned invalid data."); + } + return; + } + const event = eventFor(message); + if (event) yield event; + } + throw new SafeDriverError("The Grok agent stopped early."); + } catch (error) { + if (context.signal.aborted) return; + if (error instanceof SafeDriverError) throw error; + throw new SafeDriverError("The Grok run failed."); + } finally { + context.signal.removeEventListener("abort", onAbort); + if (this.active.get(run.runId) === active) this.active.delete(run.runId); + await connection.shutdown(); + } + } +} diff --git a/agent-subscription-gateway/src/main.ts b/agent-subscription-gateway/src/main.ts new file mode 100644 index 000000000..26528dc51 --- /dev/null +++ b/agent-subscription-gateway/src/main.ts @@ -0,0 +1,79 @@ +import { createDriver } from "./drivers/create-driver"; +import { consoleRunLogger } from "./observability/run-logger"; +import { BoundedRunQueue, HostSemaphore } from "./runs/queue"; +import { RunService } from "./runs/run-service"; +import { serveGateway } from "./server"; +import { RunStore } from "./storage/run-store"; +import { WorkspaceManager } from "./workspaces/workspace-manager"; + +function required(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required.`); + return value; +} + +function positiveInteger(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer.`); + } + return value; +} + +function nonNegativeInteger(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative integer.`); + } + return value; +} + +const provider = required("PROVIDER"); +const runStorePath = required("RUN_STORE_PATH"); +const hostSemaphorePath = required("HOST_SEMAPHORE_PATH"); +const store = new RunStore(runStorePath); +const leaseStore = + hostSemaphorePath === runStorePath ? store : new RunStore(hostSemaphorePath); +const driver = createDriver(provider); +const host = new HostSemaphore({ + limit: positiveInteger("HOST_CONCURRENCY", 1), + leaseStore, +}); +const queue = new BoundedRunQueue({ + provider, + host, + concurrency: positiveInteger("PROVIDER_CONCURRENCY", 1), + pendingLimit: nonNegativeInteger("QUEUE_CAPACITY", 1), +}); +const workspaces = new WorkspaceManager({ + provider, + repository: required("REPOSITORY_URL"), + baseBranch: required("BASE_BRANCH"), + jobRoot: required("JOB_ROOT"), +}); +const runService = new RunService({ + driver, + store, + queue, + workspaces, + logger: consoleRunLogger, +}); +const server = serveGateway({ + port: positiveInteger("PORT", 4210), + token: required("GATEWAY_TOKEN"), + driver, + runService, +}); + +function stop(): void { + server.stop(true); + store.close(); + if (leaseStore !== store) leaseStore.close(); +} + +process.once("SIGINT", stop); +process.once("SIGTERM", stop); diff --git a/agent-subscription-gateway/src/observability/run-counts.ts b/agent-subscription-gateway/src/observability/run-counts.ts new file mode 100644 index 000000000..a92ce53ab --- /dev/null +++ b/agent-subscription-gateway/src/observability/run-counts.ts @@ -0,0 +1,26 @@ +export interface RunCountSnapshot { + queueDepth: number; + activeRuns: number; +} + +export interface RunCounts { + snapshot(): RunCountSnapshot; + started(): void; + finished(): void; +} + +export class InMemoryRunCounts implements RunCounts { + private activeRuns = 0; + + snapshot(): RunCountSnapshot { + return { queueDepth: 0, activeRuns: this.activeRuns }; + } + + started(): void { + this.activeRuns += 1; + } + + finished(): void { + this.activeRuns = Math.max(0, this.activeRuns - 1); + } +} diff --git a/agent-subscription-gateway/src/observability/run-logger.ts b/agent-subscription-gateway/src/observability/run-logger.ts new file mode 100644 index 000000000..e1393818b --- /dev/null +++ b/agent-subscription-gateway/src/observability/run-logger.ts @@ -0,0 +1,15 @@ +export type RunOutcome = "finished" | "failed" | "cancelled"; + +export interface RunLogger { + terminal(event: { + provider: string; + runId: string; + outcome: RunOutcome; + }): void; +} + +export const consoleRunLogger: RunLogger = { + terminal(event) { + console.info(JSON.stringify({ event: "provider_run_terminal", ...event })); + }, +}; diff --git a/agent-subscription-gateway/src/observability/status.ts b/agent-subscription-gateway/src/observability/status.ts new file mode 100644 index 000000000..5f4f4a8fe --- /dev/null +++ b/agent-subscription-gateway/src/observability/status.ts @@ -0,0 +1,28 @@ +import type { AgentDriver } from "../drivers/agent-driver"; +import type { RunCounts } from "./run-counts"; + +export interface GatewayStatus { + status: "ok" | "ready" | "not_ready"; + provider: string; + version: string; + authReady: boolean; + queueDepth: number; + activeRuns: number; +} + +export async function gatewayStatus( + driver: AgentDriver, + counts: RunCounts, + route: "health" | "ready", +): Promise { + const authReady = await driver.isAuthReady().catch(() => false); + const current = counts.snapshot(); + return { + status: route === "health" ? "ok" : authReady ? "ready" : "not_ready", + provider: driver.provider, + version: driver.version, + authReady, + queueDepth: current.queueDepth, + activeRuns: current.activeRuns, + }; +} diff --git a/agent-subscription-gateway/src/runs/queue.ts b/agent-subscription-gateway/src/runs/queue.ts new file mode 100644 index 000000000..216c2f1d3 --- /dev/null +++ b/agent-subscription-gateway/src/runs/queue.ts @@ -0,0 +1,272 @@ +import type { RunCountSnapshot, RunCounts } from "../observability/run-counts"; + +export class CapacityError extends Error { + readonly retryable = true; + + constructor() { + super("The provider queue is full. Retry later."); + this.name = "CapacityError"; + } +} + +export class RunDisconnectedError extends Error { + constructor() { + super("The queued run lost its AG-UI connection."); + this.name = "RunDisconnectedError"; + } +} + +type Release = () => void; + +export interface HostLeaseStore { + tryAcquireHostLease(owner: string, limit: number): boolean; + releaseHostLease(owner: string): void; +} + +interface HostWaiter { + grant: (release: Release) => void; + signal: AbortSignal; + owner: string; + abort: () => void; + poll?: ReturnType; +} + +export class HostSemaphore { + private active = 0; + private readonly waiters: HostWaiter[] = []; + private readonly leaseStore?: HostLeaseStore; + private readonly pollMs: number; + private manualOwner = 0; + + readonly limit: number; + + constructor( + options: + | number + | { limit?: number; leaseStore: HostLeaseStore; pollMs?: number } = 1, + ) { + this.limit = typeof options === "number" ? options : (options.limit ?? 1); + this.leaseStore = + typeof options === "number" ? undefined : options.leaseStore; + this.pollMs = typeof options === "number" ? 50 : (options.pollMs ?? 50); + if (!Number.isInteger(this.limit) || this.limit < 1) { + throw new Error("Host concurrency must be a positive integer."); + } + } + + get activeCount(): number { + return this.active; + } + + tryAcquire(owner = `manual:${this.manualOwner++}`): Release | undefined { + if (this.active >= this.limit || this.waiters.length > 0) return undefined; + if ( + this.leaseStore && + !this.leaseStore.tryAcquireHostLease(owner, this.limit) + ) { + return undefined; + } + this.active += 1; + return this.releaseOnce(owner); + } + + acquire(signal: AbortSignal, owner: string): Promise { + if (signal.aborted) return Promise.reject(new RunDisconnectedError()); + const immediate = this.tryAcquire(owner); + if (immediate) return Promise.resolve(immediate); + + return new Promise((resolve, reject) => { + const waiter: HostWaiter = { + grant: resolve, + signal, + owner, + abort: () => { + const index = this.waiters.indexOf(waiter); + if (index >= 0) this.waiters.splice(index, 1); + if (waiter.poll) clearInterval(waiter.poll); + reject(new RunDisconnectedError()); + }, + }; + signal.addEventListener("abort", waiter.abort, { once: true }); + this.waiters.push(waiter); + if (this.leaseStore) { + waiter.poll = setInterval(() => this.grantNext(), this.pollMs); + } + }); + } + + private releaseOnce(owner: string): Release { + let released = false; + return () => { + if (released) return; + released = true; + this.active = Math.max(0, this.active - 1); + this.leaseStore?.releaseHostLease(owner); + this.grantNext(); + }; + } + + private grantNext(): void { + while (this.active < this.limit) { + const waiter = this.waiters.shift(); + if (!waiter) return; + waiter.signal.removeEventListener("abort", waiter.abort); + if (waiter.poll) clearInterval(waiter.poll); + if (waiter.signal.aborted) continue; + if ( + this.leaseStore && + !this.leaseStore.tryAcquireHostLease(waiter.owner, this.limit) + ) { + this.waiters.unshift(waiter); + waiter.poll = setInterval(() => this.grantNext(), this.pollMs); + return; + } + this.active += 1; + waiter.grant(this.releaseOnce(waiter.owner)); + } + } +} + +export interface QueuedRun { + runId: string; + signal: AbortSignal; + run: () => Promise | T; + onHeartbeat?: () => void; +} + +interface QueueItem extends QueuedRun { + resolve: (value: T | PromiseLike) => void; + reject: (reason?: unknown) => void; + heartbeat?: ReturnType; +} + +export class BoundedRunQueue implements RunCounts { + private readonly provider: string; + private readonly host: HostSemaphore; + private readonly concurrency: number; + private readonly pendingLimit: number; + private readonly heartbeatMs: number; + private readonly pending: QueueItem[] = []; + private active = 0; + private dispatching = 0; + + constructor(options: { + provider: string; + host: HostSemaphore; + concurrency?: number; + pendingLimit?: number; + heartbeatMs?: number; + }) { + this.provider = options.provider; + this.host = options.host; + this.concurrency = options.concurrency ?? 1; + this.pendingLimit = options.pendingLimit ?? 1; + this.heartbeatMs = options.heartbeatMs ?? 15_000; + if (!this.provider.trim()) throw new Error("Provider is required."); + if (!Number.isInteger(this.concurrency) || this.concurrency < 1) { + throw new Error("Provider concurrency must be a positive integer."); + } + if (!Number.isInteger(this.pendingLimit) || this.pendingLimit < 0) { + throw new Error("Pending limit must be a non-negative integer."); + } + if (!Number.isInteger(this.heartbeatMs) || this.heartbeatMs < 1) { + throw new Error("Heartbeat interval must be a positive integer."); + } + } + + enqueue(run: QueuedRun): Promise { + const owner = `${this.provider}:${run.runId}`; + const canStart = + this.active + this.dispatching < this.concurrency && + this.pending.length === 0; + const immediate = canStart ? this.host.tryAcquire(owner) : undefined; + if (!immediate && this.pending.length >= this.pendingLimit) + throw new CapacityError(); + if (run.signal.aborted) return Promise.reject(new RunDisconnectedError()); + + const result = new Promise((resolve, reject) => { + const item = { ...run, resolve, reject } as QueueItem; + if (run.onHeartbeat) { + item.heartbeat = setInterval(run.onHeartbeat, this.heartbeatMs); + } + this.pending.push(item as QueueItem); + if (immediate) this.start(item as QueueItem, immediate); + else this.drain(); + }); + return result; + } + + snapshot(): RunCountSnapshot { + return { queueDepth: this.pending.length, activeRuns: this.active }; + } + + started(): void {} + + finished(): void {} + + private drain(): void { + while ( + this.active + this.dispatching < this.concurrency && + this.pending.length > this.dispatching + ) { + const item = this.pending[this.dispatching]; + const immediate = this.host.tryAcquire(`${this.provider}:${item.runId}`); + if (immediate) { + this.start(item, immediate); + continue; + } + this.dispatching += 1; + void this.waitForHost(item); + } + } + + private async waitForHost(item: QueueItem): Promise { + let release: Release | undefined; + try { + release = await this.host.acquire( + item.signal, + `${this.provider}:${item.runId}`, + ); + const index = this.pending.indexOf(item); + if (index < 0 || item.signal.aborted) { + throw new RunDisconnectedError(); + } + this.dispatching = Math.max(0, this.dispatching - 1); + this.start(item, release); + release = undefined; + } catch (error) { + const index = this.pending.indexOf(item); + if (index >= 0) this.pending.splice(index, 1); + this.dispatching = Math.max(0, this.dispatching - 1); + if (item.heartbeat) clearInterval(item.heartbeat); + release?.(); + item.reject(error); + this.drain(); + } + } + + private start(item: QueueItem, release: Release): void { + const index = this.pending.indexOf(item); + if (index < 0) { + release(); + return; + } + this.pending.splice(index, 1); + if (item.heartbeat) clearInterval(item.heartbeat); + this.active += 1; + void (async () => { + try { + const value = await item.run(); + this.active = Math.max(0, this.active - 1); + release(); + this.drain(); + item.resolve(value); + } catch (error) { + this.active = Math.max(0, this.active - 1); + release(); + this.drain(); + item.reject(error); + } + })(); + } +} diff --git a/agent-subscription-gateway/src/runs/run-service.ts b/agent-subscription-gateway/src/runs/run-service.ts new file mode 100644 index 000000000..ce2fb1274 --- /dev/null +++ b/agent-subscription-gateway/src/runs/run-service.ts @@ -0,0 +1,252 @@ +import type { AgentDriver } from "../drivers/agent-driver"; +import type { RunLogger } from "../observability/run-logger"; +import type { GatewayRunService } from "../server/app"; +import type { GatewayRunInput } from "../server/input"; +import { runAgent } from "../server/run-agent"; +import { encodeSse, SSE_HEADERS } from "../server/sse"; +import type { RunState, RunStore, StoredRun } from "../storage/run-store"; +import type { + WorkspaceLease, + WorkspaceManager, + WorkspaceResult, +} from "../workspaces/workspace-manager"; +import { type BoundedRunQueue, CapacityError } from "./queue"; +import { prepareProviderRun } from "./session-input"; + +const RESTART_ERROR = "Gateway restarted; submit a new run."; + +interface WorkspaceOperations { + create(runId: string): Promise; + captureResult(workspace: WorkspaceLease): Promise; +} + +function priorRun(run: StoredRun) { + return { + provider: run.provider, + runId: run.runId, + threadId: run.threadId, + state: run.state, + ...(run.errorSummary ? { error: run.errorSummary } : {}), + resultAvailable: run.patch !== undefined, + }; +} + +function safeSetupFailure(error: unknown): string { + if (error instanceof CapacityError) return error.message; + return "The run could not be prepared."; +} + +function mergeSignals(first: AbortSignal, second: AbortSignal): AbortSignal { + if (typeof AbortSignal.any === "function") + return AbortSignal.any([first, second]); + const controller = new AbortController(); + const abort = () => controller.abort(); + first.addEventListener("abort", abort, { once: true }); + second.addEventListener("abort", abort, { once: true }); + return controller.signal; +} + +export class RunService implements GatewayRunService { + readonly counts: BoundedRunQueue; + private readonly provider: string; + + constructor( + private readonly options: { + driver: AgentDriver; + store: RunStore; + queue: BoundedRunQueue; + workspaces: WorkspaceOperations | WorkspaceManager; + logger: RunLogger; + }, + ) { + this.provider = options.driver.provider; + this.counts = options.queue; + options.store.recoverInterrupted(RESTART_ERROR, this.provider); + } + + respond(input: GatewayRunInput, requestSignal: AbortSignal): Response { + const reservation = this.options.store.reserveRun({ + provider: this.provider, + runId: input.runId, + threadId: input.threadId, + }); + if (!reservation.created) { + return Response.json( + { + error: "This provider run ID was already submitted.", + retryable: false, + run: priorRun(reservation.run), + }, + { status: 409 }, + ); + } + + const connection = new AbortController(); + const signal = mergeSignals(requestSignal, connection.signal); + let streamController!: ReadableStreamDefaultController; + let closed = false; + const close = () => { + if (closed) return; + closed = true; + streamController.close(); + }; + const stream = new ReadableStream({ + start(controller) { + streamController = controller; + }, + cancel() { + connection.abort(); + }, + }); + + try { + const completion = this.options.queue.enqueue({ + runId: input.runId, + signal, + onHeartbeat: () => { + if (!closed) + streamController.enqueue( + new TextEncoder().encode(": heartbeat\n\n"), + ); + }, + run: () => this.execute(input, signal, streamController), + }); + void completion.then(close, (error) => { + this.failInterrupted(input.runId, safeSetupFailure(error)); + if (!closed && !signal.aborted) { + streamController.enqueue( + encodeSse({ type: "RUN_ERROR", message: safeSetupFailure(error) }), + ); + } + close(); + }); + } catch (error) { + this.failInterrupted(input.runId, safeSetupFailure(error)); + close(); + if (error instanceof CapacityError) { + return Response.json( + { error: error.message, retryable: true }, + { status: 429, headers: { "retry-after": "5" } }, + ); + } + throw error; + } + + return new Response(stream, { headers: SSE_HEADERS }); + } + + private async execute( + input: GatewayRunInput, + signal: AbortSignal, + output: ReadableStreamDefaultController, + ): Promise { + let workspace: WorkspaceLease | undefined; + this.options.store.transitionRun(this.provider, input.runId, "preparing", [ + "queued", + ]); + try { + const activeWorkspace = await this.options.workspaces.create(input.runId); + workspace = activeWorkspace; + this.options.store.attachWorkspace( + this.provider, + input.runId, + workspace.path, + ); + const prepared = prepareProviderRun( + this.options.store, + this.provider, + input.threadId, + input.messages, + ); + this.options.store.transitionRun(this.provider, input.runId, "running", [ + "preparing", + ]); + let providerSessionId = prepared.sessionId; + let terminalRecorded = false; + const response = runAgent({ + input: { + ...input, + messages: prepared.messages, + workspacePath: workspace.path, + }, + driver: this.options.driver, + counts: this.options.queue, + logger: this.options.logger, + requestSignal: signal, + ...(prepared.sessionId ? { sessionId: prepared.sessionId } : {}), + onSession: (sessionId) => { + providerSessionId = sessionId; + }, + beforeTerminal: async (outcome) => { + if (terminalRecorded) return; + try { + const result = + await this.options.workspaces.captureResult(activeWorkspace); + this.options.store.saveResult(this.provider, input.runId, result); + } catch (error) { + if (outcome === "finished") throw error; + } + if (outcome === "finished" && providerSessionId) { + this.options.store.acknowledgeRun({ + provider: this.provider, + runId: input.runId, + sessionId: providerSessionId, + ...(prepared.acknowledgedMessageId + ? { acknowledgedMessageId: prepared.acknowledgedMessageId } + : {}), + }); + } + const next: RunState = + outcome === "finished" + ? "completed" + : outcome === "cancelled" + ? "cancelled" + : "failed"; + this.options.store.transitionRun( + this.provider, + input.runId, + next, + outcome === "finished" ? ["running"] : ["running", "cancelling"], + ); + terminalRecorded = true; + }, + }); + if (!response.body) + throw new Error("The provider stream did not have a body."); + const reader = response.body.getReader(); + while (true) { + const part = await reader.read(); + if (part.done) break; + output.enqueue(part.value); + } + } catch (error) { + const current = this.options.store.getRun(this.provider, input.runId); + if ( + current && + ["queued", "preparing", "running", "cancelling"].includes(current.state) + ) { + this.failInterrupted(input.runId, safeSetupFailure(error)); + } + throw error; + } + } + + private failInterrupted(runId: string, summary: string): void { + const current = this.options.store.getRun(this.provider, runId); + if ( + !current || + !["queued", "preparing", "running", "cancelling"].includes(current.state) + ) { + return; + } + this.options.store.transitionRun( + this.provider, + runId, + "failed", + [current.state], + { + errorSummary: summary, + }, + ); + } +} diff --git a/agent-subscription-gateway/src/runs/session-input.ts b/agent-subscription-gateway/src/runs/session-input.ts new file mode 100644 index 000000000..c45c6e027 --- /dev/null +++ b/agent-subscription-gateway/src/runs/session-input.ts @@ -0,0 +1,84 @@ +import type { RunStore } from "../storage/run-store"; + +export interface SanitizedMessage { + id: string; + role: "system" | "user" | "assistant"; + content: string; +} + +export interface PreparedProviderRun { + sessionId?: string; + messages: SanitizedMessage[]; + acknowledgedMessageId?: string; +} + +const MESSAGE_ID = /^[A-Za-z0-9._:-]{1,200}$/; +const ROLES = new Set(["system", "user", "assistant"]); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function textContent(value: unknown): string | undefined { + if (typeof value === "string") return value; + if (!Array.isArray(value)) return undefined; + const parts: string[] = []; + for (const item of value) { + if ( + !isRecord(item) || + item.type !== "text" || + typeof item.text !== "string" + ) { + continue; + } + parts.push(item.text); + } + return parts.join(""); +} + +export function sanitizeMessages( + messages: readonly unknown[], +): SanitizedMessage[] { + const seen = new Set(); + return messages.map((message) => { + if (!isRecord(message)) throw new Error("Invalid transcript message."); + const { id, role } = message; + const content = textContent(message.content); + if (typeof id !== "string" || !MESSAGE_ID.test(id)) { + throw new Error("Invalid transcript message ID."); + } + if (seen.has(id)) throw new Error(`Duplicate message ID: ${id}.`); + seen.add(id); + if (typeof role !== "string" || !ROLES.has(role)) { + throw new Error("Invalid transcript message role."); + } + if (content === undefined) + throw new Error("Invalid transcript message content."); + return { id, role: role as SanitizedMessage["role"], content }; + }); +} + +export function prepareProviderRun( + store: RunStore, + provider: string, + threadId: string, + rawMessages: readonly unknown[], +): PreparedProviderRun { + const messages = sanitizeMessages(rawMessages); + let session = store.getSession(provider, threadId); + let delta = messages; + const acknowledgedMessageId = session?.acknowledgedMessageId; + if (acknowledgedMessageId) { + const acknowledgedIndex = messages.findIndex( + (message) => message.id === acknowledgedMessageId, + ); + if (acknowledgedIndex >= 0) delta = messages.slice(acknowledgedIndex + 1); + else session = undefined; + } + const last = messages.at(-1); + return { + ...(session ? { sessionId: session.sessionId } : {}), + messages: delta, + ...(last ? { acknowledgedMessageId: last.id } : {}), + }; +} diff --git a/agent-subscription-gateway/src/server/app.ts b/agent-subscription-gateway/src/server/app.ts new file mode 100644 index 000000000..45faaff78 --- /dev/null +++ b/agent-subscription-gateway/src/server/app.ts @@ -0,0 +1,99 @@ +import type { AgentDriver } from "../drivers/agent-driver"; +import { InMemoryRunCounts, type RunCounts } from "../observability/run-counts"; +import { consoleRunLogger, type RunLogger } from "../observability/run-logger"; +import { gatewayStatus } from "../observability/status"; +import { hasManagedAgentToken } from "../../../shared/agent-authorisation"; +import { gatewayRunInput, type GatewayRunInput } from "./input"; +import { runAgent } from "./run-agent"; + +export interface GatewayRunService { + readonly counts: RunCounts; + respond(input: GatewayRunInput, signal: AbortSignal): Response; +} + +export interface GatewayOptions { + token: string; + driver: AgentDriver; + counts?: RunCounts; + logger?: RunLogger; + resolveSessionId?: ( + runId: string, + threadId: string, + ) => Promise; + runService?: GatewayRunService; +} + +export function createGatewayHandler(options: GatewayOptions) { + if (!options.token.trim()) { + throw new Error("The gateway token must not be empty."); + } + const counts = + options.counts ?? options.runService?.counts ?? new InMemoryRunCounts(); + const logger = options.logger ?? consoleRunLogger; + + return async function handle(request: Request): Promise { + const url = new URL(request.url); + + if (url.pathname === "/health" && request.method === "GET") { + return Response.json( + await gatewayStatus(options.driver, counts, "health"), + ); + } + + if (url.pathname === "/ready" && request.method === "GET") { + const status = await gatewayStatus(options.driver, counts, "ready"); + return Response.json(status, { status: status.authReady ? 200 : 503 }); + } + + if (url.pathname === "/ag-ui" && request.method === "POST") { + if (!hasManagedAgentToken(request, options.token)) { + return Response.json({ error: "Unauthorized." }, { status: 401 }); + } + + let rawInput: unknown; + try { + rawInput = await request.json(); + } catch { + return Response.json( + { error: "Invalid AG-UI request." }, + { status: 400 }, + ); + } + const input = gatewayRunInput(rawInput); + if (!input) { + return Response.json( + { error: "Invalid AG-UI request." }, + { status: 400 }, + ); + } + if (input.tools.length > 0) { + return Response.json( + { + error: + "OpenBot tool grants are not accepted by subscription workers.", + }, + { status: 400 }, + ); + } + + if (options.runService) { + return options.runService.respond(input, request.signal); + } + + const sessionId = await options.resolveSessionId?.( + input.runId, + input.threadId, + ); + return runAgent({ + input, + driver: options.driver, + counts, + logger, + requestSignal: request.signal, + ...(sessionId ? { sessionId } : {}), + }); + } + + return Response.json({ error: "Not found." }, { status: 404 }); + }; +} diff --git a/agent-subscription-gateway/src/server/index.ts b/agent-subscription-gateway/src/server/index.ts new file mode 100644 index 000000000..fa58f7d4e --- /dev/null +++ b/agent-subscription-gateway/src/server/index.ts @@ -0,0 +1,24 @@ +import type { AgentDriver } from "../drivers/agent-driver"; +import type { RunCounts } from "../observability/run-counts"; +import type { RunLogger } from "../observability/run-logger"; +import { createGatewayHandler, type GatewayRunService } from "./app"; + +export interface ServeGatewayOptions { + port: number; + token: string; + driver: AgentDriver; + counts?: RunCounts; + logger?: RunLogger; + resolveSessionId?: ( + runId: string, + threadId: string, + ) => Promise; + runService?: GatewayRunService; +} + +export function serveGateway(options: ServeGatewayOptions) { + const fetch = createGatewayHandler(options); + return Bun.serve({ port: options.port, idleTimeout: 120, fetch }); +} + +export { createGatewayHandler } from "./app"; diff --git a/agent-subscription-gateway/src/server/input.ts b/agent-subscription-gateway/src/server/input.ts new file mode 100644 index 000000000..cc4724e32 --- /dev/null +++ b/agent-subscription-gateway/src/server/input.ts @@ -0,0 +1,37 @@ +import type { DriverRun } from "../drivers/agent-driver"; + +export type GatewayRunInput = DriverRun & { tools: readonly unknown[] }; + +const ID = /^[A-Za-z0-9._:-]{1,200}$/; + +function record(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function gatewayRunInput(value: unknown): GatewayRunInput | undefined { + if (!record(value)) return undefined; + if (typeof value.threadId !== "string" || !ID.test(value.threadId)) { + return undefined; + } + if (typeof value.runId !== "string" || !ID.test(value.runId)) + return undefined; + if (!Array.isArray(value.messages)) return undefined; + if (value.tools !== undefined && !Array.isArray(value.tools)) + return undefined; + if (value.context !== undefined && !Array.isArray(value.context)) + return undefined; + if (value.state !== undefined && !record(value.state)) return undefined; + if (value.forwardedProps !== undefined && !record(value.forwardedProps)) { + return undefined; + } + + return { + threadId: value.threadId, + runId: value.runId, + messages: value.messages, + tools: value.tools ?? [], + context: value.context ?? [], + state: value.state ?? {}, + forwardedProps: value.forwardedProps ?? {}, + }; +} diff --git a/agent-subscription-gateway/src/server/run-agent.ts b/agent-subscription-gateway/src/server/run-agent.ts new file mode 100644 index 000000000..16130cb83 --- /dev/null +++ b/agent-subscription-gateway/src/server/run-agent.ts @@ -0,0 +1,205 @@ +import { + type AgentDriver, + type DriverEvent, + type DriverRun, + type DriverResumeRun, + SafeDriverError, +} from "../drivers/agent-driver"; +import type { RunCounts } from "../observability/run-counts"; +import type { RunLogger } from "../observability/run-logger"; +import type { GatewayRunInput } from "./input"; +import { type AgUiEvent, encodeSse, SSE_HEADERS } from "./sse"; + +class CancelledRun extends Error {} + +function safeFailure(error: unknown): string { + return error instanceof SafeDriverError + ? error.publicMessage + : "The provider run failed."; +} + +async function nextOrCancel( + iterator: AsyncIterator, + signal: AbortSignal, +): Promise> { + if (signal.aborted) throw new CancelledRun(); + + return new Promise>((resolve, reject) => { + const cancelled = () => reject(new CancelledRun()); + signal.addEventListener("abort", cancelled, { once: true }); + iterator.next().then( + (result) => { + signal.removeEventListener("abort", cancelled); + if (signal.aborted) reject(new CancelledRun()); + else resolve(result); + }, + (error) => { + signal.removeEventListener("abort", cancelled); + reject(error); + }, + ); + }); +} + +export function runAgent(options: { + input: GatewayRunInput; + driver: AgentDriver; + counts: RunCounts; + logger: RunLogger; + requestSignal: AbortSignal; + sessionId?: string; + onSession?: (sessionId: string) => void | Promise; + beforeTerminal?: ( + outcome: "finished" | "failed" | "cancelled", + ) => void | Promise; +}): Response { + const { + input, + driver, + counts, + logger, + requestSignal, + sessionId, + onSession, + beforeTerminal, + } = options; + const run: DriverRun = { + threadId: input.threadId, + runId: input.runId, + messages: input.messages, + context: input.context, + state: input.state, + forwardedProps: input.forwardedProps, + ...(input.workspacePath ? { workspacePath: input.workspacePath } : {}), + }; + const driverController = new AbortController(); + let cancelPromise: Promise | undefined; + const cancel = () => { + if (cancelPromise) return cancelPromise; + driverController.abort(); + cancelPromise = driver.cancel(input.runId).catch(() => undefined); + return cancelPromise; + }; + + const stream = new ReadableStream({ + start(controller) { + const send = (event: AgUiEvent) => controller.enqueue(encodeSse(event)); + const pump = async () => { + counts.started(); + let messageIndex = 0; + let messageOpen = false; + const messageId = () => `msg_${input.runId}_${messageIndex}`; + const closeMessage = () => { + if (!messageOpen) return; + send({ type: "TEXT_MESSAGE_END", messageId: messageId() }); + messageOpen = false; + messageIndex += 1; + }; + const onRequestAbort = () => void cancel(); + requestSignal.addEventListener("abort", onRequestAbort, { once: true }); + + send({ + type: "RUN_STARTED", + threadId: input.threadId, + runId: input.runId, + }); + + let outcome: "finished" | "failed" | "cancelled" = "failed"; + let iterator: AsyncIterator | undefined; + try { + const context = { signal: driverController.signal }; + const events = sessionId + ? driver.resume( + { ...run, sessionId } satisfies DriverResumeRun, + context, + ) + : driver.start(run, context); + iterator = events[Symbol.asyncIterator](); + + while (true) { + const next = await nextOrCancel(iterator, driverController.signal); + if (next.done) break; + const event = next.value; + if (event.type === "session") { + await onSession?.(event.sessionId); + continue; + } + if (event.type === "text") { + if (!event.delta) continue; + if (!messageOpen) { + send({ + type: "TEXT_MESSAGE_START", + messageId: messageId(), + role: "assistant", + }); + messageOpen = true; + } + send({ + type: "TEXT_MESSAGE_CONTENT", + messageId: messageId(), + delta: event.delta, + }); + continue; + } + if (event.type === "activity") { + closeMessage(); + send({ + type: "CUSTOM", + name: "subscription_agent_activity", + value: { + provider: driver.provider, + message: event.message, + ...(event.data ? { data: event.data } : {}), + }, + }); + continue; + } + throw new Error("Unsupported driver event."); + } + + if (driverController.signal.aborted) throw new CancelledRun(); + closeMessage(); + await beforeTerminal?.("finished"); + send({ + type: "RUN_FINISHED", + threadId: input.threadId, + runId: input.runId, + }); + outcome = "finished"; + } catch (error) { + closeMessage(); + if ( + error instanceof CancelledRun || + driverController.signal.aborted + ) { + await cancel(); + await beforeTerminal?.("cancelled"); + send({ type: "RUN_ERROR", message: "The run was cancelled." }); + outcome = "cancelled"; + } else { + await beforeTerminal?.("failed"); + send({ type: "RUN_ERROR", message: safeFailure(error) }); + outcome = "failed"; + } + } finally { + requestSignal.removeEventListener("abort", onRequestAbort); + if (outcome === "cancelled") void iterator?.return?.(); + counts.finished(); + logger.terminal({ + provider: driver.provider, + runId: input.runId, + outcome, + }); + controller.close(); + } + }; + + void pump(); + }, + cancel() { + return cancel(); + }, + }); + + return new Response(stream, { headers: SSE_HEADERS }); +} diff --git a/agent-subscription-gateway/src/server/sse.ts b/agent-subscription-gateway/src/server/sse.ts new file mode 100644 index 000000000..fab99caa6 --- /dev/null +++ b/agent-subscription-gateway/src/server/sse.ts @@ -0,0 +1,11 @@ +export type AgUiEvent = Record & { type: string }; + +export function encodeSse(event: AgUiEvent): Uint8Array { + return new TextEncoder().encode(`data: ${JSON.stringify(event)}\n\n`); +} + +export const SSE_HEADERS = { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", +} as const; diff --git a/agent-subscription-gateway/src/storage/run-store.ts b/agent-subscription-gateway/src/storage/run-store.ts new file mode 100644 index 000000000..a246de2e8 --- /dev/null +++ b/agent-subscription-gateway/src/storage/run-store.ts @@ -0,0 +1,447 @@ +import { Database } from "bun:sqlite"; + +export type RunState = + | "queued" + | "preparing" + | "running" + | "cancelling" + | "completed" + | "failed" + | "cancelled"; + +export interface CommitMetadata { + hash: string; + subject: string; + parents?: readonly string[]; + authorName?: string; + authorEmail?: string; + authoredAt?: string; +} + +export interface StoredRun { + provider: string; + runId: string; + threadId: string; + state: RunState; + workspacePath?: string; + providerSessionId?: string; + acknowledgedMessageId?: string; + errorSummary?: string; + patch?: string; + baseCommit?: string; + commits: CommitMetadata[]; + createdAt: number; + updatedAt: number; +} + +export interface StoredSession { + provider: string; + threadId: string; + sessionId: string; + acknowledgedMessageId?: string; + updatedAt: number; +} + +interface RunRow { + provider: string; + run_id: string; + thread_id: string; + state: RunState; + workspace_path: string | null; + provider_session_id: string | null; + acknowledged_message_id: string | null; + error_summary: string | null; + patch: string | null; + base_commit: string | null; + commits_json: string; + created_at: number; + updated_at: number; +} + +interface SessionRow { + provider: string; + thread_id: string; + session_id: string; + acknowledged_message_id: string | null; + updated_at: number; +} + +const SAFE_ID = /^[A-Za-z0-9._:-]{1,200}$/; +const INTERRUPTED_STATES: readonly RunState[] = [ + "queued", + "preparing", + "running", + "cancelling", +]; +const NEXT_STATES: Record = { + queued: ["preparing", "failed", "cancelled"], + preparing: ["running", "failed", "cancelled"], + running: ["cancelling", "completed", "failed", "cancelled"], + cancelling: ["cancelled", "failed"], + completed: [], + failed: [], + cancelled: [], +}; + +function requireId(label: string, value: string): void { + if (!SAFE_ID.test(value)) throw new Error(`Invalid ${label}.`); +} + +function optional(value: T | null): T | undefined { + return value === null ? undefined : value; +} + +function parseCommits(value: string): CommitMetadata[] { + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? (parsed as CommitMetadata[]) : []; + } catch { + return []; + } +} + +function storedRun(row: RunRow): StoredRun { + return { + provider: row.provider, + runId: row.run_id, + threadId: row.thread_id, + state: row.state, + ...(row.workspace_path ? { workspacePath: row.workspace_path } : {}), + ...(row.provider_session_id + ? { providerSessionId: row.provider_session_id } + : {}), + ...(row.acknowledged_message_id + ? { acknowledgedMessageId: row.acknowledged_message_id } + : {}), + ...(row.error_summary ? { errorSummary: row.error_summary } : {}), + ...(row.patch !== null ? { patch: row.patch } : {}), + ...(row.base_commit ? { baseCommit: row.base_commit } : {}), + commits: parseCommits(row.commits_json), + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +export class RunStore { + readonly path: string; + private readonly database: Database; + + constructor(path: string) { + this.path = path; + this.database = new Database(path, { create: true }); + this.database.run("PRAGMA journal_mode = WAL"); + this.database.run("PRAGMA foreign_keys = ON"); + this.database.run("PRAGMA busy_timeout = 5000"); + this.database.run(` + CREATE TABLE IF NOT EXISTS runs ( + provider TEXT NOT NULL, + run_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + state TEXT NOT NULL, + workspace_path TEXT, + provider_session_id TEXT, + acknowledged_message_id TEXT, + error_summary TEXT, + patch TEXT, + base_commit TEXT, + commits_json TEXT NOT NULL DEFAULT '[]', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (provider, run_id) + ) + `); + this.database.run(` + CREATE TABLE IF NOT EXISTS sessions ( + provider TEXT NOT NULL, + thread_id TEXT NOT NULL, + session_id TEXT NOT NULL, + acknowledged_message_id TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (provider, thread_id) + ) + `); + this.database.run(` + CREATE TABLE IF NOT EXISTS host_leases ( + owner TEXT PRIMARY KEY, + acquired_at INTEGER NOT NULL + ) + `); + } + + reserveRun(input: { provider: string; runId: string; threadId: string }): { + created: boolean; + run: StoredRun; + } { + requireId("provider", input.provider); + requireId("run ID", input.runId); + requireId("thread ID", input.threadId); + return this.database.transaction(() => { + const existing = this.getRun(input.provider, input.runId); + if (existing) return { created: false, run: existing }; + const now = Date.now(); + this.database + .query( + `INSERT INTO runs + (provider, run_id, thread_id, state, created_at, updated_at) + VALUES (?, ?, ?, 'queued', ?, ?)`, + ) + .run(input.provider, input.runId, input.threadId, now, now); + const run = this.getRun(input.provider, input.runId); + if (!run) throw new Error("Failed to reserve the run."); + return { created: true, run }; + })(); + } + + getRun(provider: string, runId: string): StoredRun | undefined { + const row = this.database + .query("SELECT * FROM runs WHERE provider = ? AND run_id = ?") + .get(provider, runId) as RunRow | null; + return row ? storedRun(row) : undefined; + } + + listRuns(): StoredRun[] { + return ( + this.database + .query("SELECT * FROM runs ORDER BY created_at, provider, run_id") + .all() as RunRow[] + ).map(storedRun); + } + + transitionRun( + provider: string, + runId: string, + next: RunState, + allowedFrom: readonly RunState[], + options: { errorSummary?: string } = {}, + ): StoredRun { + if (allowedFrom.length === 0) + throw new Error("A guarded transition needs a source state."); + if (allowedFrom.some((state) => !NEXT_STATES[state].includes(next))) { + throw new Error(`Invalid run transition to ${next}.`); + } + const placeholders = allowedFrom.map(() => "?").join(", "); + const now = Date.now(); + const result = this.database + .query( + `UPDATE runs + SET state = ?, error_summary = COALESCE(?, error_summary), updated_at = ? + WHERE provider = ? AND run_id = ? AND state IN (${placeholders})`, + ) + .run( + next, + options.errorSummary ?? null, + now, + provider, + runId, + ...allowedFrom, + ); + if (result.changes !== 1) { + throw new Error(`Invalid run transition to ${next}.`); + } + const run = this.getRun(provider, runId); + if (!run) throw new Error("Run disappeared after transition."); + return run; + } + + attachWorkspace( + provider: string, + runId: string, + workspacePath: string, + ): void { + const result = this.database + .query( + "UPDATE runs SET workspace_path = ?, updated_at = ? WHERE provider = ? AND run_id = ?", + ) + .run(workspacePath, Date.now(), provider, runId); + if (result.changes !== 1) throw new Error("Unknown run."); + } + + saveResult( + provider: string, + runId: string, + result: { + patch: string; + baseCommit: string; + commits: readonly CommitMetadata[]; + }, + ): void { + const updated = this.database + .query( + `UPDATE runs + SET patch = ?, base_commit = ?, commits_json = ?, updated_at = ? + WHERE provider = ? AND run_id = ?`, + ) + .run( + result.patch, + result.baseCommit, + JSON.stringify(result.commits), + Date.now(), + provider, + runId, + ); + if (updated.changes !== 1) throw new Error("Unknown run."); + } + + saveSession(input: { + provider: string; + threadId: string; + sessionId: string; + acknowledgedMessageId?: string; + }): void { + requireId("provider", input.provider); + requireId("thread ID", input.threadId); + requireId("provider session ID", input.sessionId); + if (input.acknowledgedMessageId) { + requireId("acknowledged message ID", input.acknowledgedMessageId); + } + this.database + .query( + `INSERT INTO sessions + (provider, thread_id, session_id, acknowledged_message_id, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(provider, thread_id) DO UPDATE SET + session_id = excluded.session_id, + acknowledged_message_id = excluded.acknowledged_message_id, + updated_at = excluded.updated_at`, + ) + .run( + input.provider, + input.threadId, + input.sessionId, + input.acknowledgedMessageId ?? null, + Date.now(), + ); + } + + getSession(provider: string, threadId: string): StoredSession | undefined { + const row = this.database + .query("SELECT * FROM sessions WHERE provider = ? AND thread_id = ?") + .get(provider, threadId) as SessionRow | null; + if (!row || !SAFE_ID.test(row.session_id)) return undefined; + const acknowledgedMessageId = optional(row.acknowledged_message_id); + return { + provider: row.provider, + threadId: row.thread_id, + sessionId: row.session_id, + ...(acknowledgedMessageId ? { acknowledgedMessageId } : {}), + updatedAt: row.updated_at, + }; + } + + acknowledgeRun(input: { + provider: string; + runId: string; + sessionId: string; + acknowledgedMessageId?: string; + }): void { + const run = this.getRun(input.provider, input.runId); + if (!run) throw new Error("Unknown run."); + this.database.transaction(() => { + this.saveSession({ + provider: input.provider, + threadId: run.threadId, + sessionId: input.sessionId, + ...(input.acknowledgedMessageId + ? { acknowledgedMessageId: input.acknowledgedMessageId } + : {}), + }); + this.database + .query( + `UPDATE runs SET provider_session_id = ?, acknowledged_message_id = ?, updated_at = ? + WHERE provider = ? AND run_id = ?`, + ) + .run( + input.sessionId, + input.acknowledgedMessageId ?? null, + Date.now(), + input.provider, + input.runId, + ); + })(); + } + + recoverInterrupted(errorSummary: string, provider?: string): number { + const placeholders = INTERRUPTED_STATES.map(() => "?").join(", "); + return this.database.transaction(() => { + const providerClause = provider ? " AND provider = ?" : ""; + const result = this.database + .query( + `UPDATE runs SET state = 'failed', error_summary = ?, updated_at = ? + WHERE state IN (${placeholders})${providerClause}`, + ) + .run( + errorSummary, + Date.now(), + ...INTERRUPTED_STATES, + ...(provider ? [provider] : []), + ); + if (provider) { + this.database + .query("DELETE FROM host_leases WHERE owner LIKE ?") + .run(`${provider}:%`); + } else { + this.database.run("DELETE FROM host_leases"); + } + return result.changes; + })(); + } + + tryAcquireHostLease(owner: string, limit: number): boolean { + requireId("host lease owner", owner); + if (!Number.isInteger(limit) || limit < 1) + throw new Error("Invalid host lease limit."); + return this.database.transaction(() => { + const existing = this.database + .query("SELECT owner FROM host_leases WHERE owner = ?") + .get(owner); + if (existing) return true; + const count = this.database + .query("SELECT COUNT(*) AS count FROM host_leases") + .get() as { + count: number; + }; + if (count.count >= limit) return false; + this.database + .query("INSERT INTO host_leases (owner, acquired_at) VALUES (?, ?)") + .run(owner, Date.now()); + return true; + })(); + } + + releaseHostLease(owner: string): void { + this.database.query("DELETE FROM host_leases WHERE owner = ?").run(owner); + } + + listWorkspaceCleanupCandidates(before: number): StoredRun[] { + return ( + this.database + .query( + `SELECT * FROM runs + WHERE workspace_path IS NOT NULL + AND state IN ('completed', 'failed', 'cancelled') + AND updated_at < ? + ORDER BY updated_at, provider, run_id`, + ) + .all(before) as RunRow[] + ).map(storedRun); + } + + clearWorkspace( + provider: string, + runId: string, + workspacePath: string, + ): boolean { + const result = this.database + .query( + `UPDATE runs SET workspace_path = NULL, updated_at = ? + WHERE provider = ? AND run_id = ? AND workspace_path = ? + AND state IN ('completed', 'failed', 'cancelled')`, + ) + .run(Date.now(), provider, runId, workspacePath); + return result.changes === 1; + } + + close(): void { + this.database.close(); + } +} diff --git a/agent-subscription-gateway/src/workspaces/cleanup.ts b/agent-subscription-gateway/src/workspaces/cleanup.ts new file mode 100644 index 000000000..1415e0b7e --- /dev/null +++ b/agent-subscription-gateway/src/workspaces/cleanup.ts @@ -0,0 +1,18 @@ +import type { RunStore } from "../storage/run-store"; +import type { WorkspaceManager } from "./workspace-manager"; + +export async function cleanupExpiredWorkspaces( + store: RunStore, + manager: Pick, + before: number, +): Promise { + const removed: string[] = []; + for (const run of store.listWorkspaceCleanupCandidates(before)) { + if (!run.workspacePath) continue; + await manager.remove(run.workspacePath); + if (store.clearWorkspace(run.provider, run.runId, run.workspacePath)) { + removed.push(run.workspacePath); + } + } + return removed; +} diff --git a/agent-subscription-gateway/src/workspaces/workspace-manager.ts b/agent-subscription-gateway/src/workspaces/workspace-manager.ts new file mode 100644 index 000000000..05632a76d --- /dev/null +++ b/agent-subscription-gateway/src/workspaces/workspace-manager.ts @@ -0,0 +1,255 @@ +import { mkdir, realpath, rm } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import { promisify } from "node:util"; +import type { CommitMetadata } from "../storage/run-store"; + +const SAFE_SEGMENT = /^[A-Za-z0-9._:-]{1,200}$/; +const execFileAsync = promisify(execFile); +const PUSH_ENVIRONMENT = new Set([ + "GH_TOKEN", + "GITHUB_TOKEN", + "GIT_ASKPASS", + "SSH_ASKPASS", + "GIT_SSH", + "GIT_SSH_COMMAND", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_SECURITY_TOKEN", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "DOCKER_HOST", + "DOCKER_CONTEXT", +]); + +export interface WorkspaceLease { + provider: string; + runId: string; + path: string; + baseCommit: string; +} + +export interface WorkspaceResult { + patch: string; + baseCommit: string; + commits: CommitMetadata[]; +} + +export function scrubProviderEnvironment( + environment: Record, +): Record { + const scrubbed: Record = {}; + for (const [name, value] of Object.entries(environment)) { + if (value === undefined) continue; + if (PUSH_ENVIRONMENT.has(name) || name.startsWith("GIT_CONFIG_")) continue; + scrubbed[name] = value; + } + scrubbed.GIT_TERMINAL_PROMPT = "0"; + return scrubbed; +} + +function assertSafeSegment(label: string, value: string): void { + if (!SAFE_SEGMENT.test(value) || value === "." || value === "..") { + throw new Error(`Invalid ${label}.`); + } +} + +async function command( + args: readonly string[], + options: { cwd?: string; env?: Record } = {}, +): Promise { + const [program, ...rawProgramArgs] = args; + if (!program) throw new Error("Workspace command is empty."); + const programArgs = + program === "git" + ? ["-c", "core.hooksPath=/dev/null", ...rawProgramArgs] + : rawProgramArgs; + try { + const { stdout } = await execFileAsync(program, programArgs, { + ...(options.cwd ? { cwd: options.cwd } : {}), + ...(options.env ? { env: options.env } : {}), + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + return stdout; + } catch { + throw new Error("Git workspace operation failed."); + } +} + +function inside(root: string, candidate: string): boolean { + const path = relative(root, candidate); + return ( + path !== "" && + path !== ".." && + !path.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) && + !isAbsolute(path) + ); +} + +export class WorkspaceManager { + private readonly provider: string; + private readonly repository: string; + private readonly baseBranch: string; + private readonly jobRoot: string; + private readonly gitEnvironment: Record; + + constructor(options: { + provider: string; + repository: string; + baseBranch: string; + jobRoot: string; + gitEnvironment?: Record; + }) { + assertSafeSegment("provider", options.provider); + assertSafeSegment("base branch", options.baseBranch); + if (!options.repository.trim()) throw new Error("Repository is required."); + if (/^https?:\/\//i.test(options.repository)) { + const repositoryUrl = new URL(options.repository); + if (repositoryUrl.username || repositoryUrl.password) { + throw new Error("Repository URL must not contain credentials."); + } + } + if (!isAbsolute(options.jobRoot)) + throw new Error("Job root must be absolute."); + this.provider = options.provider; + this.repository = options.repository; + this.baseBranch = options.baseBranch; + this.jobRoot = resolve(options.jobRoot); + this.gitEnvironment = scrubProviderEnvironment({ + ...process.env, + ...options.gitEnvironment, + }); + } + + pathFor(runId: string): string { + assertSafeSegment("run ID", runId); + const candidate = resolve(this.jobRoot, this.provider, runId); + if (!inside(this.jobRoot, candidate)) + throw new Error("Workspace escaped the job root."); + return candidate; + } + + async create(runId: string): Promise { + const target = this.pathFor(runId); + await mkdir(join(this.jobRoot, this.provider), { + recursive: true, + mode: 0o700, + }); + const cloneIsolation = isAbsolute(this.repository) + ? ["--no-hardlinks"] + : ["--no-local"]; + try { + await command( + [ + "git", + "clone", + ...cloneIsolation, + "--single-branch", + "--branch", + this.baseBranch, + "--no-checkout", + "--", + this.repository, + target, + ], + { env: this.gitEnvironment }, + ); + await command(["git", "checkout", "--detach", this.baseBranch], { + cwd: target, + env: this.gitEnvironment, + }); + await command( + [ + "git", + "remote", + "set-url", + "--push", + "origin", + "disabled://push-not-allowed", + ], + { cwd: target, env: this.gitEnvironment }, + ); + } catch (error) { + await rm(target, { recursive: true, force: true }); + throw error; + } + const canonicalRoot = await realpath(this.jobRoot); + const canonicalTarget = await realpath(target); + if (!inside(canonicalRoot, canonicalTarget)) { + await rm(target, { recursive: true, force: true }); + throw new Error("Workspace escaped the canonical job root."); + } + const baseCommit = ( + await command(["git", "rev-parse", "HEAD"], { cwd: target }) + ).trim(); + return { provider: this.provider, runId, path: target, baseCommit }; + } + + async captureResult(workspace: WorkspaceLease): Promise { + await this.assertOwned(workspace.path); + const untracked = await command( + ["git", "ls-files", "--others", "--exclude-standard", "-z"], + { cwd: workspace.path }, + ); + const untrackedPaths = untracked.split("\0").filter(Boolean); + if (untrackedPaths.length > 0) { + await command( + ["git", "add", "--intent-to-add", "--", ...untrackedPaths], + { + cwd: workspace.path, + }, + ); + } + const patch = await command( + ["git", "diff", "--binary", "--no-ext-diff", workspace.baseCommit, "--"], + { cwd: workspace.path }, + ); + const log = await command( + [ + "git", + "log", + "--format=%H%x1f%P%x1f%an%x1f%ae%x1f%aI%x1f%s%x1e", + `${workspace.baseCommit}..HEAD`, + "--", + ], + { cwd: workspace.path }, + ); + const commits = log + .split("\u001e") + .map((entry) => entry.trim()) + .filter(Boolean) + .map((entry): CommitMetadata => { + const [hash, parents, authorName, authorEmail, authoredAt, subject] = + entry.split("\u001f"); + return { + hash: hash ?? "", + subject: subject ?? "", + parents: parents ? parents.split(" ").filter(Boolean) : [], + authorName, + authorEmail, + authoredAt, + }; + }); + return { patch, baseCommit: workspace.baseCommit, commits }; + } + + async remove(path: string): Promise { + await this.assertOwned(path); + await rm(path, { recursive: true, force: true }); + } + + private async assertOwned(path: string): Promise { + const candidate = resolve(path); + if (!inside(this.jobRoot, candidate)) { + throw new Error("Workspace is outside the configured job root."); + } + const canonicalRoot = await realpath(this.jobRoot); + const canonicalCandidate = await realpath(candidate); + if (!inside(canonicalRoot, canonicalCandidate)) { + throw new Error("Workspace is outside the canonical job root."); + } + } +} diff --git a/agent-subscription-gateway/tests/ag-ui.test.ts b/agent-subscription-gateway/tests/ag-ui.test.ts new file mode 100644 index 000000000..f131b06f0 --- /dev/null +++ b/agent-subscription-gateway/tests/ag-ui.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, test } from "bun:test"; +import type { + AgentDriver, + DriverEvent, + DriverRun, + DriverRunContext, +} from "../src/drivers/agent-driver"; +import { createGatewayHandler } from "../src/server/app"; + +function request( + token = "worker-secret", + overrides: Record = {}, + signal?: AbortSignal, +) { + return new Request("http://gateway.test/ag-ui", { + method: "POST", + headers: { + "content-type": "application/json", + "x-openbot-agent-token": token, + }, + body: JSON.stringify({ + threadId: "thread-1", + runId: "run-1", + messages: [{ id: "message-1", role: "user", content: "Change it." }], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + ...overrides, + }), + signal, + }); +} + +function events(response: Response) { + return response.text().then((body) => + body + .split("\n") + .filter((line) => line.startsWith("data: ")) + .map( + (line) => + JSON.parse(line.slice("data: ".length)) as Record, + ), + ); +} + +class FakeDriver implements AgentDriver { + readonly provider = "fake"; + readonly version = "1.2.3"; + starts: DriverRun[] = []; + cancels: string[] = []; + output: DriverEvent[] = [ + { type: "activity", message: "Reading files", data: { phase: "inspect" } }, + { type: "text", delta: "Done." }, + ]; + failure: unknown; + + async isAuthReady() { + return true; + } + + async *start(run: DriverRun, _context: DriverRunContext) { + this.starts.push(run); + if (this.failure) throw this.failure; + for (const event of this.output) yield event; + } + + async *resume(run: DriverRun, context: DriverRunContext) { + yield* this.start(run, context); + } + + async cancel(runId: string) { + this.cancels.push(runId); + } +} + +describe("the AG-UI gateway", () => { + test("streams a native run in AG-UI order without tool-call events", async () => { + const driver = new FakeDriver(); + const handle = createGatewayHandler({ token: "worker-secret", driver }); + + const response = await handle(request()); + const sent = await events(response); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/event-stream"); + expect(sent.map((event) => event.type)).toEqual([ + "RUN_STARTED", + "CUSTOM", + "TEXT_MESSAGE_START", + "TEXT_MESSAGE_CONTENT", + "TEXT_MESSAGE_END", + "RUN_FINISHED", + ]); + expect(sent[1]).toMatchObject({ + name: "subscription_agent_activity", + value: { + provider: "fake", + message: "Reading files", + data: { phase: "inspect" }, + }, + }); + expect(sent.map((event) => String(event.type))).not.toContain( + "TOOL_CALL_START", + ); + expect(driver.starts).toHaveLength(1); + expect(driver.starts[0]).toMatchObject({ + runId: "run-1", + threadId: "thread-1", + }); + }); + + test("turns a driver failure into one safe terminal error", async () => { + const driver = new FakeDriver(); + driver.failure = new Error( + "provider failed with PRIVATE_PROVIDER_VALUE from /auth/secret.json", + ); + const handle = createGatewayHandler({ token: "worker-secret", driver }); + + const sent = await events(await handle(request())); + + expect(sent.map((event) => event.type)).toEqual([ + "RUN_STARTED", + "RUN_ERROR", + ]); + expect(sent[1]).toEqual({ + type: "RUN_ERROR", + message: "The provider run failed.", + }); + expect(JSON.stringify(sent)).not.toContain("PRIVATE_PROVIDER_VALUE"); + expect(JSON.stringify(sent)).not.toContain("secret.json"); + }); + + test("rejects OpenBot tool grants before native work starts", async () => { + const driver = new FakeDriver(); + const handle = createGatewayHandler({ token: "worker-secret", driver }); + + const response = await handle( + request("worker-secret", { + tools: [ + { + name: "computer_click", + description: "Click a control", + parameters: { type: "object" }, + }, + ], + }), + ); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "OpenBot tool grants are not accepted by subscription workers.", + }); + expect(driver.starts).toHaveLength(0); + }); + + test("cancels the driver once and emits one terminal event", async () => { + const driver = new FakeDriver(); + driver.start = async function* (run: DriverRun, context: DriverRunContext) { + this.starts.push(run); + yield { type: "activity", message: "Working" }; + await new Promise((resolve) => { + context.signal.addEventListener("abort", () => resolve(), { + once: true, + }); + }); + }; + const handle = createGatewayHandler({ token: "worker-secret", driver }); + const controller = new AbortController(); + const response = await handle( + request("worker-secret", {}, controller.signal), + ); + + controller.abort(); + const sent = await events(response); + + expect(driver.cancels).toEqual(["run-1"]); + expect(sent.filter((event) => event.type === "RUN_ERROR")).toEqual([ + { type: "RUN_ERROR", message: "The run was cancelled." }, + ]); + expect(sent.some((event) => event.type === "RUN_FINISHED")).toBe(false); + }); +}); diff --git a/agent-subscription-gateway/tests/auth.test.ts b/agent-subscription-gateway/tests/auth.test.ts new file mode 100644 index 000000000..cbde58cab --- /dev/null +++ b/agent-subscription-gateway/tests/auth.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "bun:test"; +import type { AgentDriver, DriverEvent } from "../src/drivers/agent-driver"; +import { createGatewayHandler } from "../src/server/app"; + +function driver() { + let starts = 0; + const value: AgentDriver = { + provider: "fake", + version: "1.0.0", + async isAuthReady() { + return true; + }, + async *start() { + starts += 1; + for (const event of [] as DriverEvent[]) yield event; + }, + async *resume() { + starts += 1; + for (const event of [] as DriverEvent[]) yield event; + }, + async cancel() {}, + }; + return { value, starts: () => starts }; +} + +function runRequest(token?: string) { + const headers = new Headers({ "content-type": "application/json" }); + if (token !== undefined) headers.set("x-openbot-agent-token", token); + return new Request("http://gateway.test/ag-ui", { + method: "POST", + headers, + body: JSON.stringify({ + threadId: "thread-1", + runId: "run-1", + messages: [], + tools: [], + }), + }); +} + +test.each([undefined, "", "wrong-token"])( + "rejects a missing or wrong managed token before calling the driver", + async (token) => { + const fake = driver(); + const handle = createGatewayHandler({ + token: "worker-secret", + driver: fake.value, + }); + + const response = await handle(runRequest(token)); + + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ error: "Unauthorized." }); + expect(fake.starts()).toBe(0); + }, +); diff --git a/agent-subscription-gateway/tests/claude-driver.test.ts b/agent-subscription-gateway/tests/claude-driver.test.ts new file mode 100644 index 000000000..78ea7504f --- /dev/null +++ b/agent-subscription-gateway/tests/claude-driver.test.ts @@ -0,0 +1,355 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { constants } from "node:os"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { + DriverEvent, + DriverRun, + DriverRunContext, +} from "../src/drivers/agent-driver"; +import { + ClaudeDriver, + type ClaudeChildProcess, + type ClaudeProcessFactory, + type ClaudeSpawnOptions, +} from "../src/drivers/claude/claude-driver"; + +const fixtures = join(import.meta.dir, "fixtures/claude-stream"); +const directories: string[] = []; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { recursive: true })), + ); +}); + +function stream(text: string): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }); +} + +function processFor(output: string, exitCode = 0): ClaudeChildProcess { + return { + stdout: stream(output), + stderr: stream("PRIVATE_STDERR\n"), + exited: Promise.resolve(exitCode), + kill() {}, + }; +} + +async function harness( + fixture = "success.jsonl", + loginExitCode = 0, + environment: Record = {}, +) { + const directory = await mkdtemp(join(tmpdir(), "openbot-claude-driver-")); + directories.push(directory); + const output = await readFile(join(fixtures, fixture), "utf8"); + const spawns: ClaudeSpawnOptions[] = []; + const processFactory: ClaudeProcessFactory = (options) => { + spawns.push(options); + if (options.command.slice(1).join(" ") === "auth status") { + return processFor( + '{"loggedIn":true,"private":"DO_NOT_PARSE"}\n', + loginExitCode, + ); + } + return processFor(output); + }; + const driver = new ClaudeDriver({ + processFactory, + binary: "/usr/local/bin/claude", + environment, + }); + return { + driver, + spawns, + run: { ...run, workspacePath: directory } satisfies DriverRun, + }; +} + +const run: DriverRun = { + threadId: "openbot-thread-1", + runId: "openbot-run-1", + messages: [ + { id: "m1", role: "system", content: "Use the repository rules." }, + { id: "m2", role: "user", content: "Run the focused tests." }, + ], + context: [], + state: {}, + forwardedProps: {}, + workspacePath: "/workspaces/claude/openbot-run-1", +}; + +const context = (signal = new AbortController().signal): DriverRunContext => ({ + signal, +}); + +async function collect( + events: AsyncIterable, +): Promise { + const result: DriverEvent[] = []; + for await (const event of events) result.push(event); + return result; +} + +describe("ClaudeDriver", () => { + test("runs the unmodified print CLI and maps text and native tool telemetry", async () => { + const { driver, spawns, run } = await harness(); + + const result = await collect(driver.start(run, context())); + + expect(spawns[0]).toMatchObject({ cwd: run.workspacePath }); + expect(spawns[0]?.command.slice(0, 2)).toEqual([ + "/usr/local/bin/claude", + "-p", + ]); + expect(spawns[0]?.command).toEqual( + expect.arrayContaining([ + "--output-format", + "stream-json", + "--verbose", + "--include-partial-messages", + "--permission-mode", + "dontAsk", + ]), + ); + expect(spawns[0]?.command.join(" ")).not.toContain( + "dangerously-skip-permissions", + ); + expect(spawns[0]?.command.join(" ")).toContain("[system m1]"); + expect(result).toEqual([ + { type: "session", sessionId: "claude-session-new" }, + { + type: "activity", + message: "Bash started", + data: { nativeTool: "Bash" }, + }, + { type: "text", delta: "Done" }, + { type: "text", delta: "." }, + { type: "activity", message: "Native tool completed" }, + ]); + expect(JSON.stringify(result)).not.toContain("PRIVATE_COMMAND"); + expect(JSON.stringify(result)).not.toContain("PRIVATE_OUTPUT"); + expect(JSON.stringify(result)).not.toContain("TOOL_CALL"); + }); + + test("resumes one stored session with only the supplied transcript delta", async () => { + const { driver, spawns, run } = await harness(); + const resumed = { + ...run, + messages: [{ id: "m3", role: "user", content: "Fix the next test." }], + sessionId: "claude-session-existing", + }; + + const result = await collect(driver.resume(resumed, context())); + + expect(spawns[0]?.command).toEqual( + expect.arrayContaining(["--resume", "claude-session-existing"]), + ); + expect(spawns[0]?.command.join(" ")).toContain("Fix the next test."); + expect(spawns[0]?.command.join(" ")).not.toContain( + "Run the focused tests.", + ); + expect(result.some((event) => event.type === "session")).toBe(false); + }); + + test("reports rate-limit retry activity and hides provider error details", async () => { + const { driver, run } = await harness("rate-limit.jsonl"); + const events: DriverEvent[] = []; + + try { + for await (const event of driver.start(run, context())) + events.push(event); + throw new Error("Expected the Claude run to fail."); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe("The Claude run failed."); + } + expect(events).toContainEqual({ + type: "activity", + message: "Claude is retrying after a rate limit", + data: { attempt: 2, maxRetries: 5 }, + }); + expect(JSON.stringify(events)).not.toContain("PRIVATE_RATE_LIMIT_DETAILS"); + }); + + test("fails closed when the CLI reports a permission denial", async () => { + const { driver, run } = await harness("permission-denied.jsonl"); + + await expect(collect(driver.start(run, context()))).rejects.toThrow( + "The Claude run requested permission beyond worker policy.", + ); + }); + + test("passes only a small non-secret environment to Claude", async () => { + const { driver, spawns, run } = await harness("success.jsonl", 0, { + PATH: "/usr/local/bin:/usr/bin", + HOME: "/home/gateway", + CLAUDE_CONFIG_DIR: "/home/gateway/.claude", + LANG: "C.UTF-8", + GATEWAY_TOKEN: "gateway-canary", + ANTHROPIC_API_KEY: "api-canary", + CLAUDE_CODE_OAUTH_TOKEN: "oauth-canary", + AWS_SECRET_ACCESS_KEY: "aws-canary", + SECRET_CANARY: "secret-canary", + }); + + await collect(driver.start(run, context())); + + expect(spawns[0]?.env).toMatchObject({ + PATH: "/usr/local/bin:/usr/bin", + HOME: "/home/gateway", + CLAUDE_CONFIG_DIR: "/home/gateway/.claude", + LANG: "C.UTF-8", + CLAUDE_CODE_DISABLE_AUTO_UPDATER: "1", + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1", + }); + expect(JSON.stringify(spawns[0]?.env)).not.toContain("canary"); + }); + + test("checks auth through the CLI without credential-file access", async () => { + const ready = await harness("success.jsonl", 0); + const missing = await harness("success.jsonl", 1); + + expect(await ready.driver.isAuthReady()).toBe(true); + expect(await missing.driver.isAuthReady()).toBe(false); + expect(ready.spawns[0]?.command).toEqual([ + "/usr/local/bin/claude", + "auth", + "status", + ]); + const source = await readFile( + join(import.meta.dir, "../src/drivers/claude/claude-driver.ts"), + "utf8", + ); + expect(source).not.toMatch(/readFile|\.credentials\.json|\.claude\.json/); + }); + + test("cancellation escalates from interrupt to terminate and kill on a stuck CLI", async () => { + const directory = await mkdtemp(join(tmpdir(), "openbot-claude-cancel-")); + directories.push(directory); + const signals: number[] = []; + let closeOutput: (() => void) | undefined; + let resolveExit: ((code: number) => void) | undefined; + const stdout = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + '{"type":"system","subtype":"init","session_id":"claude-session-cancel"}\n', + ), + ); + closeOutput = () => controller.close(); + }, + }); + const exited = new Promise((resolve) => { + resolveExit = resolve; + }); + const processFactory: ClaudeProcessFactory = () => ({ + stdout, + stderr: stream(""), + exited, + kill(signal) { + signals.push(signal ?? 0); + if (signal === constants.signals.SIGKILL) { + closeOutput?.(); + resolveExit?.(137); + } + }, + }); + const driver = new ClaudeDriver({ + processFactory, + interruptTimeoutMs: 5, + terminateTimeoutMs: 5, + }); + const controller = new AbortController(); + const iterator = driver + .start({ ...run, workspacePath: directory }, context(controller.signal)) + [Symbol.asyncIterator](); + + expect(await iterator.next()).toEqual({ + done: false, + value: { type: "session", sessionId: "claude-session-cancel" }, + }); + controller.abort(); + expect(await iterator.next()).toEqual({ done: true, value: undefined }); + expect(signals).toEqual([ + constants.signals.SIGINT, + constants.signals.SIGTERM, + constants.signals.SIGKILL, + ]); + }); + + test("ships a pinned CLI and a fail-closed managed worker policy", async () => { + const repository = join(import.meta.dir, "../.."); + const dockerfile = await readFile( + join(repository, "agent-subscription-gateway/Dockerfile"), + "utf8", + ); + const compose = await readFile( + join(repository, "deploy/aws/compose.worker.yml"), + "utf8", + ); + const settingsText = await readFile( + join(repository, "deploy/aws/claude-managed-settings.json"), + "utf8", + ); + const settings = JSON.parse(settingsText) as { + allowManagedHooksOnly: boolean; + hooks: Record; + allowedHttpHookUrls: string[]; + allowManagedMcpServersOnly: boolean; + allowedMcpServers: string[]; + allowManagedPermissionRulesOnly: boolean; + permissions: { + defaultMode: string; + disableBypassPermissionsMode: string; + }; + sandbox: { + enabled: boolean; + failIfUnavailable: boolean; + allowUnsandboxedCommands: boolean; + enableWeakerNestedSandbox: boolean; + filesystem: { denyRead: string[]; allowManagedReadPathsOnly: boolean }; + }; + }; + + expect(dockerfile).toContain("ARG CLAUDE_VERSION=2.1.271"); + expect(dockerfile).toMatch( + /"@anthropic-ai\/claude-code@\$\{CLAUDE_VERSION\}"/, + ); + expect(dockerfile).toMatch(/apt-get install[^\n]*bubblewrap[^\n]*socat/); + expect(dockerfile).toContain("chmod 0755 /etc/codex /etc/claude-code"); + expect(dockerfile).toContain("/etc/claude-code/managed-settings.json"); + expect(compose).toContain('CLAUDE_VERSION: "2.1.271"'); + expect(compose).toContain("target: /etc/claude-code/managed-settings.json"); + expect(compose).toContain("read_only: true"); + + expect(settings.allowManagedHooksOnly).toBe(true); + expect(settings.hooks).toEqual({}); + expect(settings.allowedHttpHookUrls).toEqual([]); + expect(settings.allowManagedMcpServersOnly).toBe(true); + expect(settings.allowedMcpServers).toEqual([]); + expect(settings.allowManagedPermissionRulesOnly).toBe(true); + expect(settings.permissions.defaultMode).toBe("dontAsk"); + expect(settings.permissions.disableBypassPermissionsMode).toBe("disable"); + expect(settings.sandbox).toMatchObject({ + enabled: true, + failIfUnavailable: true, + allowUnsandboxedCommands: false, + enableWeakerNestedSandbox: true, + }); + expect(settings.sandbox.filesystem.denyRead).toEqual( + expect.arrayContaining([ + "/home/gateway/.claude", + "/var/lib/openbot-gateway", + ]), + ); + expect(settings.sandbox.filesystem.allowManagedReadPathsOnly).toBe(true); + }); +}); diff --git a/agent-subscription-gateway/tests/codex-driver.test.ts b/agent-subscription-gateway/tests/codex-driver.test.ts new file mode 100644 index 000000000..9d04f9944 --- /dev/null +++ b/agent-subscription-gateway/tests/codex-driver.test.ts @@ -0,0 +1,258 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { + DriverEvent, + DriverRun, + DriverRunContext, +} from "../src/drivers/agent-driver"; +import { + CodexDriver, + type CodexProcessFactory, + type CodexSpawnOptions, +} from "../src/drivers/codex/codex-driver"; + +const fixture = join( + import.meta.dir, + "fixtures/codex-app-server/fake-app-server.ts", +); +const bun = (() => { + const path = Bun.which("bun"); + if (!path) throw new Error("Bun is required for the Codex fixture."); + return path; +})(); +const directories: string[] = []; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { recursive: true })), + ); +}); + +async function harness( + scenario: "success" | "approval" | "cancel" | "failure" = "success", + loginExitCode = 0, +) { + const directory = await mkdtemp(join(tmpdir(), "openbot-codex-driver-")); + directories.push(directory); + const capturePath = join(directory, "messages.jsonl"); + const spawns: CodexSpawnOptions[] = []; + const processFactory: CodexProcessFactory = (options) => { + spawns.push(options); + if (options.command.slice(1).join(" ") === "login status") { + return Bun.spawn([bun, "-e", `process.exit(${loginExitCode})`], { + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + } + return Bun.spawn([bun, fixture], { + cwd: options.cwd, + env: { + ...options.env, + FAKE_CODEX_SCENARIO: scenario, + FAKE_CODEX_CAPTURE: capturePath, + }, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + }; + const driver = new CodexDriver({ + processFactory, + binary: "/usr/local/bin/codex", + environment: { + PATH: process.env.PATH, + HOME: directory, + CODEX_HOME: join(directory, ".codex"), + GATEWAY_TOKEN: "must-not-reach-provider", + AWS_SECRET_ACCESS_KEY: "must-not-reach-provider", + }, + }); + const captured = async () => + (await readFile(capturePath, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + return { + driver, + captured, + spawns, + run: { ...run, workspacePath: directory } satisfies DriverRun, + }; +} + +const run: DriverRun = { + threadId: "openbot-thread-1", + runId: "openbot-run-1", + messages: [ + { id: "m1", role: "system", content: "Use the repository rules." }, + { id: "m2", role: "user", content: "Run the focused tests." }, + ], + context: [], + state: {}, + forwardedProps: {}, + workspacePath: "/workspaces/codex/openbot-run-1", +}; + +const context = (): DriverRunContext => ({ + signal: new AbortController().signal, +}); + +async function collect( + events: AsyncIterable, +): Promise { + const result: DriverEvent[] = []; + for await (const event of events) result.push(event); + return result; +} + +describe("CodexDriver", () => { + test("uses the stable app-server handshake and streams safe text and activity", async () => { + const { driver, captured, spawns, run } = await harness(); + + const result = await collect(driver.start(run, context())); + const messages = await captured(); + + expect(spawns[0]).toMatchObject({ + command: ["/usr/local/bin/codex", "app-server"], + cwd: run.workspacePath, + }); + expect(spawns[0]?.env.GATEWAY_TOKEN).toBeUndefined(); + expect(spawns[0]?.env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + expect(messages.map((message) => message.method)).toEqual([ + "initialize", + "initialized", + "thread/start", + "turn/start", + ]); + expect(messages[2]).toMatchObject({ + method: "thread/start", + params: { + cwd: run.workspacePath, + approvalPolicy: "never", + permissions: "openbot-worker", + }, + }); + expect(messages[3]).toMatchObject({ + method: "turn/start", + params: { + threadId: "codex-thread-new", + cwd: run.workspacePath, + approvalPolicy: "never", + }, + }); + expect(JSON.stringify(messages[3])).toContain("[system m1]"); + expect(JSON.stringify(messages[3])).toContain("Run the focused tests."); + expect(result).toEqual([ + { type: "session", sessionId: "codex-thread-new" }, + { + type: "activity", + message: "Command started", + data: { itemType: "commandExecution", status: "inProgress" }, + }, + { + type: "activity", + message: "Command produced output", + data: { itemType: "commandExecution" }, + }, + { + type: "activity", + message: "Command completed", + data: { itemType: "commandExecution", status: "completed" }, + }, + { type: "text", delta: "Done." }, + ]); + expect(JSON.stringify(result)).not.toContain("PRIVATE_COMMAND"); + expect(JSON.stringify(result)).not.toContain("/private/path"); + }); + + test("resumes the mapped Codex thread and sends only the supplied transcript delta", async () => { + const { driver, captured, run } = await harness(); + const resumed = { + ...run, + messages: [{ id: "m3", role: "user", content: "Now fix the next test." }], + sessionId: "codex-thread-existing", + }; + + const result = await collect(driver.resume(resumed, context())); + const messages = await captured(); + + expect(messages[2]).toEqual({ + id: 2, + method: "thread/resume", + params: { + approvalPolicy: "never", + cwd: run.workspacePath, + permissions: "openbot-worker", + threadId: "codex-thread-existing", + }, + }); + expect(JSON.stringify(messages[3])).toContain("Now fix the next test."); + expect(JSON.stringify(messages[3])).not.toContain("Run the focused tests."); + expect(result.some((event) => event.type === "session")).toBe(false); + }); + + test("fails closed when app-server asks the gateway for approval", async () => { + const { driver, captured, run } = await harness("approval"); + + await expect(collect(driver.start(run, context()))).rejects.toThrow( + "The Codex run requested interactive input and was stopped.", + ); + const messages = await captured(); + expect(messages).toContainEqual({ + id: 700, + error: { + code: -32000, + message: "Interactive requests are disabled by worker policy.", + }, + }); + expect(JSON.stringify(messages)).not.toContain("acceptForSession"); + }); + + test("maps private provider failures to a fixed terminal message", async () => { + const { driver, run } = await harness("failure"); + + await expect(collect(driver.start(run, context()))).rejects.toThrow( + "The Codex run failed.", + ); + }); + + test("cancels with turn/interrupt and stops the child process", async () => { + const { driver, captured, run } = await harness("cancel"); + const iterator = driver.start(run, context())[Symbol.asyncIterator](); + expect(await iterator.next()).toEqual({ + done: false, + value: { type: "session", sessionId: "codex-thread-new" }, + }); + expect(await iterator.next()).toMatchObject({ + done: false, + value: { type: "activity", message: "Command started" }, + }); + + await driver.cancel(run.runId); + await iterator.return?.(); + + expect(await captured()).toContainEqual({ + id: 4, + method: "turn/interrupt", + params: { threadId: "codex-thread-new", turnId: "codex-turn-1" }, + }); + }); + + test("checks login through the CLI without reading provider files", async () => { + const ready = await harness("success", 0); + const missing = await harness("success", 1); + + expect(await ready.driver.isAuthReady()).toBe(true); + expect(await missing.driver.isAuthReady()).toBe(false); + expect(ready.spawns[0]?.command).toEqual([ + "/usr/local/bin/codex", + "login", + "status", + ]); + expect(JSON.stringify(ready.spawns)).not.toContain("auth.json"); + }); +}); diff --git a/agent-subscription-gateway/tests/fixtures/claude-stream/permission-denied.jsonl b/agent-subscription-gateway/tests/fixtures/claude-stream/permission-denied.jsonl new file mode 100644 index 000000000..7020190de --- /dev/null +++ b/agent-subscription-gateway/tests/fixtures/claude-stream/permission-denied.jsonl @@ -0,0 +1,2 @@ +{"type":"system","subtype":"init","session_id":"claude-session-permission"} +{"type":"result","subtype":"error_during_execution","is_error":true,"permission_denials":[{"tool_name":"Bash","tool_input":{"command":"cat /home/gateway/.claude/.credentials.json"}}],"session_id":"claude-session-permission"} diff --git a/agent-subscription-gateway/tests/fixtures/claude-stream/rate-limit.jsonl b/agent-subscription-gateway/tests/fixtures/claude-stream/rate-limit.jsonl new file mode 100644 index 000000000..e79518a85 --- /dev/null +++ b/agent-subscription-gateway/tests/fixtures/claude-stream/rate-limit.jsonl @@ -0,0 +1,3 @@ +{"type":"system","subtype":"init","session_id":"claude-session-rate-limit"} +{"type":"system","subtype":"api_retry","attempt":2,"max_retries":5,"retry_delay_ms":1000,"error_status":429,"error":"rate_limit","session_id":"claude-session-rate-limit"} +{"type":"result","subtype":"error_during_execution","is_error":true,"errors":["PRIVATE_RATE_LIMIT_DETAILS"],"session_id":"claude-session-rate-limit"} diff --git a/agent-subscription-gateway/tests/fixtures/claude-stream/success.jsonl b/agent-subscription-gateway/tests/fixtures/claude-stream/success.jsonl new file mode 100644 index 000000000..fd1742e2e --- /dev/null +++ b/agent-subscription-gateway/tests/fixtures/claude-stream/success.jsonl @@ -0,0 +1,6 @@ +{"type":"system","subtype":"init","session_id":"claude-session-new","model":"claude-sonnet","tools":["Read","Bash"]} +{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tool-1","name":"Bash","input":{"command":"PRIVATE_COMMAND","description":"Run focused tests"}}]}} +{"type":"stream_event","event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"Done"}}} +{"type":"stream_event","event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"."}}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool-1","content":"PRIVATE_OUTPUT"}]}} +{"type":"result","subtype":"success","is_error":false,"result":"Done.","session_id":"claude-session-new"} diff --git a/agent-subscription-gateway/tests/fixtures/codex-app-server/fake-app-server.ts b/agent-subscription-gateway/tests/fixtures/codex-app-server/fake-app-server.ts new file mode 100644 index 000000000..1ba01ad29 --- /dev/null +++ b/agent-subscription-gateway/tests/fixtures/codex-app-server/fake-app-server.ts @@ -0,0 +1,147 @@ +import { appendFileSync } from "node:fs"; +import { createInterface } from "node:readline"; + +const scenario = process.env.FAKE_CODEX_SCENARIO ?? "success"; +const capturePath = process.env.FAKE_CODEX_CAPTURE; +const lines = createInterface({ + input: process.stdin, + crlfDelay: Number.POSITIVE_INFINITY, +}); + +function send(message: unknown): void { + process.stdout.write(`${JSON.stringify(message)}\n`); +} + +function capture(message: unknown): void { + if (capturePath) appendFileSync(capturePath, `${JSON.stringify(message)}\n`); +} + +for await (const line of lines) { + const message = JSON.parse(line) as { + id?: number; + method?: string; + params?: Record; + }; + capture(message); + + if (message.method === "initialize") { + send({ id: message.id, result: { userAgent: "fake-codex-app-server" } }); + } + if (message.method === "initialized") continue; + + if (message.method === "thread/start") { + send({ id: message.id, result: { thread: { id: "codex-thread-new" } } }); + continue; + } + if (message.method === "thread/resume") { + send({ + id: message.id, + result: { thread: { id: message.params?.threadId } }, + }); + continue; + } + + if (message.method === "turn/start") { + send({ + id: message.id, + result: { turn: { id: "codex-turn-1", status: "inProgress", items: [] } }, + }); + send({ + method: "item/started", + params: { + threadId: message.params?.threadId, + turnId: "codex-turn-1", + item: { + id: "command-1", + type: "commandExecution", + command: "PRIVATE_COMMAND_TEXT", + cwd: "/private/path", + status: "inProgress", + }, + }, + }); + + if (scenario === "cancel") continue; + if (scenario === "approval") { + send({ + id: 700, + method: "item/commandExecution/requestApproval", + params: { + threadId: message.params?.threadId, + turnId: "codex-turn-1", + itemId: "command-1", + command: "cat /home/gateway/.codex/auth.json", + }, + }); + continue; + } + if (scenario === "failure") { + send({ + method: "error", + params: { error: { message: "PRIVATE_PROVIDER_ERROR" } }, + }); + send({ + method: "turn/completed", + params: { + threadId: message.params?.threadId, + turn: { + id: "codex-turn-1", + status: "failed", + error: { message: "PRIVATE_PROVIDER_ERROR" }, + }, + }, + }); + continue; + } + + send({ + method: "item/commandExecution/outputDelta", + params: { + threadId: message.params?.threadId, + turnId: "codex-turn-1", + itemId: "command-1", + delta: "PRIVATE_COMMAND_OUTPUT", + }, + }); + send({ + method: "item/completed", + params: { + threadId: message.params?.threadId, + turnId: "codex-turn-1", + item: { + id: "command-1", + type: "commandExecution", + status: "completed", + }, + }, + }); + send({ + method: "item/agentMessage/delta", + params: { + threadId: message.params?.threadId, + turnId: "codex-turn-1", + itemId: "message-1", + delta: "Done.", + }, + }); + send({ + method: "turn/completed", + params: { + threadId: message.params?.threadId, + turn: { id: "codex-turn-1", status: "completed", items: [] }, + }, + }); + continue; + } + + if (message.method === "turn/interrupt") { + send({ id: message.id, result: {} }); + send({ + method: "turn/completed", + params: { + threadId: message.params?.threadId, + turn: { id: message.params?.turnId, status: "interrupted", items: [] }, + }, + }); + } +} diff --git a/agent-subscription-gateway/tests/fixtures/grok-acp/fake-agent.ts b/agent-subscription-gateway/tests/fixtures/grok-acp/fake-agent.ts new file mode 100644 index 000000000..e8525c84f --- /dev/null +++ b/agent-subscription-gateway/tests/fixtures/grok-acp/fake-agent.ts @@ -0,0 +1,107 @@ +import { appendFileSync } from "node:fs"; +import { createInterface } from "node:readline"; + +const scenario = process.env.FAKE_GROK_SCENARIO ?? "success"; +const capturePath = process.env.FAKE_GROK_CAPTURE; +const lines = createInterface({ + input: process.stdin, + crlfDelay: Number.POSITIVE_INFINITY, +}); + +function send(message: unknown): void { + process.stdout.write(`${JSON.stringify(message)}\n`); +} + +function capture(message: unknown): void { + if (capturePath) appendFileSync(capturePath, `${JSON.stringify(message)}\n`); +} + +for await (const line of lines) { + const message = JSON.parse(line) as { + id?: number; + method?: string; + params?: Record; + }; + capture(message); + + if (message.method === "initialize") { + send({ jsonrpc: "2.0", id: message.id, result: { protocolVersion: 1 } }); + continue; + } + if (message.method === "session/new") { + send({ + jsonrpc: "2.0", + id: message.id, + result: { sessionId: "grok-session-new" }, + }); + continue; + } + if (message.method === "session/load") { + send({ jsonrpc: "2.0", id: message.id, result: {} }); + continue; + } + if (message.method === "session/prompt") { + send({ + jsonrpc: "2.0", + method: "session/update", + params: { + sessionId: message.params?.sessionId, + update: { + sessionUpdate: "tool_call", + title: "PRIVATE_COMMAND_TEXT", + rawInput: { path: "/private/path" }, + }, + }, + }); + if (scenario === "cancel") continue; + if (scenario === "permission") { + send({ + jsonrpc: "2.0", + id: 700, + method: "session/request_permission", + params: { command: "cat /home/gateway/.grok/auth.json" }, + }); + continue; + } + if (scenario === "failure") { + send({ + jsonrpc: "2.0", + id: message.id, + error: { code: -32001, message: "PRIVATE_PROVIDER_ERROR" }, + }); + continue; + } + send({ + jsonrpc: "2.0", + method: "session/update", + params: { + sessionId: message.params?.sessionId, + update: { + sessionUpdate: "tool_call_update", + status: "completed", + rawOutput: "PRIVATE_COMMAND_OUTPUT", + }, + }, + }); + send({ + jsonrpc: "2.0", + method: "session/update", + params: { + sessionId: message.params?.sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Done." }, + }, + }, + }); + send({ + jsonrpc: "2.0", + id: message.id, + result: { stopReason: "end_turn" }, + }); + continue; + } + if (message.method === "session/cancel") { + process.exit(0); + } +} diff --git a/agent-subscription-gateway/tests/grok-driver.test.ts b/agent-subscription-gateway/tests/grok-driver.test.ts new file mode 100644 index 000000000..f5f659277 --- /dev/null +++ b/agent-subscription-gateway/tests/grok-driver.test.ts @@ -0,0 +1,262 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { + DriverEvent, + DriverRun, + DriverRunContext, +} from "../src/drivers/agent-driver"; +import { + GrokDriver, + type GrokProcessFactory, + type GrokSpawnOptions, +} from "../src/drivers/grok/grok-driver"; + +const fixture = join(import.meta.dir, "fixtures/grok-acp/fake-agent.ts"); +const repositoryRoot = join(import.meta.dir, "..", ".."); +const bun = (() => { + const path = Bun.which("bun"); + if (!path) throw new Error("Bun is required for the Grok fixture."); + return path; +})(); +const directories: string[] = []; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { recursive: true })), + ); +}); + +async function harness( + scenario: "success" | "permission" | "cancel" | "failure" = "success", + authExitCode = 0, +) { + const directory = await mkdtemp(join(tmpdir(), "openbot-grok-driver-")); + directories.push(directory); + const capturePath = join(directory, "messages.jsonl"); + const spawns: GrokSpawnOptions[] = []; + const processFactory: GrokProcessFactory = (options) => { + spawns.push(options); + if (options.command.slice(1).join(" ") === "models") { + return Bun.spawn([bun, "-e", `process.exit(${authExitCode})`], { + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + } + return Bun.spawn([bun, fixture], { + cwd: options.cwd, + env: { + ...options.env, + FAKE_GROK_SCENARIO: scenario, + FAKE_GROK_CAPTURE: capturePath, + }, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + }; + const driver = new GrokDriver({ + processFactory, + binary: "/usr/local/bin/grok", + environment: { + PATH: process.env.PATH, + HOME: directory, + GROK_HOME: join(directory, ".grok"), + GATEWAY_TOKEN: "must-not-reach-provider", + AWS_SECRET_ACCESS_KEY: "must-not-reach-provider", + }, + }); + const captured = async () => + (await readFile(capturePath, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + return { + driver, + captured, + spawns, + run: { ...run, workspacePath: directory } satisfies DriverRun, + }; +} + +const run: DriverRun = { + threadId: "openbot-thread-1", + runId: "openbot-run-1", + messages: [ + { id: "m1", role: "system", content: "Use the repository rules." }, + { id: "m2", role: "user", content: "Run the focused tests." }, + ], + context: [], + state: {}, + forwardedProps: {}, + workspacePath: "/workspaces/grok/openbot-run-1", +}; + +const context = (): DriverRunContext => ({ + signal: new AbortController().signal, +}); + +async function collect( + events: AsyncIterable, +): Promise { + const result: DriverEvent[] = []; + for await (const event of events) result.push(event); + return result; +} + +describe("GrokDriver", () => { + test("uses ACP with the strict sandbox and streams only safe events", async () => { + const { driver, captured, spawns, run } = await harness(); + + const result = await collect(driver.start(run, context())); + const messages = await captured(); + + expect(spawns[0]).toMatchObject({ + command: [ + "/usr/local/bin/grok", + "--no-subagents", + "--sandbox", + "strict", + "agent", + "--always-approve", + "--no-leader", + "stdio", + ], + cwd: run.workspacePath, + }); + expect(spawns[0]?.env.GATEWAY_TOKEN).toBeUndefined(); + expect(spawns[0]?.env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + expect(messages.map((message) => message.method)).toEqual([ + "initialize", + "session/new", + "session/prompt", + ]); + expect(messages[1]).toMatchObject({ + method: "session/new", + params: { + cwd: run.workspacePath, + mcpServers: [], + _meta: { yoloMode: true }, + }, + }); + expect(JSON.stringify(messages[2])).toContain("[system m1]"); + expect(result).toEqual([ + { type: "session", sessionId: "grok-session-new" }, + { type: "activity", message: "Native tool started" }, + { + type: "activity", + message: "Native tool completed", + data: { status: "completed" }, + }, + { type: "text", delta: "Done." }, + ]); + expect(JSON.stringify(result)).not.toContain("PRIVATE_COMMAND"); + expect(JSON.stringify(result)).not.toContain("/private/path"); + }); + + test("loads the mapped ACP session and sends only the transcript delta", async () => { + const { driver, captured, run } = await harness(); + const resumed = { + ...run, + messages: [{ id: "m3", role: "user", content: "Now fix the next test." }], + sessionId: "grok-session-existing", + }; + + const result = await collect(driver.resume(resumed, context())); + const messages = await captured(); + + expect(messages[1]).toMatchObject({ + method: "session/load", + params: { + sessionId: "grok-session-existing", + cwd: run.workspacePath, + mcpServers: [], + }, + }); + expect(JSON.stringify(messages[2])).toContain("Now fix the next test."); + expect(JSON.stringify(messages[2])).not.toContain("Run the focused tests."); + expect(result.some((event) => event.type === "session")).toBe(false); + }); + + test("fails closed when ACP asks the gateway for permission", async () => { + const { driver, captured, run } = await harness("permission"); + + await expect(collect(driver.start(run, context()))).rejects.toThrow( + "The Grok run requested permission beyond worker policy.", + ); + expect(await captured()).toContainEqual({ + jsonrpc: "2.0", + id: 700, + error: { + code: -32000, + message: "Interactive requests are disabled by worker policy.", + }, + }); + }); + + test("maps provider request failures to a fixed message", async () => { + const { driver, run } = await harness("failure"); + await expect(collect(driver.start(run, context()))).rejects.toThrow( + "The Grok agent rejected a request.", + ); + }); + + test("cancels the active ACP session", async () => { + const { driver, captured, run } = await harness("cancel"); + const iterator = driver.start(run, context())[Symbol.asyncIterator](); + expect(await iterator.next()).toEqual({ + done: false, + value: { type: "session", sessionId: "grok-session-new" }, + }); + expect(await iterator.next()).toMatchObject({ + done: false, + value: { type: "activity", message: "Native tool started" }, + }); + + await driver.cancel(run.runId); + await iterator.return?.(); + + expect(await captured()).toContainEqual({ + jsonrpc: "2.0", + method: "session/cancel", + params: { sessionId: "grok-session-new" }, + }); + }); + + test("checks auth without reading or printing provider files", async () => { + const ready = await harness("success", 0); + const missing = await harness("success", 1); + + expect(await ready.driver.isAuthReady()).toBe(true); + expect(await missing.driver.isAuthReady()).toBe(false); + expect(ready.spawns.at(-1)?.command).toEqual([ + "/usr/local/bin/grok", + "models", + ]); + }); + + test("ships a pinned CLI and a fail-closed worker policy", async () => { + const [dockerfile, requirements] = await Promise.all([ + readFile( + join(repositoryRoot, "agent-subscription-gateway", "Dockerfile"), + "utf8", + ), + readFile( + join(repositoryRoot, "deploy", "aws", "grok-requirements.toml"), + "utf8", + ), + ]); + + expect(dockerfile).toContain("ARG GROK_VERSION=1.0.30"); + expect(dockerfile).toMatch(/"@xai-official\/grok@\$\{GROK_VERSION\}"/); + expect(dockerfile).toContain("/etc/grok/requirements.toml"); + expect(requirements).toContain("fail_closed = true"); + expect(requirements).toContain('profile = "strict"'); + expect(requirements).toContain('"Read(/home/gateway/.grok/**)"'); + expect(requirements).toContain("enabled = false"); + expect(requirements).toContain("trace_upload = false"); + }); +}); diff --git a/agent-subscription-gateway/tests/health.test.ts b/agent-subscription-gateway/tests/health.test.ts new file mode 100644 index 000000000..7396ff815 --- /dev/null +++ b/agent-subscription-gateway/tests/health.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; +import type { AgentDriver } from "../src/drivers/agent-driver"; +import type { RunCounts } from "../src/observability/run-counts"; +import { createGatewayHandler } from "../src/server/app"; + +function driver(authReady: boolean): AgentDriver { + return { + provider: "codex", + version: "0.99.0", + async isAuthReady() { + return authReady; + }, + async *start() {}, + async *resume() {}, + async cancel() {}, + }; +} + +const counts: RunCounts = { + snapshot() { + return { queueDepth: 2, activeRuns: 1 }; + }, + started() {}, + finished() {}, +}; + +describe("gateway status", () => { + test("health stays live without provider auth and exposes no auth path", async () => { + const handle = createGatewayHandler({ + token: "worker-secret", + driver: driver(false), + counts, + }); + + const response = await handle(new Request("http://gateway.test/health")); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body).toEqual({ + status: "ok", + provider: "codex", + version: "0.99.0", + authReady: false, + queueDepth: 2, + activeRuns: 1, + }); + expect(JSON.stringify(body)).not.toContain("path"); + expect(JSON.stringify(body)).not.toContain("credential"); + }); + + test("readiness fails closed when provider auth is not ready", async () => { + const handle = createGatewayHandler({ + token: "worker-secret", + driver: driver(false), + counts, + }); + + const response = await handle(new Request("http://gateway.test/ready")); + + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ + status: "not_ready", + authReady: false, + }); + }); + + test("readiness succeeds when provider auth is ready", async () => { + const handle = createGatewayHandler({ + token: "worker-secret", + driver: driver(true), + counts, + }); + + const response = await handle(new Request("http://gateway.test/ready")); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + status: "ready", + authReady: true, + }); + }); +}); diff --git a/agent-subscription-gateway/tests/queue.test.ts b/agent-subscription-gateway/tests/queue.test.ts new file mode 100644 index 000000000..d222031bb --- /dev/null +++ b/agent-subscription-gateway/tests/queue.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, test } from "bun:test"; +import { + BoundedRunQueue, + CapacityError, + HostSemaphore, + RunDisconnectedError, +} from "../src/runs/queue"; +import { RunStore } from "../src/storage/run-store"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +describe("bounded run admission", () => { + test("runs one request, queues one request, and rejects excess capacity", async () => { + const queue = new BoundedRunQueue({ + provider: "codex", + host: new HostSemaphore(1), + concurrency: 1, + pendingLimit: 1, + }); + const first = deferred(); + const order: string[] = []; + + const firstResult = queue.enqueue({ + runId: "run-1", + signal: new AbortController().signal, + run: async () => { + order.push("run-1"); + return first.promise; + }, + }); + const secondResult = queue.enqueue({ + runId: "run-2", + signal: new AbortController().signal, + run: async () => { + order.push("run-2"); + return "second"; + }, + }); + + expect(() => + queue.enqueue({ + runId: "run-3", + signal: new AbortController().signal, + run: async () => "third", + }), + ).toThrow(CapacityError); + try { + queue.enqueue({ + runId: "run-4", + signal: new AbortController().signal, + run: async () => "fourth", + }); + } catch (error) { + expect(error).toMatchObject({ retryable: true }); + } + expect(queue.snapshot()).toEqual({ activeRuns: 1, queueDepth: 1 }); + + first.resolve("first"); + expect(await firstResult).toBe("first"); + expect(await secondResult).toBe("second"); + expect(order).toEqual(["run-1", "run-2"]); + expect(queue.snapshot()).toEqual({ activeRuns: 0, queueDepth: 0 }); + }); + + test("shares one host slot across provider queues", async () => { + const host = new HostSemaphore(1); + const codex = new BoundedRunQueue({ provider: "codex", host }); + const claude = new BoundedRunQueue({ provider: "claude", host }); + const gate = deferred(); + const started: string[] = []; + + const codexRun = codex.enqueue({ + runId: "codex-1", + signal: new AbortController().signal, + run: async () => { + started.push("codex"); + await gate.promise; + }, + }); + const claudeRun = claude.enqueue({ + runId: "claude-1", + signal: new AbortController().signal, + run: async () => { + started.push("claude"); + }, + }); + + await Promise.resolve(); + expect(started).toEqual(["codex"]); + expect(host.activeCount).toBe(1); + expect(claude.snapshot()).toEqual({ activeRuns: 0, queueDepth: 1 }); + + gate.resolve(); + await Promise.all([codexRun, claudeRun]); + expect(started).toEqual(["codex", "claude"]); + expect(host.activeCount).toBe(0); + }); + + test("heartbeats only while queued and drops a disconnected request", async () => { + const host = new HostSemaphore(1); + const releaseHost = host.tryAcquire(); + expect(releaseHost).toBeDefined(); + const queue = new BoundedRunQueue({ + provider: "grok", + host, + heartbeatMs: 5, + }); + const controller = new AbortController(); + let heartbeats = 0; + let ran = false; + + const result = queue.enqueue({ + runId: "grok-1", + signal: controller.signal, + onHeartbeat: () => { + heartbeats += 1; + }, + run: async () => { + ran = true; + }, + }); + await Bun.sleep(15); + expect(heartbeats).toBeGreaterThan(0); + controller.abort(); + await expect(result).rejects.toBeInstanceOf(RunDisconnectedError); + releaseHost?.(); + await Promise.resolve(); + expect(ran).toBe(false); + expect(queue.snapshot()).toEqual({ activeRuns: 0, queueDepth: 0 }); + }); + + test("enforces the host lease across separate semaphore instances", async () => { + const directory = await mkdtemp(join(tmpdir(), "openbot-host-lease-")); + const path = join(directory, "gateway.sqlite"); + const firstStore = new RunStore(path); + const secondStore = new RunStore(path); + const first = new HostSemaphore({ + limit: 1, + leaseStore: firstStore, + pollMs: 2, + }); + const second = new HostSemaphore({ + limit: 1, + leaseStore: secondStore, + pollMs: 2, + }); + const firstRelease = first.tryAcquire("codex:run-1"); + expect(firstRelease).toBeDefined(); + expect(second.tryAcquire("claude:run-1")).toBeUndefined(); + + const waiting = second.acquire( + new AbortController().signal, + "claude:run-1", + ); + firstRelease?.(); + const secondRelease = await waiting; + expect(second.activeCount).toBe(1); + secondRelease(); + firstStore.close(); + secondStore.close(); + await rm(directory, { recursive: true }); + }); +}); diff --git a/agent-subscription-gateway/tests/recovery.test.ts b/agent-subscription-gateway/tests/recovery.test.ts new file mode 100644 index 000000000..381b7bb4e --- /dev/null +++ b/agent-subscription-gateway/tests/recovery.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { RunStore } from "../src/storage/run-store"; + +const directories: string[] = []; + +async function store() { + const directory = await mkdtemp(join(tmpdir(), "openbot-recovery-")); + directories.push(directory); + return new RunStore(join(directory, "gateway.sqlite")); +} + +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((directory) => rm(directory, { recursive: true })), + ); +}); + +describe("restart recovery", () => { + test("fails queued and active rows once without changing terminal rows", async () => { + const runs = await store(); + for (const runId of ["queued", "preparing", "running", "completed"]) { + runs.reserveRun({ provider: "codex", runId, threadId: "thread-1" }); + } + runs.transitionRun("codex", "preparing", "preparing", ["queued"]); + runs.transitionRun("codex", "running", "preparing", ["queued"]); + runs.transitionRun("codex", "running", "running", ["preparing"]); + runs.transitionRun("codex", "completed", "preparing", ["queued"]); + runs.transitionRun("codex", "completed", "running", ["preparing"]); + runs.transitionRun("codex", "completed", "completed", ["running"]); + runs.attachWorkspace("codex", "running", "/jobs/codex/running"); + + expect( + runs.recoverInterrupted("Gateway restarted; submit a new run."), + ).toBe(3); + expect( + runs.recoverInterrupted("Gateway restarted; submit a new run."), + ).toBe(0); + expect(runs.getRun("codex", "queued")).toMatchObject({ state: "failed" }); + expect(runs.getRun("codex", "preparing")).toMatchObject({ + state: "failed", + }); + expect(runs.getRun("codex", "running")).toMatchObject({ + state: "failed", + workspacePath: "/jobs/codex/running", + errorSummary: "Gateway restarted; submit a new run.", + }); + expect(runs.getRun("codex", "completed")).toMatchObject({ + state: "completed", + }); + runs.close(); + }); + + test("uses guarded atomic transitions and retains completed result data", async () => { + const runs = await store(); + runs.reserveRun({ provider: "grok", runId: "run-1", threadId: "thread-1" }); + expect(() => + runs.transitionRun("grok", "run-1", "completed", ["queued"]), + ).toThrow(/transition/i); + runs.transitionRun("grok", "run-1", "preparing", ["queued"]); + runs.transitionRun("grok", "run-1", "running", ["preparing"]); + runs.saveResult("grok", "run-1", { + patch: "diff --git a/a b/a", + baseCommit: "a".repeat(40), + commits: [{ hash: "b".repeat(40), subject: "change" }], + }); + runs.transitionRun("grok", "run-1", "completed", ["running"]); + + const reopenedPath = runs.path; + runs.close(); + const reopened = new RunStore(reopenedPath); + expect(reopened.getRun("grok", "run-1")).toMatchObject({ + state: "completed", + patch: "diff --git a/a b/a", + baseCommit: "a".repeat(40), + commits: [{ hash: "b".repeat(40), subject: "change" }], + }); + reopened.close(); + }); +}); diff --git a/agent-subscription-gateway/tests/sessions.test.ts b/agent-subscription-gateway/tests/sessions.test.ts new file mode 100644 index 000000000..7fd0d6106 --- /dev/null +++ b/agent-subscription-gateway/tests/sessions.test.ts @@ -0,0 +1,232 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { RunStore } from "../src/storage/run-store"; +import { prepareProviderRun } from "../src/runs/session-input"; +import type { + AgentDriver, + DriverRun, + DriverRunContext, + DriverResumeRun, +} from "../src/drivers/agent-driver"; +import { createGatewayHandler } from "../src/server/app"; +import { BoundedRunQueue, HostSemaphore } from "../src/runs/queue"; +import { RunService } from "../src/runs/run-service"; + +const directories: string[] = []; + +async function store() { + const directory = await mkdtemp(join(tmpdir(), "openbot-sessions-")); + directories.push(directory); + return new RunStore(join(directory, "gateway.sqlite")); +} + +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((directory) => rm(directory, { recursive: true })), + ); +}); + +const messages = [ + { id: "m1", role: "user", content: "First", secret: "drop-me" }, + { id: "m2", role: "assistant", content: [{ type: "text", text: "Second" }] }, + { id: "m3", role: "user", content: "Third" }, +]; + +describe("run and session storage", () => { + test("reserves provider plus run ID once", async () => { + const runs = await store(); + const first = runs.reserveRun({ + provider: "codex", + runId: "run-1", + threadId: "thread-1", + }); + const duplicate = runs.reserveRun({ + provider: "codex", + runId: "run-1", + threadId: "thread-1", + }); + const otherProvider = runs.reserveRun({ + provider: "claude", + runId: "run-1", + threadId: "thread-1", + }); + + expect(first.created).toBe(true); + expect(duplicate.created).toBe(false); + expect(duplicate.run).toEqual(first.run); + expect(otherProvider.created).toBe(true); + expect(runs.listRuns()).toHaveLength(2); + runs.close(); + }); + + test("sends sanitized full history once, then only the message delta", async () => { + const runs = await store(); + + const initial = prepareProviderRun(runs, "codex", "thread-1", messages); + expect(initial).toEqual({ + messages: [ + { id: "m1", role: "user", content: "First" }, + { id: "m2", role: "assistant", content: "Second" }, + { id: "m3", role: "user", content: "Third" }, + ], + acknowledgedMessageId: "m3", + }); + + runs.saveSession({ + provider: "codex", + threadId: "thread-1", + sessionId: "provider-session-1", + acknowledgedMessageId: "m2", + }); + const resumed = prepareProviderRun(runs, "codex", "thread-1", [ + ...messages, + { id: "m4", role: "user", content: "Fourth", providerToken: "drop" }, + ]); + + expect(resumed).toEqual({ + sessionId: "provider-session-1", + messages: [ + { id: "m3", role: "user", content: "Third" }, + { id: "m4", role: "user", content: "Fourth" }, + ], + acknowledgedMessageId: "m4", + }); + expect( + prepareProviderRun(runs, "codex", "thread-1", [ + { id: "replacement", role: "user", content: "New canonical history" }, + ]), + ).toEqual({ + messages: [ + { id: "replacement", role: "user", content: "New canonical history" }, + ], + acknowledgedMessageId: "replacement", + }); + runs.close(); + }); + + test("treats an unknown mapping as a new session and rejects an invalid acknowledgement", async () => { + const runs = await store(); + expect( + prepareProviderRun(runs, "grok", "unknown", messages), + ).not.toHaveProperty("sessionId"); + expect(() => + runs.saveSession({ + provider: "grok", + threadId: "thread-1", + sessionId: "../unsafe", + acknowledgedMessageId: "m1", + }), + ).toThrow(/session/i); + expect(() => + prepareProviderRun(runs, "codex", "thread-1", [messages[0], messages[0]]), + ).toThrow(/duplicate message/i); + runs.close(); + }); + + test("runs the HTTP chain once and resumes with only the new message", async () => { + const runs = await store(); + const started: DriverRun[] = []; + const resumed: DriverResumeRun[] = []; + const driver: AgentDriver = { + provider: "codex", + version: "test", + async isAuthReady() { + return true; + }, + async *start(run: DriverRun, _context: DriverRunContext) { + started.push(run); + yield { type: "session", sessionId: "native-session-1" }; + yield { type: "text", delta: "done" }; + }, + async *resume(run: DriverResumeRun, _context: DriverRunContext) { + resumed.push(run); + yield { type: "text", delta: "again" }; + }, + async cancel() {}, + }; + const workspace = { + async create(runId: string) { + return { + provider: "codex", + runId, + path: `/fixed/jobs/codex/${runId}`, + baseCommit: "a".repeat(40), + }; + }, + async captureResult(lease: { baseCommit: string }) { + return { patch: "patch", baseCommit: lease.baseCommit, commits: [] }; + }, + }; + const queue = new BoundedRunQueue({ + provider: "codex", + host: new HostSemaphore(1), + }); + const runService = new RunService({ + driver, + store: runs, + queue, + workspaces: workspace, + logger: { terminal() {} }, + }); + const handle = createGatewayHandler({ + token: "worker-secret", + driver, + runService, + }); + const send = (runId: string, inputMessages: unknown[]) => + handle( + new Request("http://gateway.test/ag-ui", { + method: "POST", + headers: { + "content-type": "application/json", + "x-openbot-agent-token": "worker-secret", + }, + body: JSON.stringify({ + threadId: "thread-http", + runId, + messages: inputMessages, + tools: [], + }), + }), + ); + + const first = await send("http-1", messages.slice(0, 2)); + expect((await first.text()).match(/RUN_FINISHED/g)).toHaveLength(1); + expect(started).toHaveLength(1); + expect(started[0]).toMatchObject({ + workspacePath: "/fixed/jobs/codex/http-1", + messages: [ + { id: "m1", role: "user", content: "First" }, + { id: "m2", role: "assistant", content: "Second" }, + ], + }); + expect(runs.getRun("codex", "http-1")).toMatchObject({ + state: "completed", + providerSessionId: "native-session-1", + acknowledgedMessageId: "m2", + patch: "patch", + }); + + const second = await send("http-2", messages); + expect((await second.text()).match(/RUN_FINISHED/g)).toHaveLength(1); + expect(resumed).toHaveLength(1); + expect(resumed[0]).toMatchObject({ + sessionId: "native-session-1", + messages: [{ id: "m3", role: "user", content: "Third" }], + workspacePath: "/fixed/jobs/codex/http-2", + }); + + const duplicate = await send("http-2", messages); + expect(duplicate.status).toBe(409); + expect(await duplicate.json()).toMatchObject({ + retryable: false, + run: { runId: "http-2", state: "completed", resultAvailable: true }, + }); + expect(resumed).toHaveLength(1); + runs.close(); + }); +}); diff --git a/agent-subscription-gateway/tests/workspaces.test.ts b/agent-subscription-gateway/tests/workspaces.test.ts new file mode 100644 index 000000000..b9bab245e --- /dev/null +++ b/agent-subscription-gateway/tests/workspaces.test.ts @@ -0,0 +1,171 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + scrubProviderEnvironment, + WorkspaceManager, +} from "../src/workspaces/workspace-manager"; +import { cleanupExpiredWorkspaces } from "../src/workspaces/cleanup"; +import { RunStore } from "../src/storage/run-store"; + +const directories: string[] = []; + +function git(cwd: string, args: string[]) { + const result = Bun.spawnSync( + ["git", "-c", "core.hooksPath=/dev/null", ...args], + { cwd, stdout: "pipe", stderr: "pipe" }, + ); + if (result.exitCode !== 0) throw new Error(result.stderr.toString()); + return result.stdout.toString().trim(); +} + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "openbot-workspace-")); + directories.push(root); + const source = join(root, "source"); + const jobs = join(root, "jobs"); + await mkdir(source); + git(source, ["init", "--initial-branch=main"]); + git(source, ["config", "user.name", "Test User"]); + git(source, ["config", "user.email", "test@example.com"]); + await writeFile(join(source, "README.md"), "before\n"); + git(source, ["add", "README.md"]); + git(source, ["commit", "-m", "initial"]); + return { source, jobs }; +} + +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((directory) => rm(directory, { recursive: true })), + ); +}); + +describe("isolated workspaces", () => { + test("clones only the fixed repository and branch into a validated run path", async () => { + const { source, jobs } = await fixture(); + const manager = new WorkspaceManager({ + provider: "codex", + repository: source, + baseBranch: "main", + jobRoot: jobs, + }); + + const workspace = await manager.create("run-1"); + expect(workspace.path).toBe(join(jobs, "codex", "run-1")); + expect(await readFile(join(workspace.path, "README.md"), "utf8")).toBe( + "before\n", + ); + expect(git(workspace.path, ["branch", "--show-current"])).toBe(""); + expect(() => manager.pathFor("../escape")).toThrow(/run ID/i); + }, 15_000); + + test("retains a binary patch and commit metadata without provider push credentials", async () => { + const { source, jobs } = await fixture(); + const manager = new WorkspaceManager({ + provider: "claude", + repository: source, + baseBranch: "main", + jobRoot: jobs, + }); + const workspace = await manager.create("run-2"); + await writeFile(join(workspace.path, "README.md"), "after\n"); + await writeFile(join(workspace.path, "new-file.txt"), "new\n"); + + const result = await manager.captureResult(workspace); + expect(result.patch).toContain("-before"); + expect(result.patch).toContain("+after"); + expect(result.patch).toContain("new-file.txt"); + expect(result.baseCommit).toMatch(/^[0-9a-f]{40}$/); + expect(result.commits).toEqual([]); + + const environment = scrubProviderEnvironment({ + PATH: process.env.PATH, + GITHUB_TOKEN: "push-secret", + GH_TOKEN: "another-secret", + GIT_ASKPASS: "/credential-helper", + AWS_SESSION_TOKEN: "instance-secret", + DOCKER_HOST: "unix:///var/run/docker.sock", + SAFE_VALUE: "kept", + }); + expect(environment).toEqual( + expect.objectContaining({ SAFE_VALUE: "kept" }), + ); + expect(environment).not.toHaveProperty("GITHUB_TOKEN"); + expect(environment).not.toHaveProperty("GH_TOKEN"); + expect(environment).not.toHaveProperty("GIT_ASKPASS"); + expect(environment).not.toHaveProperty("AWS_SESSION_TOKEN"); + expect(environment).not.toHaveProperty("DOCKER_HOST"); + expect(git(workspace.path, ["remote", "get-url", "--push", "origin"])).toBe( + "disabled://push-not-allowed", + ); + }, 15_000); + + test("cleanup cannot cross the configured job root", async () => { + const { source, jobs } = await fixture(); + const manager = new WorkspaceManager({ + provider: "grok", + repository: source, + baseBranch: "main", + jobRoot: jobs, + }); + const first = await manager.create("run-a"); + const second = await manager.create("run-b"); + + await manager.remove(first.path); + expect(await Bun.file(join(first.path, "README.md")).exists()).toBe(false); + expect(await Bun.file(join(second.path, "README.md")).exists()).toBe(true); + await expect(manager.remove(source)).rejects.toThrow(/job root/i); + }, 15_000); + + test("cleans only expired terminal workspaces and keeps the retained result", async () => { + const { source, jobs } = await fixture(); + const manager = new WorkspaceManager({ + provider: "codex", + repository: source, + baseBranch: "main", + jobRoot: jobs, + }); + const completed = await manager.create("completed"); + const active = await manager.create("active"); + const runs = new RunStore(join(jobs, "state.sqlite")); + for (const runId of ["completed", "active"]) { + runs.reserveRun({ + provider: "codex", + runId, + threadId: `thread-${runId}`, + }); + runs.transitionRun("codex", runId, "preparing", ["queued"]); + runs.attachWorkspace( + "codex", + runId, + runId === "completed" ? completed.path : active.path, + ); + runs.transitionRun("codex", runId, "running", ["preparing"]); + } + runs.saveResult("codex", "completed", { + patch: "retained-patch", + baseCommit: completed.baseCommit, + commits: [], + }); + runs.transitionRun("codex", "completed", "completed", ["running"]); + + expect( + await cleanupExpiredWorkspaces(runs, manager, Date.now() + 1_000), + ).toEqual([completed.path]); + expect(await Bun.file(join(completed.path, "README.md")).exists()).toBe( + false, + ); + expect(await Bun.file(join(active.path, "README.md")).exists()).toBe(true); + expect(runs.getRun("codex", "completed")).toMatchObject({ + state: "completed", + patch: "retained-patch", + }); + expect(runs.getRun("codex", "completed")).not.toHaveProperty( + "workspacePath", + ); + runs.close(); + }, 15_000); +}); diff --git a/agent-subscription-gateway/tsconfig.json b/agent-subscription-gateway/tsconfig.json new file mode 100644 index 000000000..b583d655c --- /dev/null +++ b/agent-subscription-gateway/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "types": ["bun"] + }, + "include": ["src", "tests"] +} diff --git a/biome.json b/biome.json index 9887f67d0..820024a5c 100644 --- a/biome.json +++ b/biome.json @@ -10,6 +10,7 @@ "!app/src/routeTree.gen.ts", "!desktop/src-tauri/gen", "!**/target", + "!**/.terraform", "!server/drizzle" ] }, diff --git a/deploy/aws/README.md b/deploy/aws/README.md new file mode 100644 index 000000000..e0e7b4ff1 --- /dev/null +++ b/deploy/aws/README.md @@ -0,0 +1,19 @@ +# Personal AWS deployment files + +This directory holds the two-host AWS runtime: + +```text +Internet -> Caddy -> OpenBot + PostgreSQL control EC2 + | + v private VPC + Codex and Grok gateways worker EC2 + Claude disabled by default +``` + +Terraform creates the network, hosts, retained encrypted volumes, ECR repositories, alerts, and backups. The protected GitHub Actions workflow builds signed images, scans them, sends this directory to the hosts through SSM, and starts the systemd services. No host needs a GitHub key. + +Store one JSON object for each host in AWS Secrets Manager. Use `control-secret.example.json` and `worker-secret.example.json` as key lists. The deploy workflow changes only `OPENBOT_IMAGE` or `GATEWAY_IMAGE`. Provider login state stays on the retained volumes under `/srv/openbot-auth` and is not in Secrets Manager, a backup, an image, or OpenBot. + +Use digest-qualified image values. Do not use `latest`. Terraform tags the control and worker data volumes with `Backup=included-service-data`. It tags each provider auth volume with `Backup=excluded-provider-auth`. + +See [AWS deployment](../../docs/runbooks/aws-deployment.md), [provider sign-in](../../docs/runbooks/provider-sign-in.md), and [worker operations](../../docs/runbooks/worker-operations.md). diff --git a/deploy/aws/bin/deploy-via-ssm b/deploy/aws/bin/deploy-via-ssm new file mode 100755 index 000000000..7c244898a --- /dev/null +++ b/deploy/aws/bin/deploy-via-ssm @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +if [[ "$#" -ne 7 ]]; then + echo "usage: deploy-via-ssm ROLE INSTANCE_ID SECRET_ARN AWS_REGION ECR_REGISTRY IMAGE_KEY IMAGE_REFERENCE" >&2 + exit 64 +fi + +role="$1" +instance_id="$2" +secret_arn="$3" +aws_region="$4" +registry="$5" +image_key="$6" +image_reference="$7" +bundle="${DEPLOY_BUNDLE:?set DEPLOY_BUNDLE to the deploy/aws tar archive}" + +if [[ "$role" != "control" && "$role" != "worker" ]]; then + echo "role must be control or worker" >&2 + exit 65 +fi +if [[ ! "$instance_id" =~ ^i-[0-9a-f]+$ ]]; then + echo "the instance ID is invalid" >&2 + exit 65 +fi +if [[ "$image_key" != "OPENBOT_IMAGE" && "$image_key" != "GATEWAY_IMAGE" ]]; then + echo "the image key is invalid" >&2 + exit 65 +fi +if [[ ! "$image_reference" =~ ^[0-9]{12}\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com/.+@sha256:[0-9a-f]{64}$ ]]; then + echo "the image reference must use an ECR digest" >&2 + exit 65 +fi +if [[ ! -s "$bundle" ]]; then + echo "the deployment bundle is missing" >&2 + exit 66 +fi + +temporary="$(mktemp -d)" +old_secret="$temporary/old-secret.json" +new_secret="$temporary/new-secret.json" +request="$temporary/request.json" +updated=0 +command_id="" + +cleanup() { + rm -rf "$temporary" +} + +rollback() { + status=$? + trap - ERR + set +e + if [[ "$updated" -eq 1 ]]; then + aws secretsmanager put-secret-value \ + --region "$aws_region" \ + --secret-id "$secret_arn" \ + --secret-string "file://$old_secret" >/dev/null + aws ssm send-command \ + --region "$aws_region" \ + --document-name AWS-RunShellScript \ + --instance-ids "$instance_id" \ + --parameters "commands=systemctl restart openbot-${role}-env.service openbot-${role}.service" \ + >/dev/null + echo "$role rollout failed; restored the prior image setting" >&2 + fi + exit "$status" +} + +trap cleanup EXIT +trap rollback ERR + +aws secretsmanager get-secret-value \ + --region "$aws_region" \ + --secret-id "$secret_arn" \ + --query SecretString \ + --output text > "$old_secret" +jq -e 'type == "object"' "$old_secret" >/dev/null +jq --arg key "$image_key" --arg image "$image_reference" '.[$key] = $image' \ + "$old_secret" > "$new_secret" +aws secretsmanager put-secret-value \ + --region "$aws_region" \ + --secret-id "$secret_arn" \ + --secret-string "file://$new_secret" >/dev/null +updated=1 + +payload="$(base64 < "$bundle" | tr -d '\n')" +if (( ${#payload} > 18000 )); then + echo "the deployment bundle is too large for one SSM command" >&2 + false +fi + +jq -n \ + --arg instance "$instance_id" \ + --arg payload "$payload" \ + --arg role "$role" \ + --arg secret "$secret_arn" \ + --arg region "$aws_region" \ + --arg registry "$registry" \ + '{ + DocumentName: "AWS-RunShellScript", + InstanceIds: [$instance], + Parameters: {commands: [ + "set -euo pipefail", + "install -d -m 0755 /opt/openbot", + ("printf %s " + ($payload | @sh) + " | base64 --decode > /tmp/openbot-deploy.tgz"), + "tar -xzf /tmp/openbot-deploy.tgz -C /opt/openbot", + "rm -f /tmp/openbot-deploy.tgz", + ("/opt/openbot/deploy/aws/bin/install-host " + ([$role, $secret, $region, $registry] | map(@sh) | join(" "))) + ]} + }' > "$request" + +command_id="$(aws ssm send-command \ + --region "$aws_region" \ + --cli-input-json "file://$request" \ + --query Command.CommandId \ + --output text)" + +aws ssm wait command-executed \ + --region "$aws_region" \ + --command-id "$command_id" \ + --instance-id "$instance_id" + +status="$(aws ssm get-command-invocation \ + --region "$aws_region" \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query Status \ + --output text)" +if [[ "$status" != "Success" ]]; then + aws ssm get-command-invocation \ + --region "$aws_region" \ + --command-id "$command_id" \ + --instance-id "$instance_id" \ + --query StandardErrorContent \ + --output text >&2 + false +fi + +updated=0 +echo "$role deployed $image_reference" diff --git a/deploy/aws/bin/ecr-login b/deploy/aws/bin/ecr-login new file mode 100755 index 000000000..38b1c462a --- /dev/null +++ b/deploy/aws/bin/ecr-login @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 ]]; then + echo "usage: ecr-login REFERENCE_FILE" >&2 + exit 64 +fi + +reference_file="$1" +registry="$(sed -n 's/^ECR_REGISTRY=//p' "$reference_file")" +aws_region="$(sed -n 's/^AWS_REGION=//p' "$reference_file")" + +if [[ ! "$registry" =~ ^[0-9]+\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com$ || ! "$aws_region" =~ ^[a-z]{2}-[a-z]+-[0-9]+$ ]]; then + echo "the registry reference is invalid" >&2 + exit 65 +fi + +aws ecr get-login-password --region "$aws_region" \ + | docker login --username AWS --password-stdin "$registry" >/dev/null diff --git a/deploy/aws/bin/install-host b/deploy/aws/bin/install-host new file mode 100755 index 000000000..c2b5e85c1 --- /dev/null +++ b/deploy/aws/bin/install-host @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 4 ]]; then + echo "usage: install-host ROLE SECRET_ARN AWS_REGION ECR_REGISTRY" >&2 + exit 64 +fi + +role="$1" +secret_arn="$2" +aws_region="$3" +registry="$4" + +if [[ "$role" != "control" && "$role" != "worker" ]]; then + echo "role must be control or worker" >&2 + exit 65 +fi +if [[ ! "$secret_arn" =~ ^arn:aws[a-zA-Z-]*:secretsmanager:[a-z0-9-]+:[0-9]{12}:secret: ]]; then + echo "the secret ARN is invalid" >&2 + exit 65 +fi +if [[ ! "$aws_region" =~ ^[a-z]{2}(-gov)?-[a-z]+-[0-9]+$ ]]; then + echo "the AWS region is invalid" >&2 + exit 65 +fi +if [[ ! "$registry" =~ ^[0-9]{12}\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com$ ]]; then + echo "the ECR registry is invalid" >&2 + exit 65 +fi + +deploy_root="/opt/openbot/deploy/aws" +install -d -m 0700 /etc/openbot +install -m 0755 "$deploy_root/bin/ecr-login" /opt/openbot/deploy/aws/bin/ecr-login +install -m 0755 "$deploy_root/bin/render-secret-env" /opt/openbot/deploy/aws/bin/render-secret-env +install -m 0755 "$deploy_root/bin/worker-firewall" /opt/openbot/deploy/aws/bin/worker-firewall +install -m 0755 "$deploy_root/bin/publish-health-metrics" /opt/openbot/deploy/aws/bin/publish-health-metrics + +printf 'SECRET_ARN=%s\nAWS_REGION=%s\n' "$secret_arn" "$aws_region" \ + > "/etc/openbot/${role}-secret-ref" +printf 'ECR_REGISTRY=%s\nAWS_REGION=%s\n' "$registry" "$aws_region" \ + > /etc/openbot/registry-ref +chmod 0600 "/etc/openbot/${role}-secret-ref" /etc/openbot/registry-ref +printf 'OPENBOT_HOST_ROLE=%s\nAWS_REGION=%s\n' "$role" "$aws_region" > /etc/openbot/metrics-ref +chmod 0600 /etc/openbot/metrics-ref +install -m 0644 "$deploy_root/systemd/openbot-health-metrics.service" /etc/systemd/system/ +install -m 0644 "$deploy_root/systemd/openbot-health-metrics.timer" /etc/systemd/system/ + +if [[ "$role" == "control" ]]; then + install -m 0644 "$deploy_root/systemd/openbot-control-env.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-control.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-routines.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-routines.timer" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-attachment-cleanup.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-attachment-cleanup.timer" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-database-backup.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-database-backup.timer" /etc/systemd/system/ + systemctl daemon-reload + systemctl enable openbot-control-env.service openbot-control.service \ + openbot-routines.timer openbot-attachment-cleanup.timer openbot-database-backup.timer \ + openbot-health-metrics.timer + systemctl restart openbot-control-env.service + systemctl restart openbot-control.service + systemctl start openbot-routines.timer openbot-attachment-cleanup.timer \ + openbot-database-backup.timer + expected_services=(caddy openbot postgres) +else + install -m 0644 "$deploy_root/systemd/openbot-worker-env.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-worker-firewall.service" /etc/systemd/system/ + install -m 0644 "$deploy_root/systemd/openbot-worker.service" /etc/systemd/system/ + systemctl daemon-reload + systemctl enable openbot-worker-env.service openbot-worker-firewall.service openbot-worker.service + systemctl restart openbot-worker-env.service + systemctl restart openbot-worker-firewall.service + systemctl restart openbot-worker.service + expected_services=(codex-gateway grok-gateway) +fi + +systemctl start openbot-health-metrics.timer + +for _ in $(seq 1 60); do + running="$(docker compose \ + --env-file "/etc/openbot/${role}.env" \ + -f "$deploy_root/compose.${role}.yml" \ + ps --status running --services | sort)" + expected="$(printf '%s\n' "${expected_services[@]}" | sort)" + if [[ "$running" == "$expected" ]]; then + if [[ "$role" == "worker" ]]; then + curl --fail --silent --show-error http://127.0.0.1:4210/health >/dev/null \ + && curl --fail --silent --show-error http://127.0.0.1:4212/health >/dev/null \ + && exit 0 + else + exit 0 + fi + fi + sleep 2 +done + +docker compose \ + --env-file "/etc/openbot/${role}.env" \ + -f "$deploy_root/compose.${role}.yml" ps >&2 +echo "$role services did not become ready" >&2 +exit 1 diff --git a/deploy/aws/bin/publish-health-metrics b/deploy/aws/bin/publish-health-metrics new file mode 100755 index 000000000..85f49cbc6 --- /dev/null +++ b/deploy/aws/bin/publish-health-metrics @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "usage: publish-health-metrics ROLE AWS_REGION" >&2 + exit 64 +fi + +role="$1" +aws_region="$2" +if [[ "$role" != "control" && "$role" != "worker" ]]; then + echo "role must be control or worker" >&2 + exit 65 +fi + +metric() { + local name="$1" + local value="$2" + local unit="$3" + aws cloudwatch put-metric-data \ + --region "$aws_region" \ + --namespace OpenBot/Personal \ + --metric-data "MetricName=$name,Dimensions=[{Name=HostRole,Value=$role}],Value=$value,Unit=$unit" +} + +disk_used="$(df --output=pcent "/srv/openbot-${role}" | tail -1 | tr -dc '0-9')" +metric DiskUsedPercent "$disk_used" Percent + +if [[ "$role" == "control" ]]; then + healthy=0 + if docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' \ + openbot-control-openbot-1 2>/dev/null | grep -qx healthy; then + healthy=1 + fi + metric ServiceReady "$healthy" Count + exit 0 +fi + +ready=1 +queue_depth=0 +active_runs=0 +for port in 4210 4212; do + status="$(curl --fail --silent --show-error "http://127.0.0.1:${port}/health")" || { + ready=0 + continue + } + queue_depth=$((queue_depth + $(jq -er '.queueDepth | numbers' <<< "$status"))) + active_runs=$((active_runs + $(jq -er '.activeRuns | numbers' <<< "$status"))) + curl --fail --silent "http://127.0.0.1:${port}/ready" >/dev/null || ready=0 +done +metric ServiceReady "$ready" Count +metric QueueDepth "$queue_depth" Count +metric ActiveRuns "$active_runs" Count diff --git a/deploy/aws/bin/render-secret-env b/deploy/aws/bin/render-secret-env new file mode 100755 index 000000000..97466f9f5 --- /dev/null +++ b/deploy/aws/bin/render-secret-env @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +if [[ "$#" -ne 2 ]]; then + echo "usage: render-secret-env REFERENCE_FILE OUTPUT_FILE" >&2 + exit 64 +fi + +reference_file="$1" +output_file="$2" +secret_arn="$(sed -n 's/^SECRET_ARN=//p' "$reference_file")" +aws_region="$(sed -n 's/^AWS_REGION=//p' "$reference_file")" + +if [[ -z "$secret_arn" || -z "$aws_region" ]]; then + echo "the reference file must set SECRET_ARN and AWS_REGION" >&2 + exit 65 +fi + +temporary="$(mktemp "${output_file}.XXXXXX")" +trap 'rm -f "$temporary"' EXIT + +aws secretsmanager get-secret-value \ + --region "$aws_region" \ + --secret-id "$secret_arn" \ + --query SecretString \ + --output text \ + | jq -er ' + if type != "object" or + (all(to_entries[]; (.key | test("^[A-Z][A-Z0-9_]*$")) and (.value | type == "string")) | not) + then error("secret must be an object of uppercase env names and string values") + else to_entries[] | "\(.key)=\(.value | @json)" + end + ' > "$temporary" + +chmod 0600 "$temporary" +mv -f "$temporary" "$output_file" +trap - EXIT diff --git a/deploy/aws/bin/worker-firewall b/deploy/aws/bin/worker-firewall new file mode 100755 index 000000000..9df194225 --- /dev/null +++ b/deploy/aws/bin/worker-firewall @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +# IMDS remains available to root-owned host services. Containers cannot reach it. +metadata_cidr="169.254.169.254/32" +if ! iptables -C DOCKER-USER -d "$metadata_cidr" -j REJECT 2>/dev/null; then + iptables -I DOCKER-USER 1 -d "$metadata_cidr" -j REJECT +fi diff --git a/deploy/aws/caddy/Caddyfile b/deploy/aws/caddy/Caddyfile new file mode 100644 index 000000000..ed0c1200b --- /dev/null +++ b/deploy/aws/caddy/Caddyfile @@ -0,0 +1,11 @@ +{$OPENBOT_DOMAIN} { + encode zstd gzip + reverse_proxy openbot:3001 + + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains" + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + } +} diff --git a/deploy/aws/claude-managed-settings.json b/deploy/aws/claude-managed-settings.json new file mode 100644 index 000000000..5c6bb1c53 --- /dev/null +++ b/deploy/aws/claude-managed-settings.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "allowManagedHooksOnly": true, + "hooks": {}, + "allowedHttpHookUrls": [], + "httpHookAllowedEnvVars": [], + "allowManagedMcpServersOnly": true, + "allowedMcpServers": [], + "allowManagedPermissionRulesOnly": true, + "permissions": { + "defaultMode": "dontAsk", + "disableBypassPermissionsMode": "disable", + "allow": ["Agent", "Bash", "Edit", "Glob", "Grep", "Read", "Write"], + "deny": [ + "Read(//home/gateway/.claude/**)", + "Read(//var/lib/openbot-gateway/**)", + "Read(//proc/**)", + "Read(//run/**)", + "Read(//sys/**)", + "Read(./.env)", + "Read(./.env.*)", + "Edit(//home/gateway/.claude/**)", + "Edit(//var/lib/openbot-gateway/**)", + "Edit(//etc/**)", + "Edit(//usr/**)" + ] + }, + "sandbox": { + "enabled": true, + "failIfUnavailable": true, + "autoAllowBashIfSandboxed": true, + "allowUnsandboxedCommands": false, + "enableWeakerNestedSandbox": true, + "excludedCommands": [], + "filesystem": { + "denyRead": [ + "/home/gateway/.claude", + "/var/lib/openbot-gateway", + "/proc", + "/run", + "/sys" + ], + "denyWrite": [ + "/home/gateway/.claude", + "/var/lib/openbot-gateway", + "/bin", + "/etc", + "/sbin", + "/usr" + ], + "allowManagedReadPathsOnly": true + } + }, + "disableClaudeAiConnectors": true, + "strictKnownMarketplaces": [], + "includeGitInstructions": false +} diff --git a/deploy/aws/codex-config.toml b/deploy/aws/codex-config.toml new file mode 100644 index 000000000..6984c4af1 --- /dev/null +++ b/deploy/aws/codex-config.toml @@ -0,0 +1,23 @@ +default_permissions = "openbot-worker" + +[features] +network_proxy = true + +[permissions.openbot-worker] +description = "Write only in the active OpenBot job workspace." +extends = ":workspace" + +[permissions.openbot-worker.filesystem] +":root" = "deny" +":minimal" = "read" +glob_scan_max_depth = 4 + +[permissions.openbot-worker.filesystem.":workspace_roots"] +"." = "write" +"**/*.env" = "deny" + +[permissions.openbot-worker.network] +enabled = true + +[permissions.openbot-worker.network.domains] +"*" = "allow" diff --git a/deploy/aws/codex-requirements.toml b/deploy/aws/codex-requirements.toml new file mode 100644 index 000000000..b816650b3 --- /dev/null +++ b/deploy/aws/codex-requirements.toml @@ -0,0 +1,5 @@ +default_permissions = "openbot-worker" +allowed_approval_policies = ["never"] + +[allowed_permission_profiles] +"openbot-worker" = true diff --git a/deploy/aws/compose.control.yml b/deploy/aws/compose.control.yml new file mode 100644 index 000000000..910ced419 --- /dev/null +++ b/deploy/aws/compose.control.yml @@ -0,0 +1,154 @@ +name: openbot-control + +x-aws-logging: &aws-logging + driver: awslogs + options: + awslogs-region: ${AWS_REGION:?set in the rendered control env file} + awslogs-group: ${CONTROL_LOG_GROUP:?set in the rendered control env file} + awslogs-stream: openbot-control + +services: + postgres: + image: ${POSTGRES_IMAGE:?set a digest-qualified image reference} + restart: unless-stopped + environment: + POSTGRES_DB: openbot + POSTGRES_USER: openbot + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set in the rendered control env file} + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - type: bind + source: /srv/openbot-control/postgres + target: /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U openbot -d openbot"] + interval: 10s + timeout: 5s + retries: 12 + networks: [control-data] + logging: *aws-logging + + openbot: + image: ${OPENBOT_IMAGE:?set a digest-qualified image reference} + restart: unless-stopped + env_file: + - path: ${CONTROL_ENV_FILE:?set the root-only rendered control env file} + required: true + environment: + NODE_ENV: production + PORT: "3001" + DATABASE_URL: ${DATABASE_URL:?set in the rendered control env file} + OPENBOT_SINGLE_USER: "false" + AGENT_ENDPOINT_ALLOWED_HOSTS: worker.openbot.internal:4210,worker.openbot.internal:4211,worker.openbot.internal:4212 + depends_on: + postgres: + condition: service_healthy + expose: ["3001"] + healthcheck: + test: ["CMD-SHELL", "bun -e \"await fetch('http://localhost:3001/health').then(r => { if (!r.ok) process.exit(1) })\""] + interval: 15s + timeout: 5s + retries: 8 + networks: [control-data, web] + logging: *aws-logging + + caddy: + image: ${CADDY_IMAGE:?set a digest-qualified image reference} + restart: unless-stopped + environment: + OPENBOT_DOMAIN: ${OPENBOT_DOMAIN:?set the public OpenBot domain} + ports: + - target: 80 + published: "80" + protocol: tcp + - target: 443 + published: "443" + protocol: tcp + - target: 443 + published: "443" + protocol: udp + volumes: + - type: bind + source: ./caddy/Caddyfile + target: /etc/caddy/Caddyfile + read_only: true + - type: bind + source: /srv/openbot-control/caddy/data + target: /data + - type: bind + source: /srv/openbot-control/caddy/config + target: /config + depends_on: [openbot] + networks: [web] + logging: *aws-logging + + routine-sweep: + profiles: [jobs] + image: ${OPENBOT_IMAGE:?set a digest-qualified image reference} + restart: "no" + entrypoint: ["bun"] + command: ["scripts/fire-routines.ts"] + working_dir: /app/server + env_file: + - path: ${CONTROL_ENV_FILE:?set the root-only rendered control env file} + required: true + environment: + DATABASE_URL: ${DATABASE_URL:?set in the rendered control env file} + SERVER_INTERNAL_URL: http://openbot:3001 + WORKER_SHARED_SECRET: ${WORKER_SHARED_SECRET:?set in the rendered control env file} + networks: [control-data, web] + logging: *aws-logging + + migrate: + profiles: [jobs] + image: ${OPENBOT_IMAGE:?set a digest-qualified image reference} + restart: "no" + entrypoint: ["sh"] + command: ["-c", "cd /app/server && bun scripts/migrate.ts"] + env_file: + - path: ${CONTROL_ENV_FILE:?set the root-only rendered control env file} + required: true + environment: + DATABASE_URL: ${DATABASE_URL:?set in the rendered control env file} + networks: [control-data] + logging: *aws-logging + + database-backup: + profiles: [jobs] + image: ${POSTGRES_IMAGE:?set a digest-qualified image reference} + restart: "no" + environment: + PGHOST: postgres + PGDATABASE: openbot + PGUSER: openbot + PGPASSWORD: ${POSTGRES_PASSWORD:?set in the rendered control env file} + entrypoint: ["sh"] + command: + - -c + - >- + umask 077 && + pg_dump --format=custom --file=/backups/openbot-$$(date -u +%Y%m%dT%H%M%SZ).dump && + find /backups -type f -name 'openbot-*.dump' -mtime +7 -delete + volumes: + - type: bind + source: /srv/openbot-control/backups + target: /backups + networks: [control-data] + logging: *aws-logging + + attachment-cleanup: + profiles: [jobs] + image: ${OPENBOT_IMAGE:?set a digest-qualified image reference} + restart: "no" + entrypoint: ["bun"] + command: ["scripts/cull-staged-attachments.ts", "24"] + working_dir: /app/server + environment: + DATABASE_URL: ${DATABASE_URL:?set in the rendered control env file} + networks: [control-data] + logging: *aws-logging + +networks: + control-data: + internal: true + web: {} diff --git a/deploy/aws/compose.worker.yml b/deploy/aws/compose.worker.yml new file mode 100644 index 000000000..a581eba20 --- /dev/null +++ b/deploy/aws/compose.worker.yml @@ -0,0 +1,154 @@ +name: openbot-worker + +x-gateway: &gateway + image: ${GATEWAY_IMAGE:?set a digest-qualified image reference} + restart: unless-stopped + user: "10001:10001" + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + stop_grace_period: 30s + env_file: + - path: ${WORKER_ENV_FILE:?set the root-only rendered worker env file} + required: true + environment: &gateway-environment + HOME: /home/gateway + REPOSITORY_URL: ${REPOSITORY_URL:?set the one allowed repository} + BASE_BRANCH: ${BASE_BRANCH:?set the allowed base branch} + QUEUE_CAPACITY: "1" + PROVIDER_CONCURRENCY: "1" + HOST_CONCURRENCY: "1" + HOST_SEMAPHORE_PATH: /var/lib/openbot-gateway/host-semaphore.sqlite + AWS_EC2_METADATA_DISABLED: "true" + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=256m + - /run:rw,noexec,nosuid,nodev,size=16m + - /home/gateway/.cache:rw,noexec,nosuid,nodev,size=256m + healthcheck: + test: ["CMD-SHELL", "bun -e \"await fetch('http://localhost:' + process.env.PORT + '/health').then(r => { if (!r.ok) process.exit(1) })\""] + interval: 15s + timeout: 5s + retries: 5 + deploy: + resources: + limits: + cpus: "2.0" + memory: 6g + +services: + codex-gateway: + <<: *gateway + # Codex uses Bubblewrap for its Linux file and network sandbox. Docker's default seccomp + # profile blocks the unprivileged namespace call. The outer container stays non-root, + # drops all capabilities, keeps no-new-privileges, and has no Docker socket. + security_opt: ["no-new-privileges:true", "seccomp=unconfined"] + environment: + <<: *gateway-environment + PROVIDER: codex + GATEWAY_TOKEN: ${CODEX_GATEWAY_TOKEN:?set in the rendered worker env file} + PORT: "4210" + JOB_ROOT: /workspaces + RUN_STORE_PATH: /var/lib/openbot-gateway/codex.sqlite + CODEX_HOME: /home/gateway/.codex + CODEX_VERSION: "0.154.0" + CODEX_DISABLE_AUTO_UPDATE: "true" + ports: + - target: 4210 + published: "4210" + protocol: tcp + volumes: + - type: bind + source: /srv/openbot-worker/state + target: /var/lib/openbot-gateway + - type: bind + source: /srv/openbot-worker/workspaces/codex + target: /workspaces + - type: bind + source: /srv/openbot-auth/codex + target: /home/gateway/.codex + - type: bind + source: ./codex-config.toml + target: /home/gateway/.codex/config.toml + read_only: true + logging: + driver: awslogs + options: + awslogs-region: ${AWS_REGION:?set in the rendered worker env file} + awslogs-group: ${WORKER_LOG_GROUP:?set in the rendered worker env file} + awslogs-stream: gateway-codex + + claude-gateway: + <<: *gateway + profiles: ["claude-subscription"] + environment: + <<: *gateway-environment + PROVIDER: claude + GATEWAY_TOKEN: ${CLAUDE_GATEWAY_TOKEN:?set in the rendered worker env file} + PORT: "4211" + JOB_ROOT: /workspaces + RUN_STORE_PATH: /var/lib/openbot-gateway/claude.sqlite + CLAUDE_CONFIG_DIR: /home/gateway/.claude + CLAUDE_VERSION: "2.1.271" + CLAUDE_CODE_DISABLE_AUTO_UPDATER: "1" + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + ports: + - target: 4211 + published: "4211" + protocol: tcp + volumes: + - type: bind + source: /srv/openbot-worker/state + target: /var/lib/openbot-gateway + - type: bind + source: /srv/openbot-worker/workspaces/claude + target: /workspaces + - type: bind + source: /srv/openbot-auth/claude + target: /home/gateway/.claude + - type: bind + source: ./claude-managed-settings.json + target: /etc/claude-code/managed-settings.json + read_only: true + logging: + driver: awslogs + options: + awslogs-region: ${AWS_REGION:?set in the rendered worker env file} + awslogs-group: ${WORKER_LOG_GROUP:?set in the rendered worker env file} + awslogs-stream: gateway-claude + + grok-gateway: + <<: *gateway + # Grok's strict Linux sandbox uses an unprivileged user namespace. Docker's default seccomp + # profile blocks that call. The process stays non-root, has no capabilities, has no Docker + # socket, and keeps no-new-privileges while Grok installs its tighter file and network policy. + security_opt: ["no-new-privileges:true", "seccomp=unconfined"] + environment: + <<: *gateway-environment + PROVIDER: grok + GATEWAY_TOKEN: ${GROK_GATEWAY_TOKEN:?set in the rendered worker env file} + PORT: "4212" + JOB_ROOT: /workspaces + RUN_STORE_PATH: /var/lib/openbot-gateway/grok.sqlite + GROK_HOME: /home/gateway/.grok + GROK_VERSION: "1.0.30" + GROK_DISABLE_AUTOUPDATER: "1" + ports: + - target: 4212 + published: "4212" + protocol: tcp + volumes: + - type: bind + source: /srv/openbot-worker/state + target: /var/lib/openbot-gateway + - type: bind + source: /srv/openbot-worker/workspaces/grok + target: /workspaces + - type: bind + source: /srv/openbot-auth/grok + target: /home/gateway/.grok + logging: + driver: awslogs + options: + awslogs-region: ${AWS_REGION:?set in the rendered worker env file} + awslogs-group: ${WORKER_LOG_GROUP:?set in the rendered worker env file} + awslogs-stream: gateway-grok diff --git a/deploy/aws/control-secret.example.json b/deploy/aws/control-secret.example.json new file mode 100644 index 000000000..d9eede4e9 --- /dev/null +++ b/deploy/aws/control-secret.example.json @@ -0,0 +1,26 @@ +{ + "CONTROL_ENV_FILE": "/etc/openbot/control.env", + "AWS_REGION": "us-east-1", + "CONTROL_LOG_GROUP": "/openbot/openbot-personal/control", + "OPENBOT_IMAGE": "ACCOUNT.dkr.ecr.REGION.amazonaws.com/openbot@sha256:REPLACE", + "POSTGRES_IMAGE": "docker.io/pgvector/pgvector@sha256:REPLACE", + "CADDY_IMAGE": "docker.io/library/caddy@sha256:REPLACE", + "OPENBOT_DOMAIN": "openbot.example.com", + "POSTGRES_PASSWORD": "REPLACE", + "DATABASE_URL": "postgres://openbot:REPLACE@postgres:5432/openbot", + "KEY_ENCRYPTION_KEY": "REPLACE", + "INTELLIGENCE_API_URL": "https://api.intelligence.copilotkit.ai", + "INTELLIGENCE_GATEWAY_WS_URL": "wss://realtime.intelligence.copilotkit.ai", + "INTELLIGENCE_API_KEY": "REPLACE", + "BETTER_AUTH_URL": "https://openbot.example.com", + "BETTER_AUTH_SECRET": "REPLACE", + "GOOGLE_OAUTH_CLIENT_ID": "REPLACE", + "GOOGLE_OAUTH_CLIENT_SECRET": "REPLACE", + "OPENBOT_OWNER_EMAIL": "owner@example.com", + "INITIAL_ADMIN_EMAILS": "owner@example.com", + "WORKER_SHARED_SECRET": "REPLACE", + "CODEX_AGENT_AG_UI_URL": "http://worker.openbot.internal:4210/ag-ui", + "CODEX_AGENT_AG_UI_TOKEN": "MATCH_CODEX_GATEWAY_TOKEN", + "GROK_AGENT_AG_UI_URL": "http://worker.openbot.internal:4212/ag-ui", + "GROK_AGENT_AG_UI_TOKEN": "MATCH_GROK_GATEWAY_TOKEN" +} diff --git a/deploy/aws/grok-requirements.toml b/deploy/aws/grok-requirements.toml new file mode 100644 index 000000000..b47b24bee --- /dev/null +++ b/deploy/aws/grok-requirements.toml @@ -0,0 +1,64 @@ +fail_closed = true +disable_web_search = false + +[cli] +auto_update = false +use_leader = false + +[sandbox] +profile = "strict" +auto_allow_bash = true + +[shell_environment_policy] +inherit = "core" +ignore_default_excludes = false +include_only = ["PATH", "HOME", "LANG", "LC_ALL", "TZ", "TMPDIR", "USER"] + +[session] +load_envrc = false + +[permission] +deny = [ + "Read(/home/gateway/.grok/**)", + "Edit(/home/gateway/.grok/**)", + "Write(/home/gateway/.grok/**)", + "Read(**/.env)", + "Read(**/.env.local)", + "Read(**/*.pem)", + "Read(**/*.key)", + "Bash(git push *)", + "Bash(gh *)", + "Bash(aws *)", + "Bash(docker *)", +] + +[compat.claude] +hooks = false +mcps = false + +[compat.codex] +hooks = false + +[compat.cursor] +hooks = false +mcps = false + +[managed_mcps] +enabled = false +gateway_tools_enabled = false + +[memory] +enabled = false + +[relay] +enabled = false + +[subagents] +enabled = false + +[telemetry] +otel_enabled = false +trace_upload = false + +[tools] +respect_gitignore = true diff --git a/deploy/aws/systemd/openbot-attachment-cleanup.service b/deploy/aws/systemd/openbot-attachment-cleanup.service new file mode 100644 index 000000000..4c4208222 --- /dev/null +++ b/deploy/aws/systemd/openbot-attachment-cleanup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Remove expired staged OpenBot attachments +Requires=openbot-control.service +After=openbot-control.service + +[Service] +Type=oneshot +WorkingDirectory=/opt/openbot/deploy/aws +ExecStart=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml --profile jobs run --rm --no-deps attachment-cleanup diff --git a/deploy/aws/systemd/openbot-attachment-cleanup.timer b/deploy/aws/systemd/openbot-attachment-cleanup.timer new file mode 100644 index 000000000..559622588 --- /dev/null +++ b/deploy/aws/systemd/openbot-attachment-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Remove expired staged OpenBot attachments hourly + +[Timer] +OnCalendar=hourly +Persistent=true +RandomizedDelaySec=5m +Unit=openbot-attachment-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/deploy/aws/systemd/openbot-control-env.service b/deploy/aws/systemd/openbot-control-env.service new file mode 100644 index 000000000..a9ffb7cf9 --- /dev/null +++ b/deploy/aws/systemd/openbot-control-env.service @@ -0,0 +1,12 @@ +[Unit] +Description=Render OpenBot control environment from Secrets Manager +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/opt/openbot/deploy/aws/bin/render-secret-env /etc/openbot/control-secret-ref /etc/openbot/control.env +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/aws/systemd/openbot-control.service b/deploy/aws/systemd/openbot-control.service new file mode 100644 index 000000000..5704b86a0 --- /dev/null +++ b/deploy/aws/systemd/openbot-control.service @@ -0,0 +1,21 @@ +[Unit] +Description=OpenBot control services +Requires=docker.service openbot-control-env.service +After=docker.service openbot-control-env.service network-online.target +RequiresMountsFor=/srv/openbot-control + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/openbot/deploy/aws +ExecStartPre=/opt/openbot/deploy/aws/bin/ecr-login /etc/openbot/registry-ref +ExecStartPre=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml pull +ExecStartPre=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml up --detach --wait postgres +ExecStartPre=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml --profile jobs run --rm --no-deps migrate +ExecStart=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml up --detach --remove-orphans +ExecStop=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml down +TimeoutStartSec=600 +TimeoutStopSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/aws/systemd/openbot-database-backup.service b/deploy/aws/systemd/openbot-database-backup.service new file mode 100644 index 000000000..dbcd0b53c --- /dev/null +++ b/deploy/aws/systemd/openbot-database-backup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Create an application-consistent OpenBot database backup +Requires=openbot-control.service +After=openbot-control.service + +[Service] +Type=oneshot +WorkingDirectory=/opt/openbot/deploy/aws +ExecStart=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml --profile jobs run --rm --no-deps database-backup diff --git a/deploy/aws/systemd/openbot-database-backup.timer b/deploy/aws/systemd/openbot-database-backup.timer new file mode 100644 index 000000000..a989b1239 --- /dev/null +++ b/deploy/aws/systemd/openbot-database-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Back up the OpenBot database each day + +[Timer] +OnCalendar=*-*-* 04:30:00 +Persistent=true +RandomizedDelaySec=5m +Unit=openbot-database-backup.service + +[Install] +WantedBy=timers.target diff --git a/deploy/aws/systemd/openbot-health-metrics.service b/deploy/aws/systemd/openbot-health-metrics.service new file mode 100644 index 000000000..6e3ac633a --- /dev/null +++ b/deploy/aws/systemd/openbot-health-metrics.service @@ -0,0 +1,9 @@ +[Unit] +Description=Publish OpenBot service health metrics +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=/etc/openbot/metrics-ref +ExecStart=/opt/openbot/deploy/aws/bin/publish-health-metrics ${OPENBOT_HOST_ROLE} ${AWS_REGION} diff --git a/deploy/aws/systemd/openbot-health-metrics.timer b/deploy/aws/systemd/openbot-health-metrics.timer new file mode 100644 index 000000000..d9e33a924 --- /dev/null +++ b/deploy/aws/systemd/openbot-health-metrics.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check OpenBot service health each minute + +[Timer] +OnCalendar=*-*-* *:*:00 +Persistent=true +RandomizedDelaySec=10 +Unit=openbot-health-metrics.service + +[Install] +WantedBy=timers.target diff --git a/deploy/aws/systemd/openbot-routines.service b/deploy/aws/systemd/openbot-routines.service new file mode 100644 index 000000000..5d68d17f6 --- /dev/null +++ b/deploy/aws/systemd/openbot-routines.service @@ -0,0 +1,9 @@ +[Unit] +Description=Run one OpenBot routine sweep +Requires=openbot-control.service +After=openbot-control.service + +[Service] +Type=oneshot +WorkingDirectory=/opt/openbot/deploy/aws +ExecStart=/usr/bin/docker compose --env-file /etc/openbot/control.env -f compose.control.yml --profile jobs run --rm --no-deps routine-sweep diff --git a/deploy/aws/systemd/openbot-routines.timer b/deploy/aws/systemd/openbot-routines.timer new file mode 100644 index 000000000..4ee293674 --- /dev/null +++ b/deploy/aws/systemd/openbot-routines.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run OpenBot routine sweeps every minute + +[Timer] +OnCalendar=*-*-* *:*:00 +Persistent=true +RandomizedDelaySec=5 +Unit=openbot-routines.service + +[Install] +WantedBy=timers.target diff --git a/deploy/aws/systemd/openbot-worker-env.service b/deploy/aws/systemd/openbot-worker-env.service new file mode 100644 index 000000000..49e20eaa3 --- /dev/null +++ b/deploy/aws/systemd/openbot-worker-env.service @@ -0,0 +1,12 @@ +[Unit] +Description=Render OpenBot worker environment from Secrets Manager +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/opt/openbot/deploy/aws/bin/render-secret-env /etc/openbot/worker-secret-ref /etc/openbot/worker.env +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/aws/systemd/openbot-worker-firewall.service b/deploy/aws/systemd/openbot-worker-firewall.service new file mode 100644 index 000000000..3f7c6a802 --- /dev/null +++ b/deploy/aws/systemd/openbot-worker-firewall.service @@ -0,0 +1,13 @@ +[Unit] +Description=Block OpenBot worker containers from EC2 instance metadata +Requires=docker.service +After=docker.service +Before=openbot-worker.service + +[Service] +Type=oneshot +ExecStart=/opt/openbot/deploy/aws/bin/worker-firewall +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/aws/systemd/openbot-worker.service b/deploy/aws/systemd/openbot-worker.service new file mode 100644 index 000000000..8459c68db --- /dev/null +++ b/deploy/aws/systemd/openbot-worker.service @@ -0,0 +1,19 @@ +[Unit] +Description=OpenBot subscription worker services +Requires=docker.service openbot-worker-env.service openbot-worker-firewall.service +After=docker.service openbot-worker-env.service openbot-worker-firewall.service network-online.target +RequiresMountsFor=/srv/openbot-worker /srv/openbot-auth/codex /srv/openbot-auth/claude /srv/openbot-auth/grok + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/opt/openbot/deploy/aws +ExecStartPre=/opt/openbot/deploy/aws/bin/ecr-login /etc/openbot/registry-ref +ExecStartPre=/usr/bin/docker compose --env-file /etc/openbot/worker.env -f compose.worker.yml pull +ExecStart=/usr/bin/docker compose --env-file /etc/openbot/worker.env -f compose.worker.yml up --detach --remove-orphans +ExecStop=/usr/bin/docker compose --env-file /etc/openbot/worker.env -f compose.worker.yml down +TimeoutStartSec=600 +TimeoutStopSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/aws/worker-secret.example.json b/deploy/aws/worker-secret.example.json new file mode 100644 index 000000000..a776bf4e1 --- /dev/null +++ b/deploy/aws/worker-secret.example.json @@ -0,0 +1,11 @@ +{ + "WORKER_ENV_FILE": "/etc/openbot/worker.env", + "AWS_REGION": "us-east-1", + "WORKER_LOG_GROUP": "/openbot/openbot-personal/worker", + "GATEWAY_IMAGE": "ACCOUNT.dkr.ecr.REGION.amazonaws.com/subscription-gateway@sha256:REPLACE", + "CODEX_GATEWAY_TOKEN": "REPLACE_WITH_DISTINCT_RANDOM_VALUE", + "CLAUDE_GATEWAY_TOKEN": "REPLACE_WITH_DISTINCT_RANDOM_VALUE", + "GROK_GATEWAY_TOKEN": "REPLACE_WITH_DISTINCT_RANDOM_VALUE", + "REPOSITORY_URL": "https://github.com/Tbsheff/OpenBot.git", + "BASE_BRANCH": "main" +} diff --git a/docs/configuration.md b/docs/configuration.md index 5d5e5a80c..6e5e018ec 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,6 +38,28 @@ coworker without its own endpoint is refused. A leftover token with no URL is ig one-container image has no Bot process, so leave the URL unset there. `scripts/start.sh` points it at `agent-langgraph` on a laptop. +Subscription coding workers use one URL and token pair per provider: + +| Provider | Endpoint | Gateway token | +| -------- | -------- | ------------- | +| Codex | `CODEX_AGENT_AG_UI_URL` | `CODEX_AGENT_TOKEN` | +| Claude | `CLAUDE_AGENT_AG_UI_URL` | `CLAUDE_AGENT_TOKEN` | +| Grok | `GROK_AGENT_AG_UI_URL` | `GROK_AGENT_TOKEN` | + +Each pair is optional. Set Codex first, then Claude, then Grok. A provider with neither value stays +off the roster. A half-set pair stops startup. Each endpoint must also have its exact host and port +in `AGENT_ENDPOINT_ALLOWED_HOSTS`. For example: + +```dotenv +CODEX_AGENT_AG_UI_URL=http://subscription-workers.openbot.internal:4210/ag-ui +CODEX_AGENT_TOKEN=... +AGENT_ENDPOINT_ALLOWED_HOSTS=subscription-workers.openbot.internal:4210 +``` + +In production, inject each token from AWS Secrets Manager into the OpenBot process. OpenBot sends +the token only to its exact configured endpoint. These tokens authenticate OpenBot to the gateways; +they are not Codex, Claude, or Grok login data. Provider login state stays on the worker host. + ## General variables | Variable | Default | Meaning | @@ -176,6 +198,7 @@ Two things are worth knowing before pointing a deployment at any gateway. Not ev | `BETTER_AUTH_URL` | Public API server base URL, where OAuth callbacks return. Required with any provider. | | `TRUSTED_ORIGINS` | Comma-separated app origins accepted by the API, plus every host in a registered OIDC provider's discovery document. | | `INITIAL_ADMIN_EMAILS` | Comma-separated administrators. **Required** with any provider. | +| `OPENBOT_OWNER_EMAIL` | The only email admitted to this private deployment. **Required** with any provider. Normalized to lower case. | | `OPENBOT_PUBLIC_URL` | Public address of this API. Defaults to `BETTER_AUTH_URL`. | | `OPENBOT_APP_URL` | Where the browser app is served. Defaults to the first `TRUSTED_ORIGINS` entry. | @@ -187,8 +210,13 @@ configuration at all. **Any one provider turns sign-in on**, and several may be configured at once. Each provider's id and secret must be set together, Okta additionally needs its issuer, and any of them requires -`BETTER_AUTH_SECRET`, `BETTER_AUTH_URL` and `INITIAL_ADMIN_EMAILS`. Every incomplete combination is -refused at start-up rather than at somebody's first attempt to sign in. +`BETTER_AUTH_SECRET`, `BETTER_AUTH_URL`, `OPENBOT_OWNER_EMAIL` and `INITIAL_ADMIN_EMAILS`. Every +incomplete combination is refused at start-up rather than at somebody's first attempt to sign in. + +`OPENBOT_OWNER_EMAIL` is an admission check, not a role grant. OpenBot checks it before it creates a +user or session. A valid account from the configured identity provider still gets a refusal when +its normalized email differs. `INITIAL_ADMIN_EMAILS` separately decides who has the administrator +role; the owner email can be listed there, but the settings do not replace each other. `INITIAL_ADMIN_EMAILS` is required because nothing else grants the administrator role at first: an address it names becomes an administrator at every sign-in and cannot be demoted from the People @@ -250,7 +278,7 @@ then is a row nothing will read. | `COMPUTER_SUPERVISOR_URL` | Supervisor URL for per-Bot computers. If absent, Bots share `AGENT_COMPUTER_URL`. | | `SUPERVISOR_TOKEN` | Bearer token required by the supervisor. | | `AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS` | Local-only private-host browsing when `true`. A deployment running with `NODE_ENV=production` refuses to start while it is set. Cloud metadata addresses are refused either way. | -| `AGENT_ENDPOINT_ALLOWED_HOSTS` | Private addresses an agent may be registered at, comma separated; unset (none) by default. Host, optionally with a port. Exact match; no wildcards. Never-allowed addresses cannot be named. | +| `AGENT_ENDPOINT_ALLOWED_HOSTS` | Private addresses an agent may be registered at, comma separated; unset (none) by default. Host, optionally with a port. Exact match; no wildcards. Subscription workers require their exact host and port. Never-allowed addresses cannot be named. | | `AGENT_COMPUTER_POLICY` | JSON action policy: `{"mode":"enforce","deny":[...],"allow":[...]}`. | | `COMPUTER_RUNTIME` | Set to `runsc` to run supervised computers under gVisor. | | `COMPUTER_SANDBOX` | Set to `on` to enable Chromium's own sandbox where the host permits user namespaces. Which way it went is printed at start-up. | diff --git a/docs/coworkers.md b/docs/coworkers.md index 5f4558c5c..1e06e2d8d 100644 --- a/docs/coworkers.md +++ b/docs/coworkers.md @@ -71,6 +71,35 @@ then need their own endpoint, and a package agent whose endpoint expands to noth rather than registered against a missing host. A leftover token with no URL is ignored. Package-provided agents otherwise use their own `agents.yaml` configuration. +## Cloud subscription coworkers + +The default tenant package has optional Codex, Claude, and Grok coworkers. Each one appears only +when its own endpoint and gateway token pair is set. This supports a staged release: configure +Codex, verify it, then add Claude, then Grok. + +```dotenv +CODEX_AGENT_AG_UI_URL=http://subscription-workers.openbot.internal:4210/ag-ui +CODEX_AGENT_TOKEN=... +CLAUDE_AGENT_AG_UI_URL=http://subscription-workers.openbot.internal:4211/ag-ui +CLAUDE_AGENT_TOKEN=... +GROK_AGENT_AG_UI_URL=http://subscription-workers.openbot.internal:4212/ag-ui +GROK_AGENT_TOKEN=... +AGENT_ENDPOINT_ALLOWED_HOSTS=subscription-workers.openbot.internal:4210,subscription-workers.openbot.internal:4211,subscription-workers.openbot.internal:4212 +``` + +The private Route 53 name is stable while the worker host changes. The port list is exact: listing +`4210` does not admit `4211`, another host, or any other private address. Metadata addresses remain +blocked under all settings. + +OpenBot holds only the gateway bearer values and sends each one only to its exact configured URL. +Use deployment secrets for them. The official coding programs own their subscription login state on +the worker host; do not put provider tokens or login files in OpenBot, the tenant package, or these +gateway settings. + +These package coworkers are deployment-owned. Their endpoint and gateway token come from the +environment, not from the coworker edit form. Customer-created remote coworkers keep using the +write-only authorization header flow and encrypted credential vault described below. + ## Register an external AG-UI agent In `agents.yaml`: diff --git a/docs/deployment.md b/docs/deployment.md index 86c17f9b3..2120fadc1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -181,6 +181,8 @@ supervisor is still not in this image, so every replica shares the one browser i ## Platform notes +**Private two-host AWS subscription workers.** This fork also has a personal EC2 deployment that keeps OpenBot on one control host and Codex and Grok on a separate worker host. It uses ECR digests, SSM, retained provider auth volumes, fixed concurrency, owner email admission, CloudWatch alerts, and AWS Backup. Claude is present but disabled until its release gate passes. See [the AWS deployment runbook](runbooks/aws-deployment.md). This is a private single-owner mode, not a way to share personal provider subscriptions. + **Google Cloud Run.** Set memory to at least 2 GB. More than one instance is fine (see Replicas above); each instance has its own browser, so a Bot's logins stay on whichever instance served them. Cloud Run runs every diff --git a/docs/runbooks/aws-deployment.md b/docs/runbooks/aws-deployment.md new file mode 100644 index 000000000..69704c293 --- /dev/null +++ b/docs/runbooks/aws-deployment.md @@ -0,0 +1,77 @@ +# Deploy the private AWS stack + +This runbook deploys one owner-only OpenBot control host and one cloud worker host. It does not support shared use of personal provider accounts. + +## 1. Set the AWS prerequisites + +Use an AWS account where you can run Terraform. Choose one region. Create these items before the stack: + +- two Secrets Manager secrets, one from `deploy/aws/control-secret.example.json` and one from `deploy/aws/worker-secret.example.json`; +- the account-wide GitHub Actions OIDC provider for `https://token.actions.githubusercontent.com`, with audience `sts.amazonaws.com`; +- a public DNS `A` record you can later point to the control Elastic IP; +- Google, Microsoft, or Okta OAuth values for that public domain; +- CopilotKit Intelligence values required by OpenBot. + +Generate all secret values on your own machine. Use a distinct random value for each provider gateway token. The matching control and worker token values must be equal for that provider. Do not put any provider login token in either AWS secret. + +Resolve and store digest-qualified PostgreSQL and Caddy image references in the control secret. Leave the OpenBot and gateway image fields as valid digest placeholders until the first deploy workflow replaces them. + +## 2. Create the AWS stack + +Copy the example variables and set the fixed AMI, secret ARNs, GitHub repository, OIDC provider ARN, and alert email: + +```bash +cp infra/aws/environments/personal/terraform.tfvars.example \ + infra/aws/environments/personal/terraform.tfvars + +terraform -chdir=infra/aws/environments/personal init +terraform -chdir=infra/aws/environments/personal fmt -check +terraform -chdir=infra/aws/environments/personal validate +terraform -chdir=infra/aws/environments/personal plan -out=openbot.tfplan +terraform -chdir=infra/aws/environments/personal apply openbot.tfplan +``` + +Confirm the SNS email subscription. Point the public DNS record at `control_public_ip`. Wait for DNS to resolve before the first control deploy so Caddy can get a TLS certificate. + +The worker has a public address only for outbound provider and repository access. Its security group has no public inbound rule and no SSH rule. Use SSM for host access. + +## 3. Configure the protected GitHub environment + +Create a GitHub environment named `production` in `Tbsheff/OpenBot`. Require your approval. Add these environment variables from Terraform outputs and AWS: + +| Variable | Value | +| --- | --- | +| `AWS_ROLE_ARN` | `github_actions_deploy_role_arn` output | +| `AWS_REGION` | Terraform region | +| `AWS_CONTROL_INSTANCE_ID` | `control_instance_id` output | +| `AWS_WORKER_INSTANCE_ID` | `worker_instance_id` output | +| `AWS_CONTROL_SECRET_ARN` | control secret ARN | +| `AWS_WORKER_SECRET_ARN` | worker secret ARN | +| `AWS_OPENBOT_ECR_REPOSITORY_URL` | `ecr_repository_urls["openbot-control"]` | +| `AWS_GATEWAY_ECR_REPOSITORY_URL` | `ecr_repository_urls["subscription-gateway"]` | + +Do not add long-lived AWS access keys. The role trust accepts only the repository's `production` environment OIDC subject. + +## 4. Deploy + +Run `Deploy personal AWS stack` from the GitHub Actions page. Select `all`. The workflow does this in order: + +1. Build the OpenBot and gateway images for `linux/amd64`. +2. Push them to immutable ECR repositories. +3. Sign each digest with the workflow OIDC identity and scan for unfixed critical findings. +4. Update only the image field in each existing Secrets Manager JSON object. +5. Send the small host bundle through SSM. +6. Deploy the worker before the control host. +7. Restore the prior image field if a host service fails to start. + +The control service starts PostgreSQL, runs migrations once, and then starts OpenBot and Caddy. The worker starts Codex and Grok. Claude stays off until its legal release gate passes. + +## 5. Sign in and accept the release + +Follow [provider sign-in](provider-sign-in.md). Then sign in to the private OpenBot URL with the exact email in `OPENBOT_OWNER_EMAIL`. + +Run one new Codex task and one new Grok task from OpenBot. For each task, confirm streamed text, retained patch data, continuation on the same thread, cancellation, and no automatic push. Replace the worker container and confirm the provider is still signed in. Do not enable Claude as part of this gate. + +## Limits of a local test + +Terraform validation, Compose rendering, fake provider protocol tests, and local signed-in provider tests do not prove the AWS path. The release is end-to-end only after the protected workflow, DNS and TLS, owner IdP sign-in, provider device sign-ins, OpenBot runs, container replacement, one alert test, and one restore test pass in the target AWS account. diff --git a/docs/runbooks/provider-sign-in.md b/docs/runbooks/provider-sign-in.md new file mode 100644 index 000000000..682abdd51 --- /dev/null +++ b/docs/runbooks/provider-sign-in.md @@ -0,0 +1,129 @@ +# Provider sign-in + +Provider sign-in changes only the provider-owned auth volume on the worker host. Do not put provider credentials in AWS Secrets Manager, the OpenBot database, an image, or this repository. + +## Before you start + +Deploy the worker host and start `openbot-worker.service`. Get its instance ID from Terraform: + +```bash +terraform -chdir=infra/aws/environments/personal output -raw worker_instance_id +``` + +Start an AWS Systems Manager session. Replace the instance ID with the value from the prior command: + +```bash +aws ssm start-session --target i-0123456789abcdef0 +``` + +On the worker host, change to the deployment directory: + +```bash +cd /opt/openbot/deploy/aws +``` + +## Sign in to Codex + +Start the official device-code flow inside the Codex gateway container: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml exec codex-gateway codex login --device-auth +``` + +Open the shown link in your own browser. Sign in to the intended ChatGPT account and enter the one-time code. Do not paste the code or any token into OpenBot, logs, chat, or source control. + +Check the login through the CLI. This command reports the auth method without reading or printing `auth.json`: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml exec codex-gateway codex login status +``` + +Then check gateway readiness from the worker host: + +```bash +curl --fail --silent http://127.0.0.1:4210/ready +``` + +The response must show `"provider":"codex"` and `"authReady":true`. Exit the SSM session when the check passes. + +Codex owns `/home/gateway/.codex` in the container. Compose mounts it from the retained encrypted volume at `/srv/openbot-auth/codex`, so the login survives a container or image replacement. The Codex service cannot mount the Claude or Grok auth volumes. + +If readiness later reports `authReady:false`, repeat the device-code flow. Do not copy the auth cache into an image or secret store. + +## Claude release gate + +Do not enable or sign in to the Claude gateway until the owner records a current authentication-use review for this exact owner-only deployment. Anthropic's [legal and compliance page](https://code.claude.com/docs/en/legal-and-compliance) says subscription OAuth is for ordinary use of Claude Code and other native Anthropic applications. It directs developers who build products or services to API-key authentication and bars routing Free, Pro, or Max credentials on behalf of users. This private, self-hosted, single-owner use is not an on-behalf-of-users service, but the page does not state that this wrapper is allowed. + +Treat that gap as a stop condition. Get written guidance from Anthropic or use a supported API, Team, or Enterprise authentication route before the first live run. Repeat this check before each Claude CLI upgrade. Do not treat a working login as proof that the use is allowed. + +Anthropic also states that, from June 15, 2026, `claude -p` use on subscription plans draws from a separate monthly Agent SDK credit. Check the current plan allowance before enabling this worker. + +## Sign in to Claude after the release gate passes + +Start the official Claude.ai login inside the Claude gateway container: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml --profile claude-subscription up -d claude-gateway + +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml --profile claude-subscription exec claude-gateway claude auth login +``` + +Select the Claude.ai subscription route, not the Console route. On an SSM or container session, the browser callback will usually not reach the remote CLI. Copy the shown URL into your own browser. After sign-in, paste the shown one-time login code back into the terminal when Claude asks for it. + +Do not run `claude setup-token`. Do not set `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_API_KEY`, or `ANTHROPIC_AUTH_TOKEN`. Those paths would turn the login into an application credential or select API billing. The gateway starts the unmodified CLI and never reads its credential file. + +Check the selected login method without opening or copying the credential file: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml --profile claude-subscription exec claude-gateway claude auth status --text +``` + +The status must show a Claude.ai subscription login. Then check the pinned version, managed policy, and gateway readiness: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml --profile claude-subscription exec claude-gateway claude --version + +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml --profile claude-subscription exec claude-gateway claude doctor + +curl --fail --silent http://127.0.0.1:4211/ready +``` + +The version must match the pinned image version. `claude doctor` must report that the Linux managed settings loaded from `/etc/claude-code/managed-settings.json`. The readiness response must show `"provider":"claude"` and `"authReady":true`. + +Claude owns `CLAUDE_CONFIG_DIR=/home/gateway/.claude`. Compose mounts that directory from `/srv/openbot-auth/claude` on its retained encrypted volume. The Claude service cannot mount Codex or Grok login data. The managed policy blocks native tool access to the Claude login directory and gateway state. It uses Claude's documented weaker nested sandbox mode because the outer non-root container drops all capabilities, has a read-only root, has no Docker socket, and cannot create a privileged namespace. Do not run the Claude gateway with a wider container profile. Exit the SSM session after all checks pass. + +If readiness later reports `authReady:false`, run `claude auth login` again. Never print, copy, back up, or move the Claude credential file. + +## Sign in to Grok + +Start xAI's device-code flow inside the Grok gateway container: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml exec grok-gateway grok login --device-auth +``` + +Open the shown link in your own browser. Sign in to the intended Grok account and enter the one-time code. Do not paste the code or any token into OpenBot, logs, chat, or source control. + +Check the installed version and the enforced policy without reading Grok's auth files: + +```bash +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml exec grok-gateway grok --version + +sudo docker compose --env-file /etc/openbot/worker.env \ + -f compose.worker.yml exec grok-gateway grok inspect --json >/dev/null + +curl --fail --silent http://127.0.0.1:4212/ready +``` + +The version must match the pinned image version. The readiness response must show `"provider":"grok"` and `"authReady":true`. The root-owned requirements file pins the strict Linux sandbox, removes child-process secrets, blocks reads of Grok login data and common repository secret files, disables provider subagents, and blocks direct publish tools. + +Grok owns `GROK_HOME=/home/gateway/.grok`. Compose mounts that directory from `/srv/openbot-auth/grok`, so the login survives a container or image replacement. The Grok service cannot mount Codex or Claude login data. If readiness later reports `authReady:false`, repeat the device-code flow. Never print, copy, back up, or move the Grok credential files. diff --git a/docs/runbooks/worker-operations.md b/docs/runbooks/worker-operations.md new file mode 100644 index 000000000..8d27e1ad1 --- /dev/null +++ b/docs/runbooks/worker-operations.md @@ -0,0 +1,49 @@ +# Operate the personal workers + +## Check health + +Use SSM to start a session on the worker. Run: + +```bash +systemctl status openbot-worker.service openbot-health-metrics.timer +sudo docker compose --env-file /etc/openbot/worker.env \ + -f /opt/openbot/deploy/aws/compose.worker.yml ps +curl --fail --silent http://127.0.0.1:4210/ready +curl --fail --silent http://127.0.0.1:4212/ready +``` + +`/health` proves the gateway process is live. `/ready` also checks the provider login. A failed readiness check does not mean the host is down. + +CloudWatch receives container logs and these `OpenBot/Personal` metrics each minute: `ServiceReady`, `DiskUsedPercent`, `QueueDepth`, and `ActiveRuns`. Alerts cover EC2 status, missing or failed service checks, 85 percent disk use, a queue that stays nonempty for 30 minutes, and three provider failures in five minutes. Keep prompt text, tokens, repository files, patches, and provider errors out of logs and alarm messages. + +## Drain and deploy + +The host limit is one active provider run and one queued run. Before a planned deploy, wait until `ActiveRuns` is zero and `QueueDepth` is zero. Run the protected AWS deploy workflow. It uses digest references and restores the prior image setting if startup fails. It does not change or copy provider auth volumes. + +For a manual rollback, select the prior enabled version of the worker secret in Secrets Manager, restore its `GATEWAY_IMAGE` digest, and run: + +```bash +sudo systemctl restart openbot-worker-env.service openbot-worker.service +``` + +Use the same process with `OPENBOT_IMAGE` and the control services for a control rollback. A schema change can make an image-only control rollback unsafe. Read the migration before deploying it and restore the matching database backup when the schema is not backward-compatible. + +## Back up and restore + +The control host writes a custom PostgreSQL dump each day at 04:30 UTC and keeps seven days on its encrypted data volume. AWS Backup snapshots both service-data volumes each day at 05:00 UTC and keeps them for 30 days. The backup selection includes only volumes tagged `Backup=included-service-data`. Provider auth volumes use `Backup=excluded-provider-auth` and must stay out of backup plans. + +Test restore in a separate volume and temporary database. Stop writes, restore the dump with `pg_restore`, start the matching OpenBot digest, and check the owner, coworker, thread, run, and artifact records. Do not overwrite the active volume during a drill. If a provider auth volume is lost, revoke that provider session and run the device sign-in again. Never restore provider login data from a service backup. + +## Recover capacity and disk + +On restart, the gateway marks active and queued ledger rows failed and releases their host leases. It does not resume a native agent without the AG-UI client. If `ActiveRuns` stays above zero with no task open, save the safe gateway log lines, restart the affected service, and check that the metric returns to zero. + +When disk use reaches 85 percent, first remove expired terminal workspaces through the normal cleanup path. Keep the stored patch and commit metadata. Do not delete SQLite ledgers or provider auth directories. Increase the retained EBS volume only after you know which path is growing. + +## Upgrade a provider + +Change one pinned CLI version and its policy at a time. Run its fixture suite, build and scan the image, sign in on a staging auth volume, and pass new-run, continuation, cancellation, failure, and replacement checks before release. Recheck the provider's current terms and login guidance. Claude stays disabled unless Anthropic gives a supported route for this exact use. + +## Monthly cost check + +Review the current AWS bill and the Terraform plan each month. Treat any ALB, NAT Gateway, RDS, EKS, larger EC2 size, added public IPv4 address, longer log retention, or larger backup retention as a budget change. Confirm the plan in the AWS Pricing Calculator for the chosen region before applying it. diff --git a/examples/fintech/agents.yaml b/examples/fintech/agents.yaml index a647508ea..0bc44941b 100644 --- a/examples/fintech/agents.yaml +++ b/examples/fintech/agents.yaml @@ -69,6 +69,33 @@ agents: type: remote-ag-ui endpoint: ${MANAGED_AGENT_AG_UI_URL:-} + # Subscription-backed coding coworkers. Each row removes itself until that provider's endpoint + # is configured, so Codex can ship before Claude and Grok. Gateway tokens stay in deployment + # secrets and are bound to these exact URLs by the server; they never enter this package. + - id: codex + name: Codex + title: Software Engineer + role_description: Work on software engineering tasks with the Codex coding agent. + avatar_seed: codex + type: remote-ag-ui + endpoint: ${CODEX_AGENT_AG_UI_URL:-} + + - id: claude + name: Claude + title: Software Engineer + role_description: Work on software engineering tasks with the Claude Code agent. + avatar_seed: claude + type: remote-ag-ui + endpoint: ${CLAUDE_AGENT_AG_UI_URL:-} + + - id: grok + name: Grok + title: Software Engineer + role_description: Work on software engineering tasks with the Grok coding agent. + avatar_seed: grok + type: remote-ag-ui + endpoint: ${GROK_AGENT_AG_UI_URL:-} + # The Bot somebody picked in setup. # # ONE ROW, AND IT EXISTS ONLY ONCE SOMETHING IS PICKED. A Bot whose endpoint interpolates to diff --git a/infra/aws/environments/personal/.terraform.lock.hcl b/infra/aws/environments/personal/.terraform.lock.hcl new file mode 100644 index 000000000..b57336f80 --- /dev/null +++ b/infra/aws/environments/personal/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.14.1" + constraints = "6.14.1" + hashes = [ + "h1:oacWXQzS6BmkN3lcKKxafYOGUQcUOpneJikw8hqLUyA=", + "zh:14d0b4b3dffb3368e6257136bbab1f93d419863dd65d99ef80ca2c1dd3c72a1e", + "zh:1de3601251f87a0a989c4b3474baa2efcaf491804f8d7afe15421b728bac5dc5", + "zh:2cfe42b853a3b4117bdbb73e5715035eac9b8d753d6e653fd5f30a807a36b985", + "zh:3dd8a0336face356928faf2396065634739ef2c3ac3dcaa655570df205559fd9", + "zh:42712baca386b84e089b1db8b7844038557f4039b32d8702611aa67eadef7d0f", + "zh:4ffc698099e4d7ffc6b0490a4e78ad66b041afd54e988b8bf8e229bcdd4b3ead", + "zh:52a6a3b01cb34394b0d06b273b27702fb9d795290a02e5824e198315787e8446", + "zh:56eae388c48a844401e44811719dc23be84de538468fd12b7265b06acbf4b51d", + "zh:614a918fdf27416b2ee2ce1737895b791f59f9deff3b61246c62a992eabfb8eb", + "zh:68605e159177b57fdc4a26bb2caff69a7b69593a601145b7ab5a86fd44b28b9f", + "zh:771ac00fd5f211052d735ff0e4b9ec67288abd1e22ffea4ed774aec73c7e5687", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a1355841161e5b53dc3078c88aae1972fd4a9c0d30309b18b1951137b96571fa", + "zh:a3c8ca40c1fa7ad76d3d4c3c0039b66a93cc96399e757d2caa0b5cdedce9d3e8", + "zh:c77e02a72ef9eb0eb65faaf84c33af843520622dbb51ec31d04ca371bd4d4ee8", + ] +} diff --git a/infra/aws/environments/personal/main.tf b/infra/aws/environments/personal/main.tf new file mode 100644 index 000000000..1ebe4434d --- /dev/null +++ b/infra/aws/environments/personal/main.tf @@ -0,0 +1,103 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Application = "OpenBot" + Environment = "personal" + ManagedBy = "Terraform" + } + } +} + +data "aws_availability_zones" "available" { + state = "available" +} + +data "aws_ami" "selected" { + most_recent = false + owners = ["amazon"] + + filter { + name = "image-id" + values = [var.ami_id] + } + + filter { + name = "architecture" + values = ["x86_64"] + } +} + +locals { + availability_zones = length(var.availability_zones) >= 2 ? slice(var.availability_zones, 0, 2) : slice(data.aws_availability_zones.available.names, 0, 2) +} + +resource "aws_kms_key" "ebs" { + description = "OpenBot personal EBS and log encryption" + enable_key_rotation = true + deletion_window_in_days = 30 +} + +resource "aws_kms_alias" "ebs" { + name = "alias/${var.name}-storage" + target_key_id = aws_kms_key.ebs.key_id +} + +resource "aws_ebs_encryption_by_default" "this" { + enabled = true +} + +resource "aws_ebs_default_kms_key" "this" { + key_arn = aws_kms_key.ebs.arn +} + +module "network" { + source = "../../modules/network" + + name = var.name + vpc_cidr = var.vpc_cidr + availability_zones = local.availability_zones + private_zone_name = "openbot.internal" +} + +module "registry" { + source = "../../modules/registry" + name = var.name +} + +module "control" { + source = "../../modules/control-host" + + name = var.name + ami_id = data.aws_ami.selected.id + instance_type = "t3.medium" + subnet_id = module.network.control_subnet_id + security_group_id = module.network.control_security_group_id + availability_zone = local.availability_zones[0] + kms_key_arn = aws_kms_key.ebs.arn + repository_arns = module.registry.repository_arns + secret_arns = var.control_secret_arns +} + +module "worker" { + source = "../../modules/worker-host" + + name = var.name + ami_id = data.aws_ami.selected.id + instance_type = "t3.xlarge" + subnet_id = module.network.worker_subnet_id + security_group_id = module.network.worker_security_group_id + availability_zone = local.availability_zones[1] + kms_key_arn = aws_kms_key.ebs.arn + repository_arns = module.registry.repository_arns + secret_arns = var.worker_secret_arns +} + +resource "aws_route53_record" "worker" { + zone_id = module.network.private_zone_id + name = var.worker_private_dns_name + type = "A" + ttl = 30 + records = [module.worker.private_ip] +} diff --git a/infra/aws/environments/personal/operations.tf b/infra/aws/environments/personal/operations.tf new file mode 100644 index 000000000..3d596f778 --- /dev/null +++ b/infra/aws/environments/personal/operations.tf @@ -0,0 +1,251 @@ +resource "aws_sns_topic" "alerts" { + name = "${var.name}-alerts" + kms_master_key_id = "alias/aws/sns" +} + +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +data "aws_iam_policy_document" "github_deploy_assume" { + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [var.github_oidc_provider_arn] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${var.github_repository}:environment:production"] + } + } +} + +resource "aws_iam_role" "github_deploy" { + name = "${var.name}-github-deploy" + assume_role_policy = data.aws_iam_policy_document.github_deploy_assume.json +} + +resource "aws_iam_role_policy" "github_deploy" { + name = "${var.name}-github-deploy" + role = aws_iam_role.github_deploy.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = concat([ + { + Sid = "EcrLogin" + Effect = "Allow" + Action = ["ecr:GetAuthorizationToken"] + Resource = "*" + }, + { + Sid = "PublishImages" + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:CompleteLayerUpload", + "ecr:DescribeImages", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:ListImages", + "ecr:PutImage", + "ecr:UploadLayerPart" + ] + Resource = module.registry.repository_arns + }, + { + Sid = "RunDeploymentCommands" + Effect = "Allow" + Action = ["ssm:SendCommand"] + Resource = [ + "arn:${data.aws_partition.current.partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/${module.control.instance_id}", + "arn:${data.aws_partition.current.partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/${module.worker.instance_id}", + "arn:${data.aws_partition.current.partition}:ssm:${var.aws_region}::document/AWS-RunShellScript" + ] + }, + { + Sid = "ReadDeploymentCommands" + Effect = "Allow" + Action = ["ssm:GetCommandInvocation"] + Resource = "*" + } + ], length(concat(var.control_secret_arns, var.worker_secret_arns)) == 0 ? [] : [ + { + Sid = "UpdateImageReferences" + Effect = "Allow" + Action = [ + "secretsmanager:GetSecretValue", + "secretsmanager:PutSecretValue" + ] + Resource = concat(var.control_secret_arns, var.worker_secret_arns) + } + ]) + }) +} + +resource "aws_sns_topic_subscription" "email" { + count = var.alert_email == null ? 0 : 1 + + topic_arn = aws_sns_topic.alerts.arn + protocol = "email" + endpoint = var.alert_email +} + +locals { + alarm_actions = [aws_sns_topic.alerts.arn] +} + +resource "aws_cloudwatch_metric_alarm" "instance_status" { + for_each = { + control = module.control.instance_id + worker = module.worker.instance_id + } + + alarm_name = "${var.name}-${each.key}-instance-status" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 2 + metric_name = "StatusCheckFailed" + namespace = "AWS/EC2" + period = 60 + statistic = "Maximum" + threshold = 1 + treat_missing_data = "breaching" + dimensions = { InstanceId = each.value } + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +resource "aws_cloudwatch_metric_alarm" "service_unready" { + for_each = toset(["control", "worker"]) + + alarm_name = "${var.name}-${each.key}-service-unready" + comparison_operator = "LessThanThreshold" + evaluation_periods = 3 + metric_name = "ServiceReady" + namespace = "OpenBot/Personal" + period = 60 + statistic = "Minimum" + threshold = 1 + treat_missing_data = "breaching" + dimensions = { HostRole = each.key } + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +resource "aws_cloudwatch_metric_alarm" "disk_pressure" { + for_each = toset(["control", "worker"]) + + alarm_name = "${var.name}-${each.key}-disk-pressure" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 3 + metric_name = "DiskUsedPercent" + namespace = "OpenBot/Personal" + period = 60 + statistic = "Maximum" + threshold = 85 + treat_missing_data = "breaching" + dimensions = { HostRole = each.key } + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +resource "aws_cloudwatch_metric_alarm" "queue_stuck" { + alarm_name = "${var.name}-worker-queue-stuck" + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 30 + metric_name = "QueueDepth" + namespace = "OpenBot/Personal" + period = 60 + statistic = "Minimum" + threshold = 0 + treat_missing_data = "breaching" + dimensions = { HostRole = "worker" } + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +resource "aws_cloudwatch_log_metric_filter" "provider_failures" { + name = "${var.name}-provider-terminal-failures" + pattern = "{ $.event = \"provider_run_terminal\" && $.outcome = \"failed\" }" + log_group_name = module.worker.log_group_name + + metric_transformation { + name = "ProviderTerminalFailures" + namespace = "OpenBot/Personal" + value = "1" + default_value = "0" + } +} + +resource "aws_cloudwatch_metric_alarm" "provider_failures" { + alarm_name = "${var.name}-provider-terminal-failures" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 1 + metric_name = aws_cloudwatch_log_metric_filter.provider_failures.metric_transformation[0].name + namespace = aws_cloudwatch_log_metric_filter.provider_failures.metric_transformation[0].namespace + period = 300 + statistic = "Sum" + threshold = 3 + treat_missing_data = "notBreaching" + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +data "aws_iam_policy_document" "backup_assume" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["backup.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "backup" { + name = "${var.name}-backup" + assume_role_policy = data.aws_iam_policy_document.backup_assume.json +} + +resource "aws_iam_role_policy_attachment" "backup" { + role = aws_iam_role.backup.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForBackup" +} + +resource "aws_backup_vault" "service_data" { + name = "${var.name}-service-data" + kms_key_arn = aws_kms_key.ebs.arn +} + +resource "aws_backup_plan" "daily" { + name = "${var.name}-daily" + + rule { + rule_name = "daily-retained-service-data" + target_vault_name = aws_backup_vault.service_data.name + schedule = "cron(0 5 * * ? *)" + start_window = 60 + completion_window = 180 + + lifecycle { + delete_after = 30 + } + } +} + +resource "aws_backup_selection" "service_data" { + name = "${var.name}-service-data-only" + iam_role_arn = aws_iam_role.backup.arn + plan_id = aws_backup_plan.daily.id + resources = ["*"] + + selection_tag { + type = "STRINGEQUALS" + key = "Backup" + value = "included-service-data" + } +} diff --git a/infra/aws/environments/personal/outputs.tf b/infra/aws/environments/personal/outputs.tf new file mode 100644 index 000000000..dd5803d3e --- /dev/null +++ b/infra/aws/environments/personal/outputs.tf @@ -0,0 +1,46 @@ +output "control_instance_id" { + value = module.control.instance_id +} + +output "control_public_ip" { + value = module.control.public_ip +} + +output "worker_instance_id" { + value = module.worker.instance_id +} + +output "worker_private_dns_name" { + value = aws_route53_record.worker.fqdn +} + +output "ecr_repository_urls" { + value = module.registry.repository_urls +} + +output "retained_volume_ids" { + value = { + control_data = module.control.data_volume_id + worker_data = module.worker.data_volume_id + provider_auth = module.worker.auth_volume_ids + } +} + +output "log_group_names" { + value = { + control = module.control.log_group_name + worker = module.worker.log_group_name + } +} + +output "alert_topic_arn" { + value = aws_sns_topic.alerts.arn +} + +output "backup_vault_name" { + value = aws_backup_vault.service_data.name +} + +output "github_actions_deploy_role_arn" { + value = aws_iam_role.github_deploy.arn +} diff --git a/infra/aws/environments/personal/terraform.tfvars.example b/infra/aws/environments/personal/terraform.tfvars.example new file mode 100644 index 000000000..3de3bb3a0 --- /dev/null +++ b/infra/aws/environments/personal/terraform.tfvars.example @@ -0,0 +1,13 @@ +aws_region = "us-east-1" +ami_id = "ami-replace-with-one-fixed-al2023-x86-64-id" +alert_email = "owner@example.com" +github_repository = "Tbsheff/OpenBot" +github_oidc_provider_arn = "arn:aws:iam::123456789012:oidc-provider/token.actions.githubusercontent.com" + +# Store JSON objects in Secrets Manager. Terraform receives only their ARNs. +control_secret_arns = [ + "arn:aws:secretsmanager:us-east-1:123456789012:secret:openbot/control-replace", +] +worker_secret_arns = [ + "arn:aws:secretsmanager:us-east-1:123456789012:secret:openbot/worker-replace", +] diff --git a/infra/aws/environments/personal/variables.tf b/infra/aws/environments/personal/variables.tf new file mode 100644 index 000000000..5a8a106ce --- /dev/null +++ b/infra/aws/environments/personal/variables.tf @@ -0,0 +1,83 @@ +variable "aws_region" { + description = "AWS region for all OpenBot resources." + type = string + default = "us-east-1" +} + +variable "name" { + description = "Short resource-name prefix." + type = string + default = "openbot-personal" +} + +variable "vpc_cidr" { + type = string + default = "10.42.0.0/20" +} + +variable "ami_id" { + description = "Pinned Amazon Linux 2023 x86_64 AMI. Resolve /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 once, then store that AMI ID here." + type = string + + validation { + condition = can(regex("^ami-[0-9a-f]+$", var.ami_id)) + error_message = "ami_id must be one fixed EC2 AMI ID." + } +} + +variable "availability_zones" { + description = "Optional fixed pair of availability zones. The first two available zones are used when empty." + type = list(string) + default = [] + + validation { + condition = length(var.availability_zones) == 0 || length(var.availability_zones) >= 2 + error_message = "Set no availability zones or at least two." + } +} + +variable "control_secret_arns" { + description = "Secrets Manager ARNs that may populate /etc/openbot/control.env at deploy time. Values never enter Terraform." + type = list(string) + default = [] +} + +variable "worker_secret_arns" { + description = "Secrets Manager ARNs that may populate /etc/openbot/worker.env at deploy time. Values never enter Terraform." + type = list(string) + default = [] +} + +variable "worker_private_dns_name" { + description = "Stable private name used by the OpenBot coworker endpoints." + type = string + default = "worker.openbot.internal" +} + +variable "alert_email" { + description = "Optional email address for health and capacity alarms. The subscription must be confirmed before alerts arrive." + type = string + default = null + nullable = true +} + +variable "github_repository" { + description = "GitHub owner and repository allowed to deploy through the production environment." + type = string + default = "Tbsheff/OpenBot" + + validation { + condition = can(regex("^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", var.github_repository)) + error_message = "github_repository must have owner/repository form." + } +} + +variable "github_oidc_provider_arn" { + description = "ARN of the account-wide token.actions.githubusercontent.com OIDC provider. Import or create that provider once before this stack." + type = string + + validation { + condition = can(regex(":oidc-provider/token\\.actions\\.githubusercontent\\.com$", var.github_oidc_provider_arn)) + error_message = "github_oidc_provider_arn must name the GitHub Actions OIDC provider." + } +} diff --git a/infra/aws/environments/personal/versions.tf b/infra/aws/environments/personal/versions.tf new file mode 100644 index 000000000..931f89e5e --- /dev/null +++ b/infra/aws/environments/personal/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "= 1.14.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "= 6.14.1" + } + } +} diff --git a/infra/aws/modules/control-host/bootstrap.sh.tftpl b/infra/aws/modules/control-host/bootstrap.sh.tftpl new file mode 100644 index 000000000..9b3ae340a --- /dev/null +++ b/infra/aws/modules/control-host/bootstrap.sh.tftpl @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +dnf install -y docker jq +systemctl enable --now docker + +volume_id="${data_volume_id}" +device="/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_$${volume_id//-/}" +for _ in $$(seq 1 120); do + [[ -e "$${device}" ]] && break + sleep 2 +done +[[ -e "$${device}" ]] + +if ! blkid "$${device}" >/dev/null 2>&1; then + mkfs.xfs "$${device}" +fi +mkdir -p /srv/openbot-control +uuid="$$(blkid -s UUID -o value "$${device}")" +grep -q "UUID=$${uuid}" /etc/fstab || echo "UUID=$${uuid} /srv/openbot-control xfs defaults,nofail 0 2" >> /etc/fstab +mount /srv/openbot-control +mkdir -p /srv/openbot-control/postgres /srv/openbot-control/backups /srv/openbot-control/caddy/data /srv/openbot-control/caddy/config +chown -R 999:999 /srv/openbot-control/postgres /srv/openbot-control/backups +chmod 0700 /srv/openbot-control/postgres /srv/openbot-control/backups +mkdir -p /etc/openbot /opt/openbot +chmod 0700 /etc/openbot diff --git a/infra/aws/modules/control-host/main.tf b/infra/aws/modules/control-host/main.tf new file mode 100644 index 000000000..228a1fc77 --- /dev/null +++ b/infra/aws/modules/control-host/main.tf @@ -0,0 +1,145 @@ +data "aws_iam_policy_document" "assume_role" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_cloudwatch_log_group" "this" { + name = "/openbot/${var.name}/control" + retention_in_days = 30 + kms_key_id = var.kms_key_arn +} + +resource "aws_iam_role" "this" { + name = "${var.name}-control" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "ssm" { + role = aws_iam_role.this.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_role_policy" "runtime" { + name = "${var.name}-control-runtime" + role = aws_iam_role.this.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = concat([ + { + Sid = "EcrLogin" + Effect = "Allow" + Action = ["ecr:GetAuthorizationToken"] + Resource = "*" + }, + { + Sid = "EcrPull" + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + Resource = var.repository_arns + }, + { + Sid = "Logs" + Effect = "Allow" + Action = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ] + Resource = "${aws_cloudwatch_log_group.this.arn}:*" + }, + { + Sid = "HealthMetrics" + Effect = "Allow" + Action = ["cloudwatch:PutMetricData"] + Resource = "*" + Condition = { + StringEquals = { "cloudwatch:namespace" = "OpenBot/Personal" } + } + } + ], length(var.secret_arns) == 0 ? [] : [ + { + Sid = "DeploymentSecrets" + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = var.secret_arns + } + ]) + }) +} + +resource "aws_iam_instance_profile" "this" { + name = "${var.name}-control" + role = aws_iam_role.this.name +} + +resource "aws_ebs_volume" "data" { + availability_zone = var.availability_zone + size = var.data_volume_size_gib + type = "gp3" + encrypted = true + kms_key_id = var.kms_key_arn + + tags = { + Name = "${var.name}-control-data" + Backup = "included-service-data" + } + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_instance" "this" { + ami = var.ami_id + instance_type = var.instance_type + subnet_id = var.subnet_id + vpc_security_group_ids = [var.security_group_id] + iam_instance_profile = aws_iam_instance_profile.this.name + associate_public_ip_address = true + user_data_replace_on_change = true + user_data = templatefile("${path.module}/bootstrap.sh.tftpl", { + data_volume_id = aws_ebs_volume.data.id + }) + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "disabled" + } + + root_block_device { + encrypted = true + kms_key_id = var.kms_key_arn + volume_type = "gp3" + volume_size = 30 + delete_on_termination = true + } + + tags = { Name = "${var.name}-control" } +} + +resource "aws_volume_attachment" "data" { + device_name = "/dev/sdf" + instance_id = aws_instance.this.id + volume_id = aws_ebs_volume.data.id + stop_instance_before_detaching = true + skip_destroy = true + # A separately attached EBS volume has delete_on_termination = false. +} + +resource "aws_eip" "control" { + domain = "vpc" + instance = aws_instance.this.id + + tags = { Name = "${var.name}-control" } +} diff --git a/infra/aws/modules/control-host/outputs.tf b/infra/aws/modules/control-host/outputs.tf new file mode 100644 index 000000000..f6fc9ad4b --- /dev/null +++ b/infra/aws/modules/control-host/outputs.tf @@ -0,0 +1,19 @@ +output "instance_id" { + value = aws_instance.this.id +} + +output "public_ip" { + value = aws_eip.control.public_ip +} + +output "private_ip" { + value = aws_instance.this.private_ip +} + +output "data_volume_id" { + value = aws_ebs_volume.data.id +} + +output "log_group_name" { + value = aws_cloudwatch_log_group.this.name +} diff --git a/infra/aws/modules/control-host/variables.tf b/infra/aws/modules/control-host/variables.tf new file mode 100644 index 000000000..e4c5379a3 --- /dev/null +++ b/infra/aws/modules/control-host/variables.tf @@ -0,0 +1,45 @@ +variable "name" { + type = string + description = "Prefix for control-host resources." +} + +variable "ami_id" { + type = string + description = "Pinned or SSM-resolved AMD64 Amazon Linux 2023 AMI ID." +} + +variable "subnet_id" { + type = string +} + +variable "security_group_id" { + type = string +} + +variable "availability_zone" { + type = string +} + +variable "kms_key_arn" { + type = string +} + +variable "repository_arns" { + type = list(string) +} + +variable "secret_arns" { + description = "Secrets Manager references that the host can fetch into a root-only env file." + type = list(string) + default = [] +} + +variable "instance_type" { + type = string + default = "t3.medium" +} + +variable "data_volume_size_gib" { + type = number + default = 30 +} diff --git a/infra/aws/modules/network/main.tf b/infra/aws/modules/network/main.tf new file mode 100644 index 000000000..c27b7b647 --- /dev/null +++ b/infra/aws/modules/network/main.tf @@ -0,0 +1,106 @@ +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_hostnames = true + enable_dns_support = true + + tags = { Name = var.name } +} + +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + tags = { Name = "${var.name}-internet" } +} + +resource "aws_subnet" "public" { + count = 2 + + vpc_id = aws_vpc.this.id + availability_zone = var.availability_zones[count.index] + cidr_block = cidrsubnet(var.vpc_cidr, 4, count.index) + map_public_ip_on_launch = true + + tags = { + Name = "${var.name}-public-${count.index + 1}" + Tier = "public" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.this.id + } + + tags = { Name = "${var.name}-public" } +} + +resource "aws_route_table_association" "public" { + count = 2 + + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +resource "aws_security_group" "control" { + name = "${var.name}-control" + description = "Public TLS ingress for the OpenBot control host" + vpc_id = aws_vpc.this.id + + tags = { Name = "${var.name}-control" } +} + +resource "aws_vpc_security_group_ingress_rule" "control_web" { + for_each = toset(["80", "443"]) + + security_group_id = aws_security_group.control.id + description = "Public web ingress" + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "tcp" + from_port = tonumber(each.value) + to_port = tonumber(each.value) +} + +resource "aws_vpc_security_group_egress_rule" "control" { + security_group_id = aws_security_group.control.id + description = "Control host outbound access" + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" +} + +resource "aws_security_group" "worker" { + name = "${var.name}-worker" + description = "Private AG-UI ingress from the control host only" + vpc_id = aws_vpc.this.id + + tags = { Name = "${var.name}-worker" } +} + +resource "aws_vpc_security_group_ingress_rule" "worker_gateway" { + for_each = toset(["4210", "4211", "4212"]) + + security_group_id = aws_security_group.worker.id + referenced_security_group_id = aws_security_group.control.id + description = "AG-UI ${each.value} from control" + ip_protocol = "tcp" + from_port = tonumber(each.value) + to_port = tonumber(each.value) +} + +resource "aws_vpc_security_group_egress_rule" "worker" { + security_group_id = aws_security_group.worker.id + description = "Worker outbound access for provider programs and ECR" + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" +} + +resource "aws_route53_zone" "private" { + name = var.private_zone_name + + vpc { + vpc_id = aws_vpc.this.id + } + + tags = { Name = "${var.name}-private" } +} diff --git a/infra/aws/modules/network/outputs.tf b/infra/aws/modules/network/outputs.tf new file mode 100644 index 000000000..a797bfb56 --- /dev/null +++ b/infra/aws/modules/network/outputs.tf @@ -0,0 +1,27 @@ +output "vpc_id" { + value = aws_vpc.this.id +} + +output "control_subnet_id" { + value = aws_subnet.public[0].id +} + +output "worker_subnet_id" { + value = aws_subnet.public[1].id +} + +output "control_security_group_id" { + value = aws_security_group.control.id +} + +output "worker_security_group_id" { + value = aws_security_group.worker.id +} + +output "private_zone_id" { + value = aws_route53_zone.private.zone_id +} + +output "private_zone_name" { + value = aws_route53_zone.private.name +} diff --git a/infra/aws/modules/network/variables.tf b/infra/aws/modules/network/variables.tf new file mode 100644 index 000000000..4eb86d7df --- /dev/null +++ b/infra/aws/modules/network/variables.tf @@ -0,0 +1,25 @@ +variable "name" { + description = "Prefix for network resource names." + type = string +} + +variable "vpc_cidr" { + description = "IPv4 range for the private deployment network." + type = string +} + +variable "availability_zones" { + description = "Two availability zones used by the public control and worker subnets." + type = list(string) + + validation { + condition = length(var.availability_zones) >= 2 + error_message = "At least two availability zones are required." + } +} + +variable "private_zone_name" { + description = "Private Route 53 zone used for worker addressing." + type = string + default = "openbot.internal" +} diff --git a/infra/aws/modules/registry/main.tf b/infra/aws/modules/registry/main.tf new file mode 100644 index 000000000..5e982176c --- /dev/null +++ b/infra/aws/modules/registry/main.tf @@ -0,0 +1,34 @@ +resource "aws_ecr_repository" "this" { + for_each = var.repository_names + + name = "${var.name}/${each.value}" + image_tag_mutability = "IMMUTABLE" + + encryption_configuration { + encryption_type = "AES256" + } + + image_scanning_configuration { + scan_on_push = true + } + + tags = { Name = "${var.name}-${each.value}" } +} + +resource "aws_ecr_lifecycle_policy" "this" { + for_each = aws_ecr_repository.this + + repository = each.value.name + policy = jsonencode({ + rules = [{ + rulePriority = 1 + description = "Keep the newest 20 release images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = 20 + } + action = { type = "expire" } + }] + }) +} diff --git a/infra/aws/modules/registry/outputs.tf b/infra/aws/modules/registry/outputs.tf new file mode 100644 index 000000000..a9b596f43 --- /dev/null +++ b/infra/aws/modules/registry/outputs.tf @@ -0,0 +1,7 @@ +output "repository_arns" { + value = [for repository in aws_ecr_repository.this : repository.arn] +} + +output "repository_urls" { + value = { for name, repository in aws_ecr_repository.this : name => repository.repository_url } +} diff --git a/infra/aws/modules/registry/variables.tf b/infra/aws/modules/registry/variables.tf new file mode 100644 index 000000000..9831f7ca2 --- /dev/null +++ b/infra/aws/modules/registry/variables.tf @@ -0,0 +1,10 @@ +variable "name" { + description = "Prefix for registry resources." + type = string +} + +variable "repository_names" { + description = "Private image repositories used by the deployment." + type = set(string) + default = ["openbot-control", "subscription-gateway"] +} diff --git a/infra/aws/modules/worker-host/bootstrap.sh.tftpl b/infra/aws/modules/worker-host/bootstrap.sh.tftpl new file mode 100644 index 000000000..6b8e122a2 --- /dev/null +++ b/infra/aws/modules/worker-host/bootstrap.sh.tftpl @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +dnf install -y docker jq iptables +systemctl enable --now docker + +mount_volume() { + local volume_id="$1" + local mount_path="$2" + local mode="$3" + local device="/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_$${volume_id//-/}" + for _ in $$(seq 1 120); do + [[ -e "$${device}" ]] && break + sleep 2 + done + [[ -e "$${device}" ]] + if ! blkid "$${device}" >/dev/null 2>&1; then + mkfs.xfs "$${device}" + fi + mkdir -p "$${mount_path}" + local uuid + uuid="$$(blkid -s UUID -o value "$${device}")" + grep -q "UUID=$${uuid}" /etc/fstab || echo "UUID=$${uuid} $${mount_path} xfs defaults,nofail 0 2" >> /etc/fstab + mount "$${mount_path}" + chmod "$${mode}" "$${mount_path}" +} + +mount_volume "${data_volume_id}" /srv/openbot-worker 0700 +mount_volume "${codex_auth_id}" /srv/openbot-auth/codex 0700 +mount_volume "${claude_auth_id}" /srv/openbot-auth/claude 0700 +mount_volume "${grok_auth_id}" /srv/openbot-auth/grok 0700 + +mkdir -p /srv/openbot-worker/state /srv/openbot-worker/workspaces/{codex,claude,grok} +chown -R 10001:10001 /srv/openbot-worker /srv/openbot-auth +chmod -R 0700 /srv/openbot-worker +mkdir -p /etc/openbot /opt/openbot +chmod 0700 /etc/openbot diff --git a/infra/aws/modules/worker-host/main.tf b/infra/aws/modules/worker-host/main.tf new file mode 100644 index 000000000..721b5f3db --- /dev/null +++ b/infra/aws/modules/worker-host/main.tf @@ -0,0 +1,181 @@ +locals { + providers = toset(["codex", "claude", "grok"]) + auth_devices = { + codex = "/dev/sdg" + claude = "/dev/sdh" + grok = "/dev/sdi" + } +} + +data "aws_iam_policy_document" "assume_role" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_cloudwatch_log_group" "this" { + name = "/openbot/${var.name}/worker" + retention_in_days = 30 + kms_key_id = var.kms_key_arn +} + +resource "aws_iam_role" "this" { + name = "${var.name}-worker" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "ssm" { + role = aws_iam_role.this.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_role_policy" "runtime" { + name = "${var.name}-worker-runtime" + role = aws_iam_role.this.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = concat([ + { + Sid = "EcrLogin" + Effect = "Allow" + Action = ["ecr:GetAuthorizationToken"] + Resource = "*" + }, + { + Sid = "EcrPull" + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + Resource = var.repository_arns + }, + { + Sid = "Logs" + Effect = "Allow" + Action = [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ] + Resource = "${aws_cloudwatch_log_group.this.arn}:*" + }, + { + Sid = "HealthMetrics" + Effect = "Allow" + Action = ["cloudwatch:PutMetricData"] + Resource = "*" + Condition = { + StringEquals = { "cloudwatch:namespace" = "OpenBot/Personal" } + } + } + ], length(var.secret_arns) == 0 ? [] : [ + { + Sid = "DeploymentSecrets" + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = var.secret_arns + } + ]) + }) +} + +resource "aws_iam_instance_profile" "this" { + name = "${var.name}-worker" + role = aws_iam_role.this.name +} + +resource "aws_ebs_volume" "data" { + availability_zone = var.availability_zone + size = var.data_volume_size_gib + type = "gp3" + encrypted = true + kms_key_id = var.kms_key_arn + + tags = { + Name = "${var.name}-worker-data" + Backup = "included-service-data" + } + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_ebs_volume" "auth" { + for_each = local.providers + + availability_zone = var.availability_zone + size = var.auth_volume_size_gib + type = "gp3" + encrypted = true + kms_key_id = var.kms_key_arn + + tags = { + Name = "${var.name}-${each.key}-auth" + Backup = "excluded-provider-auth" + Provider = each.key + } + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_instance" "this" { + ami = var.ami_id + instance_type = var.instance_type + subnet_id = var.subnet_id + vpc_security_group_ids = [var.security_group_id] + iam_instance_profile = aws_iam_instance_profile.this.name + associate_public_ip_address = true + user_data_replace_on_change = true + user_data = templatefile("${path.module}/bootstrap.sh.tftpl", { + data_volume_id = aws_ebs_volume.data.id + codex_auth_id = aws_ebs_volume.auth["codex"].id + claude_auth_id = aws_ebs_volume.auth["claude"].id + grok_auth_id = aws_ebs_volume.auth["grok"].id + }) + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 1 + instance_metadata_tags = "disabled" + } + + root_block_device { + encrypted = true + kms_key_id = var.kms_key_arn + volume_type = "gp3" + volume_size = 30 + delete_on_termination = true + } + + tags = { Name = "${var.name}-worker" } +} + +resource "aws_volume_attachment" "data" { + device_name = "/dev/sdf" + instance_id = aws_instance.this.id + volume_id = aws_ebs_volume.data.id + stop_instance_before_detaching = true + skip_destroy = true + # A separately attached EBS volume has delete_on_termination = false. +} + +resource "aws_volume_attachment" "auth" { + for_each = local.providers + + device_name = local.auth_devices[each.key] + instance_id = aws_instance.this.id + volume_id = aws_ebs_volume.auth[each.key].id + stop_instance_before_detaching = true + skip_destroy = true + # A separately attached EBS volume has delete_on_termination = false. +} diff --git a/infra/aws/modules/worker-host/outputs.tf b/infra/aws/modules/worker-host/outputs.tf new file mode 100644 index 000000000..dee1be4ee --- /dev/null +++ b/infra/aws/modules/worker-host/outputs.tf @@ -0,0 +1,23 @@ +output "instance_id" { + value = aws_instance.this.id +} + +output "public_ip" { + value = aws_instance.this.public_ip +} + +output "private_ip" { + value = aws_instance.this.private_ip +} + +output "data_volume_id" { + value = aws_ebs_volume.data.id +} + +output "auth_volume_ids" { + value = { for provider, volume in aws_ebs_volume.auth : provider => volume.id } +} + +output "log_group_name" { + value = aws_cloudwatch_log_group.this.name +} diff --git a/infra/aws/modules/worker-host/variables.tf b/infra/aws/modules/worker-host/variables.tf new file mode 100644 index 000000000..a3094dcfa --- /dev/null +++ b/infra/aws/modules/worker-host/variables.tf @@ -0,0 +1,50 @@ +variable "name" { + type = string + description = "Prefix for worker-host resources." +} + +variable "ami_id" { + type = string + description = "Pinned or SSM-resolved AMD64 Amazon Linux 2023 AMI ID." +} + +variable "subnet_id" { + type = string +} + +variable "security_group_id" { + type = string +} + +variable "availability_zone" { + type = string +} + +variable "kms_key_arn" { + type = string +} + +variable "repository_arns" { + type = list(string) +} + +variable "secret_arns" { + description = "Secrets Manager references that the host can fetch into a root-only env file." + type = list(string) + default = [] +} + +variable "instance_type" { + type = string + default = "t3.xlarge" +} + +variable "data_volume_size_gib" { + type = number + default = 100 +} + +variable "auth_volume_size_gib" { + type = number + default = 10 +} diff --git a/infra/aws/versions.tf b/infra/aws/versions.tf new file mode 100644 index 000000000..931f89e5e --- /dev/null +++ b/infra/aws/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "= 1.14.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "= 6.14.1" + } + } +} diff --git a/package.json b/package.json index 838d85081..4ef9a3c79 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,9 @@ "build": "bun run generate:app-config && bun run --filter '*' build", "dev": "bun run generate:app-config && bun run --filter app --filter server --parallel dev", "generate:app-config": "bun --env-file=.env scripts/generate-app-config.ts", - "format": "bunx biome format --write .", - "format:check": "bunx biome format .", - "lint": "bunx biome lint --error-on-warnings .", + "format": "bun x biome format --write .", + "format:check": "bun x biome format .", + "lint": "bun x biome lint --error-on-warnings .", "test": "bun test", "typecheck": "bun run --filter '*' typecheck", "test:ci": "bun scripts/test-ci.ts", diff --git a/server/src/agents/runtime-agents.ts b/server/src/agents/runtime-agents.ts index 0e682b994..71574297b 100644 --- a/server/src/agents/runtime-agents.ts +++ b/server/src/agents/runtime-agents.ts @@ -63,15 +63,26 @@ export function createRuntimeAgentLoader( // Config parses URLs, while package rows retain their original spelling. Compare both // in canonical form so scheme/host case cannot silently drop the deployment token. const endpoint = managedEndpointIdentity(agent.endpoint); - const ours = - endpoint !== undefined && - [managedAgent.endpoint, managedAgent.alsoRun] - .filter((url): url is URL => url !== undefined) - .some((url) => endpoint === managedEndpointIdentity(url)); - if (ours) { + const deploymentEndpoint = endpoint + ? [ + ...(managedAgent.token + ? [managedAgent.endpoint, managedAgent.alsoRun] + .filter((url): url is URL => url !== undefined) + .map((url) => ({ + endpoint: url, + token: managedAgent.token as string, + })) + : []), + ...(managedAgent.subscriptionWorkers ?? []), + ].find( + (candidate) => + endpoint === managedEndpointIdentity(candidate.endpoint), + ) + : undefined; + if (deploymentEndpoint) { agent.headers = { ...agent.headers, - "x-openbot-agent-token": managedAgent.token, + "x-openbot-agent-token": deploymentEndpoint.token, }; } } diff --git a/server/src/auth/index.ts b/server/src/auth/index.ts index 09d97b97b..bad23c492 100644 --- a/server/src/auth/index.ts +++ b/server/src/auth/index.ts @@ -109,6 +109,31 @@ export function mapEntraProfile(profile: Record) { return { email }; } +function isConfiguredOwner(ownerEmail: string, candidateEmail: string) { + return candidateEmail.trim().toLowerCase() === ownerEmail; +} + +async function refuseNonOwner( + ownerEmail: string, + candidateEmail: string, + auditStore: AuditStore | undefined, + userId?: string, +): Promise { + if (isConfiguredOwner(ownerEmail, candidateEmail)) return; + await record(auditStore, { + eventType: "session.refused", + targetType: "person", + ...(userId ? { targetId: userId, actorUserId: userId } : {}), + payload: { + email: candidateEmail, + reason: "identity is not the configured owner", + }, + }); + throw new APIError("FORBIDDEN", { + message: "Only the configured owner can access this deployment.", + }); +} + export function createAuth( config: DeploymentConfig, database: Database, @@ -244,6 +269,7 @@ export function createAuth( * list is keyed on the address rather than the id. */ before: async (user) => { + await refuseNonOwner(authConfig.ownerEmail, user.email, auditStore); if (await isRevoked?.(user.email)) { // The row a removed person coming back produces. Nothing else records the attempt: // no user row is written and no session exists to look at afterwards. @@ -288,6 +314,14 @@ export function createAuth( .from(users) .where(eq(users.id, session.userId)) .limit(1); + if (user) { + await refuseNonOwner( + authConfig.ownerEmail, + user.email, + auditStore, + session.userId, + ); + } if (user && (await isRevoked?.(user.email))) { await record(auditStore, { eventType: "session.refused", diff --git a/server/src/config.ts b/server/src/config.ts index 9ae774ce8..0b98e5f04 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -83,6 +83,7 @@ export type AuthConfig = { baseUrl: string; secret: string; trustedOrigins: string[]; + ownerEmail: string; initialAdminEmails: string[]; google?: OAuthClient; /** @@ -117,7 +118,7 @@ export type ManagedAgentConfig = { /** The bundled Bot, absent when this deployment's provider cannot run it. */ endpoint?: URL; /** Secret sent only to endpoints this deployment runs. Never stored in an agent row. */ - token: string; + token?: string; /** * The harness picked during setup, when there is one. * @@ -125,6 +126,15 @@ export type ManagedAgentConfig = { * it chose, holding this token. It gets the same header for the same reason. */ alsoRun?: URL; + subscriptionWorkers?: SubscriptionWorkerConfig[]; +}; + +export type SubscriptionWorkerProvider = "codex" | "claude" | "grok"; + +export type SubscriptionWorkerConfig = { + provider: SubscriptionWorkerProvider; + endpoint: URL; + token: string; }; /** @@ -436,6 +446,7 @@ function optionalHttpUrl( */ function managedAgentConfig( environment: Environment, + allowedHosts: ReadonlySet, ): ManagedAgentConfig | undefined { const endpoint = optionalHttpUrl(environment, "MANAGED_AGENT_AG_UI_URL"); // BYO writes a URL too, but does not run our image or hold our deployment token. @@ -453,7 +464,14 @@ function managedAgentConfig( "MANAGED_AGENT_TOKEN must be set when an installed PICKED_HARNESS_URL is set", ); } - if ((!endpoint && !alsoRun) || !token) { + const subscriptionWorkers = subscriptionWorkerConfig( + environment, + allowedHosts, + ); + if ( + (!endpoint && !alsoRun && subscriptionWorkers.length === 0) || + (!token && subscriptionWorkers.length === 0) + ) { return undefined; } /* @@ -467,11 +485,38 @@ function managedAgentConfig( */ return { ...(endpoint ? { endpoint } : {}), - token, + ...(token ? { token } : {}), ...(alsoRun ? { alsoRun } : {}), + subscriptionWorkers, }; } +function subscriptionWorkerConfig( + environment: Environment, + allowedHosts: ReadonlySet, +): SubscriptionWorkerConfig[] { + const providers = ["codex", "claude", "grok"] as const; + return providers.flatMap((provider) => { + const prefix = provider.toUpperCase(); + const endpoint = optionalHttpUrl(environment, `${prefix}_AGENT_AG_UI_URL`); + const token = optional(environment, `${prefix}_AGENT_TOKEN`); + if (Boolean(endpoint) !== Boolean(token)) { + throw new Error( + `${prefix}_AGENT_AG_UI_URL and ${prefix}_AGENT_TOKEN must be set together`, + ); + } + if (!endpoint || !token) return []; + + const allowedHost = endpoint.host.toLowerCase(); + if (!allowedHosts.has(allowedHost)) { + throw new Error( + `AGENT_ENDPOINT_ALLOWED_HOSTS must include ${allowedHost} for ${prefix}_AGENT_AG_UI_URL`, + ); + } + return [{ provider, endpoint, token }]; + }); +} + function oauthClient( environment: Environment, provider: "GOOGLE" | "MICROSOFT" | "OKTA", @@ -554,6 +599,18 @@ function authConfig( ); } + const ownerEmail = optional(environment, "OPENBOT_OWNER_EMAIL") + ?.trim() + .toLowerCase(); + if (!ownerEmail) { + throw new Error( + "Sign-in requires OPENBOT_OWNER_EMAIL naming the one identity this private deployment admits", + ); + } + if (!/^[^\s@]+@[^\s@]+$/.test(ownerEmail)) { + throw new Error("OPENBOT_OWNER_EMAIL must be a valid email address"); + } + return { baseUrl, secret, @@ -565,6 +622,7 @@ function authConfig( * `127.0.0.1:3010`, which is the address the rest of this deployment hands out. */ ["http://127.0.0.1:3010", "http://[::1]:3010", "http://localhost:3010"], + ownerEmail, initialAdminEmails, ...(google ? { google } : {}), ...(microsoft ? { microsoft } : {}), @@ -994,7 +1052,8 @@ export function loadConfig( ): DeploymentConfig { const google = oauthClient(environment, "GOOGLE"); const auth = authConfig(environment, google); - const managedAgent = managedAgentConfig(environment); + const allowedAgentHosts = agentEndpointAllowedHosts(environment); + const managedAgent = managedAgentConfig(environment, allowedAgentHosts); const workerSharedSecret = optional(environment, "WORKER_SHARED_SECRET"); return { @@ -1002,7 +1061,7 @@ export function loadConfig( databaseUrl: required(environment, "DATABASE_URL"), keyEncryptionKey: keyEncryptionKey(environment), ...(managedAgent ? { managedAgent } : {}), - agentEndpointAllowedHosts: agentEndpointAllowedHosts(environment), + agentEndpointAllowedHosts: allowedAgentHosts, deploymentId: optional(environment, "DEPLOYMENT_ID"), publicUrl: ( optional(environment, "OPENBOT_PUBLIC_URL") ?? auth?.baseUrl diff --git a/server/tests/agent-endpoint.test.ts b/server/tests/agent-endpoint.test.ts index a2ec786d3..433439c94 100644 --- a/server/tests/agent-endpoint.test.ts +++ b/server/tests/agent-endpoint.test.ts @@ -80,6 +80,28 @@ describe("what may be registered as an agent", () => { }); }); +test("only the three exact private worker ports are allowed", () => { + const host = "10.0.0.5"; + const allowedHosts = new Set( + [4210, 4211, 4212].map((port) => `${host}:${port}`), + ); + + for (const port of [4210, 4211, 4212]) { + expect( + checkAgentEndpoint(`http://${host}:${port}/ag-ui`, { allowedHosts }) + .allowed, + ).toBe(true); + } + expect( + checkAgentEndpoint(`http://${host}:4213/ag-ui`, { allowedHosts }).allowed, + ).toBe(false); + expect( + checkAgentEndpoint("http://10.0.0.6:4210/ag-ui", { + allowedHosts, + }).allowed, + ).toBe(false); +}); + describe("the agent form", () => { const base = { name: "Sales Bot", diff --git a/server/tests/auth-owner-allowlist.test.ts b/server/tests/auth-owner-allowlist.test.ts new file mode 100644 index 000000000..b2951dd39 --- /dev/null +++ b/server/tests/auth-owner-allowlist.test.ts @@ -0,0 +1,78 @@ +import { expect, mock, test } from "bun:test"; +import { createAuth } from "../src/auth"; + +type BeforeUser = (user: { id: string; email: string }) => Promise; +type BeforeSession = (session: { + id: string; + userId: string; + createdAt: Date; +}) => Promise; + +function databaseReturning(email: string) { + const query = { + from: () => query, + where: () => query, + limit: async () => [{ email }], + }; + return { + select: () => query, + } as never; +} + +function admissionHooks( + email: string, + isRevoked: (email: string) => Promise = async () => false, +) { + const auth = createAuth( + { + auth: { + baseUrl: "http://localhost:3001", + secret: "a-long-enough-local-development-auth-secret", + trustedOrigins: ["http://localhost:3010"], + ownerEmail: "owner@openbot.test", + initialAdminEmails: ["admin@openbot.test"], + google: { clientId: "client", clientSecret: "secret" }, + }, + keyEncryptionKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + } as never, + databaseReturning(email), + isRevoked, + ); + const hooks = auth.options.databaseHooks; + if (!hooks?.user?.create?.before || !hooks.session?.create?.before) { + throw new Error("owner admission hooks are not installed"); + } + return { + beforeUser: hooks.user.create.before as BeforeUser, + beforeSession: hooks.session.create.before as BeforeSession, + }; +} + +test("the normalized owner is admitted before first user creation", async () => { + const revoked = mock(async () => false); + const { beforeUser } = admissionHooks("owner@openbot.test", revoked); + const user = { id: "owner", email: " OWNER@OpenBot.Test " }; + + await expect(beforeUser(user)).resolves.toEqual({ data: user }); + expect(revoked).toHaveBeenCalledWith(" OWNER@OpenBot.Test "); +}); + +test("another valid identity is denied before revocation or user creation work", async () => { + const revoked = mock(async () => false); + const { beforeUser } = admissionHooks("other@openbot.test", revoked); + + await expect( + beforeUser({ id: "other", email: "other@openbot.test" }), + ).rejects.toThrow("configured owner"); + expect(revoked).not.toHaveBeenCalled(); +}); + +test("an existing non-owner is denied before a new session is created", async () => { + const revoked = mock(async () => false); + const { beforeSession } = admissionHooks("other@openbot.test", revoked); + + await expect( + beforeSession({ id: "session", userId: "other", createdAt: new Date() }), + ).rejects.toThrow("configured owner"); + expect(revoked).not.toHaveBeenCalled(); +}); diff --git a/server/tests/config.test.ts b/server/tests/config.test.ts index 125d7361e..790a82653 100644 --- a/server/tests/config.test.ts +++ b/server/tests/config.test.ts @@ -12,6 +12,7 @@ const baseEnvironment = { GOOGLE_OAUTH_CLIENT_SECRET: "google-client-secret", BETTER_AUTH_SECRET: "a-long-enough-local-development-auth-secret", BETTER_AUTH_URL: "http://localhost:3001", + OPENBOT_OWNER_EMAIL: "Owner@OpenBot.Test ", INITIAL_ADMIN_EMAILS: "admin@openbot.test", INTELLIGENCE_API_URL: "http://localhost:7100", INTELLIGENCE_GATEWAY_WS_URL: "ws://localhost:7103", @@ -67,6 +68,7 @@ describe("deployment configuration", () => { expect(config.managedAgent).toEqual({ endpoint: new URL("http://localhost:4200/ag-ui"), token: "managed-agent-token", + subscriptionWorkers: [], }); expect(config.tenantPackageDirectory).toBe("../examples/fintech"); }); @@ -199,6 +201,85 @@ describe("deployment configuration", () => { ).toThrow("MANAGED_AGENT_AG_UI_URL"); }); + test("configures each subscription worker independently in release order", () => { + const workerHost = "subscription-workers.openbot.internal"; + const config = loadConfig({ + ...baseEnvironment, + AGENT_ENDPOINT_ALLOWED_HOSTS: [4210, 4211, 4212] + .map((port) => `${workerHost}:${port}`) + .join(","), + CODEX_AGENT_AG_UI_URL: `http://${workerHost}:4210/ag-ui`, + CODEX_AGENT_TOKEN: "codex-gateway-token", + CLAUDE_AGENT_AG_UI_URL: `http://${workerHost}:4211/ag-ui`, + CLAUDE_AGENT_TOKEN: "claude-gateway-token", + GROK_AGENT_AG_UI_URL: `http://${workerHost}:4212/ag-ui`, + GROK_AGENT_TOKEN: "grok-gateway-token", + }); + + expect(config.managedAgent?.subscriptionWorkers).toEqual([ + { + provider: "codex", + endpoint: new URL(`http://${workerHost}:4210/ag-ui`), + token: "codex-gateway-token", + }, + { + provider: "claude", + endpoint: new URL(`http://${workerHost}:4211/ag-ui`), + token: "claude-gateway-token", + }, + { + provider: "grok", + endpoint: new URL(`http://${workerHost}:4212/ag-ui`), + token: "grok-gateway-token", + }, + ]); + }); + + test.each(["CODEX", "CLAUDE", "GROK"])( + "refuses a half-configured %s subscription worker", + (provider) => { + expect(() => + loadConfig({ + ...baseEnvironment, + [`${provider}_AGENT_AG_UI_URL`]: + "http://subscription-workers.openbot.internal:4210/ag-ui", + }), + ).toThrow(`${provider}_AGENT_TOKEN`); + }, + ); + + test("one configured provider does not require later providers", () => { + const config = loadConfig({ + ...baseEnvironment, + AGENT_ENDPOINT_ALLOWED_HOSTS: + "subscription-workers.openbot.internal:4210", + CODEX_AGENT_AG_UI_URL: + "http://subscription-workers.openbot.internal:4210/ag-ui", + CODEX_AGENT_TOKEN: "codex-gateway-token", + }); + + expect( + config.managedAgent?.subscriptionWorkers?.map( + (worker) => worker.provider, + ), + ).toEqual(["codex"]); + }); + + test("requires the exact private worker host and port in the endpoint allow-list", () => { + expect(() => + loadConfig({ + ...baseEnvironment, + AGENT_ENDPOINT_ALLOWED_HOSTS: + "subscription-workers.openbot.internal:4211", + CODEX_AGENT_AG_UI_URL: + "http://subscription-workers.openbot.internal:4210/ag-ui", + CODEX_AGENT_TOKEN: "codex-gateway-token", + }), + ).toThrow( + "AGENT_ENDPOINT_ALLOWED_HOSTS must include subscription-workers.openbot.internal:4210", + ); + }); + test("requires a base64-encoded 32-byte key-encryption key", () => { expect(() => loadConfig({ @@ -291,6 +372,7 @@ describe("deployment configuration", () => { "http://[::1]:3010", "http://localhost:3010", ], + ownerEmail: "owner@openbot.test", initialAdminEmails: ["admin@openbot.test", "owner@openbot.test"], }); }); @@ -415,6 +497,23 @@ describe("deployment configuration", () => { expect(() => loadConfig(withoutAdmins)).toThrow("INITIAL_ADMIN_EMAILS"); }); + test("refuses sign-in without the one owner identity admission setting", () => { + const { OPENBOT_OWNER_EMAIL: _none, ...withoutOwner } = baseEnvironment; + + expect(() => loadConfig(withoutOwner)).toThrow("OPENBOT_OWNER_EMAIL"); + }); + + test("normalizes the owner email independently of administrator assignment", () => { + const config = loadConfig({ + ...baseEnvironment, + OPENBOT_OWNER_EMAIL: " Owner@OpenBot.Test ", + INITIAL_ADMIN_EMAILS: "admin@openbot.test", + }); + + expect(config.auth?.ownerEmail).toBe("owner@openbot.test"); + expect(config.auth?.initialAdminEmails).toEqual(["admin@openbot.test"]); + }); + test("asks for no administrator when nothing signs anybody in", () => { // One administrator either way, and no list to write. Requiring one here as well would mean a // deployment had to name an administrator for a mode that has exactly one. diff --git a/server/tests/picked-harness-auth.test.ts b/server/tests/picked-harness-auth.test.ts index 25ca80e58..b54b89035 100644 --- a/server/tests/picked-harness-auth.test.ts +++ b/server/tests/picked-harness-auth.test.ts @@ -20,7 +20,8 @@ const encryptionKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; async function runPicked(options: { bundled: boolean; installed: boolean; - target?: "picked" | "bundled" | "customer"; + target?: "picked" | "bundled" | "customer" | "codex" | "claude" | "grok"; + subscription?: "codex" | "claude" | "grok"; customerAuth?: boolean; spelling?: "uppercase"; configuredQuery?: string; @@ -29,6 +30,9 @@ async function runPicked(options: { invalidCompanion?: boolean; packageProducer?: boolean; }) { + const expectedToken = options.subscription + ? `${options.subscription}-deployment-token` + : fixtureToken; const requests: { path: string; search: string; @@ -44,9 +48,11 @@ async function runPicked(options: { const managed = options.expectedManaged ?? (path.replace(/\/+$/, "") === "/bundled/ag-ui" || - (path.replace(/\/+$/, "") === "/picked/ag-ui" && options.installed)); + (path.replace(/\/+$/, "") === "/picked/ag-ui" && options.installed) || + (options.subscription !== undefined && + path.replace(/\/+$/, "") === `/${options.subscription}/ag-ui`)); const authorized = managed - ? request.headers.get("x-openbot-agent-token") === fixtureToken + ? request.headers.get("x-openbot-agent-token") === expectedToken : options.customerAuth ? request.headers.get("Authorization") === "Bearer synthetic-customer-key" && @@ -95,6 +101,12 @@ async function runPicked(options: { MANAGED_AGENT_AG_UI_URL: options.bundled ? endpoint("bundled") : "", PICKED_HARNESS_URL: endpoint("picked"), PICKED_HARNESS_KIND: "remote-ag-ui", + CODEX_AGENT_AG_UI_URL: + options.subscription === "codex" ? endpoint("codex") : "", + CLAUDE_AGENT_AG_UI_URL: + options.subscription === "claude" ? endpoint("claude") : "", + GROK_AGENT_AG_UI_URL: + options.subscription === "grok" ? endpoint("grok") : "", }; const previous = new Map( Object.keys(publicEnvironment).map((key) => [key, process.env[key]]), @@ -105,11 +117,13 @@ async function runPicked(options: { fileURLToPath(new URL("../../examples/fintech", import.meta.url)), ); const picked = tenant.agents.find( - (agent) => agent.id === "picked-harness", + (agent) => agent.id === (options.subscription ?? "picked-harness"), ); if (!picked || typeof picked.configuration.endpoint !== "string") throw new Error("Expected endpoint from default package producer"); - expect(picked.configuration.endpoint).toBe(endpoint("picked")); + expect(picked.configuration.endpoint).toBe( + endpoint(options.subscription ?? "picked"), + ); storedEndpoint = picked.configuration.endpoint; } finally { for (const [key, value] of previous) { @@ -129,6 +143,18 @@ async function runPicked(options: { PICKED_HARNESS_IMAGE: options.installed ? "localhost/synthetic-harness:fixture" : "", + AGENT_ENDPOINT_ALLOWED_HOSTS: options.subscription + ? `127.0.0.1:${server.port}` + : "", + ...(options.subscription + ? { + [`${options.subscription.toUpperCase()}_AGENT_AG_UI_URL`]: endpoint( + options.subscription, + ), + [`${options.subscription.toUpperCase()}_AGENT_TOKEN`]: + expectedToken, + } + : {}), }), ); const database = createDatabase(config.databaseUrl); @@ -159,7 +185,7 @@ async function runPicked(options: { } const rows = [ { - id: "picked-harness", + id: options.subscription ?? "picked-harness", name: "Picked Harness", type: "remote_ag_ui", title: "Synthetic harness", @@ -250,7 +276,7 @@ async function runPicked(options: { { provider: "openai", defaultModel: "unused" }, null, ); - const agent = agents["picked-harness"]; + const agent = agents[options.subscription ?? "picked-harness"]; if (!agent) throw new Error("Expected picked harness from production loader"); agent.threadId = "synthetic-auth-thread"; @@ -330,6 +356,22 @@ test("an unrelated customer endpoint receives no deployment token", async () => ); }); +test.each(["codex", "claude", "grok"] as const)( + "the %s subscription coworker receives only its bound deployment token", + async (provider) => { + const result = await runPicked({ + bundled: false, + installed: false, + target: provider, + subscription: provider, + packageProducer: true, + }); + expect(result.requests.map((request) => request.status)).toEqual([200]); + expect(result.requests[0]?.headerNames).toContain("x-openbot-agent-token"); + expect(result.failed).toBe(false); + }, +); + test("a picked installed harness requires a token even when the bundled Bot is omitted", () => { expect(() => loadConfig( diff --git a/server/tests/support/environment.ts b/server/tests/support/environment.ts index 60725d959..c8cdd6ef4 100644 --- a/server/tests/support/environment.ts +++ b/server/tests/support/environment.ts @@ -17,6 +17,7 @@ export function testEnvironment( GOOGLE_OAUTH_CLIENT_SECRET: "google-client-secret", BETTER_AUTH_SECRET: "a-long-enough-local-development-auth-secret", BETTER_AUTH_URL: "http://localhost:3001", + OPENBOT_OWNER_EMAIL: "admin@openbot.test", // Required whenever a provider is configured: nothing else grants the administrator role. INITIAL_ADMIN_EMAILS: "admin@openbot.test", // Required. See server/src/config.ts: there is no runtime without Intelligence. diff --git a/server/tests/tenant-package.test.ts b/server/tests/tenant-package.test.ts index ab0e837a6..4e41febf9 100644 --- a/server/tests/tenant-package.test.ts +++ b/server/tests/tenant-package.test.ts @@ -459,6 +459,50 @@ describe("tenant YAML validation", () => { }); }); + test("loads subscription coworkers independently in Codex, Claude, Grok order", async () => { + const names = [ + "CODEX_AGENT_AG_UI_URL", + "CLAUDE_AGENT_AG_UI_URL", + "GROK_AGENT_AG_UI_URL", + ] as const; + const previous = new Map(names.map((name) => [name, process.env[name]])); + try { + process.env.CODEX_AGENT_AG_UI_URL = + "http://subscription-workers.openbot.internal:4210/ag-ui"; + process.env.CLAUDE_AGENT_AG_UI_URL = ""; + process.env.GROK_AGENT_AG_UI_URL = ""; + const source = fileURLToPath( + new URL("../../examples/fintech", import.meta.url), + ); + const codexOnly = await loadTenantPackage(source); + expect( + codexOnly.agents + .filter((agent) => ["codex", "claude", "grok"].includes(agent.id)) + .map((agent) => agent.id), + ).toEqual(["codex"]); + + process.env.CLAUDE_AGENT_AG_UI_URL = + "http://subscription-workers.openbot.internal:4211/ag-ui"; + process.env.GROK_AGENT_AG_UI_URL = + "http://subscription-workers.openbot.internal:4212/ag-ui"; + const allProviders = await loadTenantPackage(source); + expect( + allProviders.agents + .filter((agent) => ["codex", "claude", "grok"].includes(agent.id)) + .map((agent) => [agent.id, agent.configuration.endpoint]), + ).toEqual([ + ["codex", "http://subscription-workers.openbot.internal:4210/ag-ui"], + ["claude", "http://subscription-workers.openbot.internal:4211/ag-ui"], + ["grok", "http://subscription-workers.openbot.internal:4212/ag-ui"], + ]); + } finally { + for (const [name, value] of previous) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + }); + test("accepts the complete fintech package and normalizes agent types", () => { const tenantPackage = validateTenantPackage({ brand: `tenant:\n id: fintech\n product_name: Ledgerline\nskin:\n stylesheet: theme.css`, diff --git a/tests/aws-compose.test.ts b/tests/aws-compose.test.ts new file mode 100644 index 000000000..d6e6c6bd5 --- /dev/null +++ b/tests/aws-compose.test.ts @@ -0,0 +1,250 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const repositoryRoot = join(import.meta.dir, ".."); +const deployRoot = join(repositoryRoot, "deploy", "aws"); +const digest = `sha256:${"a".repeat(64)}`; + +interface ComposeService { + environment: Record; + read_only: boolean; + cap_drop: string[]; + security_opt: string[]; + ports: Array<{ target: number; published: string }>; + deploy: { resources: { limits: { memory: string } } }; + volumes: Array<{ source: string; target: string; read_only?: boolean }>; + command: string[]; +} + +function renderCompose(file: string, profile?: string) { + const temporary = mkdtempSync(join(tmpdir(), "openbot-aws-compose-")); + const envFile = join(temporary, "runtime.env"); + writeFileSync(envFile, "PLACEHOLDER=not-a-secret\n"); + try { + const args = [ + "compose", + "--env-file", + "/dev/null", + "-f", + join(deployRoot, file), + ...(profile ? ["--profile", profile] : []), + "config", + "--format", + "json", + ]; + return JSON.parse( + execFileSync("docker", args, { + cwd: repositoryRoot, + encoding: "utf8", + env: { + PATH: process.env.PATH ?? "", + AWS_REGION: "us-east-1", + CONTROL_LOG_GROUP: "/openbot/test/control", + WORKER_LOG_GROUP: "/openbot/test/worker", + CONTROL_ENV_FILE: envFile, + WORKER_ENV_FILE: envFile, + OPENBOT_IMAGE: `example.invalid/openbot@${digest}`, + POSTGRES_IMAGE: `example.invalid/postgres@${digest}`, + CADDY_IMAGE: `example.invalid/caddy@${digest}`, + GATEWAY_IMAGE: `example.invalid/gateway@${digest}`, + POSTGRES_PASSWORD: "placeholder", + DATABASE_URL: "postgres://openbot:placeholder@postgres:5432/openbot", + CODEX_GATEWAY_TOKEN: "codex-placeholder", + CLAUDE_GATEWAY_TOKEN: "claude-placeholder", + GROK_GATEWAY_TOKEN: "grok-placeholder", + REPOSITORY_URL: "https://github.com/Tbsheff/OpenBot.git", + BASE_BRANCH: "main", + OPENBOT_DOMAIN: "openbot.example.com", + WORKER_SHARED_SECRET: "placeholder", + }, + }), + ) as { + services: Record; + }; + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +} + +test("renders one hardened gateway per provider with fixed private ports", () => { + const config = renderCompose("compose.worker.yml", "*"); + const expected = { codex: 4210, claude: 4211, grok: 4212 } as const; + + for (const [provider, port] of Object.entries(expected)) { + const service = config.services[`${provider}-gateway`]; + expect(service.environment.PROVIDER).toBe(provider); + expect(service.environment.PORT).toBe(String(port)); + expect(service.environment.GATEWAY_TOKEN).toBe(`${provider}-placeholder`); + expect(service.read_only).toBe(true); + expect(service.cap_drop).toContain("ALL"); + expect(service.security_opt).toContain("no-new-privileges:true"); + expect(service.ports).toContainEqual( + expect.objectContaining({ target: port, published: String(port) }), + ); + expect(service.deploy.resources.limits.memory).toBeTruthy(); + } + expect(config.services["codex-gateway"].security_opt).toContain( + "seccomp=unconfined", + ); + expect(config.services["claude-gateway"].security_opt).not.toContain( + "seccomp=unconfined", + ); + expect(config.services["grok-gateway"].security_opt).toContain( + "seccomp=unconfined", + ); +}); + +test("keeps each provider auth mount distinct while sharing only gateway state", () => { + const config = renderCompose("compose.worker.yml", "*"); + const authTargets = { + codex: "/home/gateway/.codex", + claude: "/home/gateway/.claude", + grok: "/home/gateway/.grok", + } as const; + + for (const provider of Object.keys(authTargets) as Array< + keyof typeof authTargets + >) { + const service = config.services[`${provider}-gateway`]; + const volumes = service.volumes as Array<{ + source: string; + target: string; + }>; + expect(volumes).toContainEqual( + expect.objectContaining({ + source: `/srv/openbot-auth/${provider}`, + target: authTargets[provider], + }), + ); + expect(volumes).toContainEqual( + expect.objectContaining({ + source: "/srv/openbot-worker/state", + target: "/var/lib/openbot-gateway", + }), + ); + for (const other of Object.keys(authTargets).filter( + (name) => name !== provider, + )) { + expect( + volumes.some((volume) => + volume.source.includes(`/openbot-auth/${other}`), + ), + ).toBe(false); + } + } + + expect(config.services["codex-gateway"].volumes).toContainEqual( + expect.objectContaining({ + source: join(deployRoot, "codex-config.toml"), + target: "/home/gateway/.codex/config.toml", + read_only: true, + }), + ); +}); + +test("keeps the Claude subscription service off until its release gate passes", () => { + const defaultConfig = renderCompose("compose.worker.yml"); + const enabledConfig = renderCompose( + "compose.worker.yml", + "claude-subscription", + ); + + expect(defaultConfig.services["claude-gateway"]).toBeUndefined(); + expect(enabledConfig.services["claude-gateway"]).toBeTruthy(); +}); + +test("does not expose host control surfaces to gateway containers", () => { + const source = readFileSync(join(deployRoot, "compose.worker.yml"), "utf8"); + + expect(source).not.toContain("/var/run/docker.sock"); + expect(source).toContain('AWS_EC2_METADATA_DISABLED: "true"'); + expect(source).toContain( + "HOST_SEMAPHORE_PATH: /var/lib/openbot-gateway/host-semaphore.sqlite", + ); + expect(source).toContain("read_only: true"); +}); + +test("renders the TLS control stack and external scheduled jobs", () => { + const config = renderCompose("compose.control.yml", "jobs"); + + expect(Object.keys(config.services)).toEqual( + expect.arrayContaining([ + "postgres", + "openbot", + "caddy", + "routine-sweep", + "attachment-cleanup", + "database-backup", + "migrate", + ]), + ); + expect(config.services.caddy.ports).toEqual( + expect.arrayContaining([ + expect.objectContaining({ target: 80, published: "80" }), + expect.objectContaining({ target: 443, published: "443" }), + ]), + ); + expect(config.services["routine-sweep"].command).toContain( + "scripts/fire-routines.ts", + ); + expect(config.services["attachment-cleanup"].command).toContain( + "scripts/cull-staged-attachments.ts", + ); + expect(config.services.migrate.command.join(" ")).toContain( + "scripts/migrate.ts", + ); + expect(config.services["database-backup"].command.join(" ")).toContain( + "pg_dump", + ); + expect(config.services["routine-sweep"].environment).not.toHaveProperty( + "CODEX_HOME", + ); + expect(config.services["routine-sweep"].environment).not.toHaveProperty( + "CLAUDE_CONFIG_DIR", + ); + expect(config.services["routine-sweep"].environment).not.toHaveProperty( + "GROK_HOME", + ); +}); + +test("requires digest-qualified production image inputs", () => { + const control = readFileSync(join(deployRoot, "compose.control.yml"), "utf8"); + const worker = readFileSync(join(deployRoot, "compose.worker.yml"), "utf8"); + + for (const variable of ["OPENBOT_IMAGE", "POSTGRES_IMAGE", "CADDY_IMAGE"]) { + expect(control).toContain( + `\${${variable}:?set a digest-qualified image reference}`, + ); + } + expect(worker).toMatch( + /\$\{GATEWAY_IMAGE:\?set a digest-qualified image reference\}/, + ); + expect(`${control}\n${worker}`).not.toMatch(/image:\s+[^\n]*:latest\b/); +}); + +test("installs system services for Compose, metadata blocking, and both schedules", () => { + const systemd = join(deployRoot, "systemd"); + const unitNames = [ + "openbot-control.service", + "openbot-worker.service", + "openbot-worker-firewall.service", + "openbot-routines.timer", + "openbot-routines.service", + "openbot-attachment-cleanup.timer", + "openbot-attachment-cleanup.service", + "openbot-database-backup.timer", + "openbot-database-backup.service", + "openbot-health-metrics.timer", + "openbot-health-metrics.service", + ]; + + for (const unit of unitNames) { + expect(readFileSync(join(systemd, unit), "utf8").trim()).not.toBeEmpty(); + } + expect( + readFileSync(join(deployRoot, "bin", "worker-firewall"), "utf8"), + ).toContain("169.254.169.254/32"); +}); diff --git a/tests/aws-infra.test.ts b/tests/aws-infra.test.ts new file mode 100644 index 000000000..64b8c8ced --- /dev/null +++ b/tests/aws-infra.test.ts @@ -0,0 +1,141 @@ +import { expect, test } from "bun:test"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; + +const root = join(import.meta.dir, "..", "infra", "aws"); + +function terraformFiles(directory = root): string[] { + return readdirSync(directory) + .flatMap((name) => { + const path = join(directory, name); + return statSync(path).isDirectory() + ? terraformFiles(path) + : path.endsWith(".tf") + ? [path] + : []; + }) + .sort(); +} + +function terraformText(): string { + return terraformFiles() + .map((path) => `# ${relative(root, path)}\n${readFileSync(path, "utf8")}`) + .join("\n"); +} + +test("uses the fixed two-host EC2 shape without costly managed services", () => { + const source = terraformText(); + + expect(source).toMatch(/instance_type\s*=\s*"t3\.medium"/); + expect(source).toMatch(/instance_type\s*=\s*"t3\.xlarge"/); + expect(source).toContain("al2023-ami-kernel-default-x86_64"); + for (const resource of [ + "aws_eks_cluster", + "aws_nat_gateway", + "aws_lb", + "aws_db_instance", + ]) { + expect(source).not.toContain(`resource "${resource}"`); + } +}); + +test("opens only TLS on control and only the three gateway ports from control", () => { + const network = readFileSync( + join(root, "modules", "network", "main.tf"), + "utf8", + ); + + expect(network).toContain('for_each = toset(["80", "443"])'); + expect(network).toContain('for_each = toset(["4210", "4211", "4212"])'); + expect(network).toMatch( + /referenced_security_group_id\s*=\s*aws_security_group\.control\.id/, + ); + expect(network).not.toMatch(/from_port\s*=\s*22\b/); + expect(network).not.toMatch(/to_port\s*=\s*22\b/); + const workerIngress = network.slice( + network.indexOf( + 'resource "aws_vpc_security_group_ingress_rule" "worker_gateway"', + ), + network.indexOf('resource "aws_vpc_security_group_egress_rule" "worker"'), + ); + expect(workerIngress).not.toContain("cidr_ipv4"); +}); + +test("uses SSM, exact ECR and log permissions, and no wildcard IAM action", () => { + const source = terraformText(); + + expect(source).toContain("AmazonSSMManagedInstanceCore"); + expect(source).toContain("ecr:GetAuthorizationToken"); + expect(source).toContain("ecr:BatchGetImage"); + expect(source).toContain("logs:PutLogEvents"); + expect(source).not.toMatch(/actions\s*=\s*\[[^\]]*"\*"/s); +}); + +test("requires IMDSv2 and retains encrypted data and provider auth volumes", () => { + const source = terraformText(); + + expect(source.match(/http_tokens\s*=\s*"required"/g)?.length).toBe(2); + expect(source.match(/http_put_response_hop_limit\s*=\s*1/g)?.length).toBe(2); + expect(source.match(/encrypted\s*=\s*true/g)?.length).toBeGreaterThanOrEqual( + 5, + ); + expect(source).toContain('providers = toset(["codex", "claude", "grok"])'); + expect( + source.match(/delete_on_termination\s*=\s*false/g)?.length, + ).toBeGreaterThanOrEqual(2); + expect( + source.match(/skip_destroy\s*=\s*true/g)?.length, + ).toBeGreaterThanOrEqual(2); + expect( + source.match(/prevent_destroy\s*=\s*true/g)?.length, + ).toBeGreaterThanOrEqual(3); + expect(source).toMatch(/Backup\s*=\s*"excluded-provider-auth"/); + expect(source).toMatch(/Backup\s*=\s*"included-service-data"/); +}); + +test("gives control a stable address and worker a private DNS record", () => { + const source = terraformText(); + + expect(source).toContain('resource "aws_eip" "control"'); + expect(source).toContain('resource "aws_route53_zone" "private"'); + expect(source).toContain('resource "aws_route53_record" "worker"'); + expect(source).toContain("worker.openbot.internal"); +}); + +test("passes only secret references through Terraform and pins providers", () => { + const source = terraformText(); + + expect(source).toContain('version = "= 6.14.1"'); + expect(source).toContain("control_secret_arns"); + expect(source).toContain("worker_secret_arns"); + expect(source).not.toContain("aws_secretsmanager_secret_version"); + expect(source).not.toMatch(/secret_string\s*=/); +}); + +test("backs up only service data and provisions owner alerts", () => { + const source = terraformText(); + + expect(source).toContain('resource "aws_backup_plan" "daily"'); + expect(source).toContain('value = "included-service-data"'); + expect(source).not.toMatch( + /selection_tag[\s\S]{0,200}excluded-provider-auth/, + ); + expect(source).toContain('namespace = "OpenBot/Personal"'); + expect(source).toContain('resource "aws_sns_topic" "alerts"'); + expect(source).toContain('metric_name = "QueueDepth"'); + expect(source).toContain('metric_name = "DiskUsedPercent"'); +}); + +test("limits the GitHub deploy role to production OIDC and deployment resources", () => { + const source = terraformText(); + + expect(source).toContain("AssumeRoleWithWebIdentity"); + expect(source).toContain( + ["repo:", "$", "{var.github_repository}", ":environment:production"].join( + "", + ), + ); + expect(source).toContain("token.actions.githubusercontent.com:aud"); + expect(source).toContain("AWS-RunShellScript"); + expect(source).toContain("secretsmanager:PutSecretValue"); +}); diff --git a/tests/release-config.test.ts b/tests/release-config.test.ts new file mode 100644 index 000000000..c0a9c6779 --- /dev/null +++ b/tests/release-config.test.ts @@ -0,0 +1,48 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); +const read = (path: string) => readFileSync(join(root, path), "utf8"); + +test("tests and scans the standalone gateway without cloud credentials", () => { + const workflow = read(".github/workflows/subscription-gateway.yml"); + + expect(workflow).toContain("bun install --frozen-lockfile"); + expect(workflow).toContain("bun run typecheck"); + expect(workflow).toContain("bun run test"); + expect(workflow).toContain("platforms: linux/amd64"); + expect(workflow).toContain("--severity CRITICAL"); + expect(workflow).not.toContain("AWS_ACCESS_KEY_ID"); +}); + +test("uses protected OIDC deployment with signed digest images and SSM", () => { + const workflow = read(".github/workflows/aws-deploy.yml"); + + expect(workflow).toContain("environment: production"); + expect(workflow).toContain("id-token: write"); + expect(workflow).toContain("role-to-assume:"); + expect(workflow).toContain("cosign sign --yes"); + expect(workflow).toContain("deploy-via-ssm"); + expect(workflow).toContain("Deploy worker first"); + expect(workflow).not.toContain("AWS_ACCESS_KEY_ID"); + expect(workflow).not.toMatch(/image[^\n]*:latest/); +}); + +test("rolls a failed host start back without changing provider auth", () => { + const deploy = read("deploy/aws/bin/deploy-via-ssm"); + + expect(deploy).toContain("restored the prior image setting"); + expect(deploy).toContain("file://$old_secret"); + expect(deploy).toContain("secretsmanager put-secret-value"); + expect(deploy).not.toContain("/srv/openbot-auth"); +}); + +test("publishes the gateway and uses the current fork package namespace", () => { + const images = JSON.parse(read(".github/published-images.json")) as string[]; + const workflow = read(".github/workflows/publish-release.yml"); + + expect(images).toContain("agent-subscription-gateway"); + expect(workflow).toContain("$" + "{GITHUB_REPOSITORY_OWNER,,}/openbot"); + expect(workflow).not.toContain("ghcr.io/copilotkit/openbot"); +}); diff --git a/tests/runbook-contract.test.ts b/tests/runbook-contract.test.ts new file mode 100644 index 000000000..ec3b9ead5 --- /dev/null +++ b/tests/runbook-contract.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); +const read = (path: string) => readFileSync(join(root, path), "utf8"); + +test("documents the complete AWS launch and honest end-to-end gate", () => { + const deployment = read("docs/runbooks/aws-deployment.md"); + + for (const required of [ + "Terraform", + "Secrets Manager", + "GitHub Actions OIDC", + "production", + "DNS", + "TLS", + "provider device sign-ins", + "container replacement", + "not prove the AWS path", + ]) { + expect(deployment).toContain(required); + } +}); + +test("documents alerts, rollback, backup restore, and cost checks", () => { + const operations = read("docs/runbooks/worker-operations.md"); + + for (const required of [ + "ServiceReady", + "QueueDepth", + "manual rollback", + "pg_restore", + "Backup=excluded-provider-auth", + "AWS Pricing Calculator", + ]) { + expect(operations).toContain(required); + } +}); + +test("keeps provider sign-ins out of shared secret stores and gates Claude", () => { + const signIn = read("docs/runbooks/provider-sign-in.md"); + + expect(signIn).toContain( + "Do not put provider credentials in AWS Secrets Manager", + ); + expect(signIn).toContain("Claude release gate"); + expect(signIn).toContain("written guidance from Anthropic"); + expect(signIn).toContain("codex login --device-auth"); + expect(signIn).toContain("grok login --device-auth"); +});