diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 09d96892a1..2ff6ff1bf8 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -33,7 +33,7 @@ jobs: with: persist-credentials: false - name: Install dependencies - run: yarn install --frozen-lockfile + run: yarn install --immutable --mode=skip-build - name: Run prettier run: yarn format-check - name: Run linter @@ -50,3 +50,42 @@ jobs: name: coverage-reports path: ./**/coverage retention-days: 5 + + microvm-lifecycle-hooks: + name: Build MicroVM lifecycle hook + runs-on: ubuntu-latest + container: + image: node:24@sha256:aa648b387728c25f81ff811799bbf8de39df66d7e2d9b3ab55cc6300cb9175d9 + defaults: + run: + working-directory: ./lambdas + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install dependencies + run: yarn install --immutable --mode=skip-build + + - name: Run prettier + run: yarn prettier --check "services/microvm-lifecycle-hooks/**/*.{ts,json,md}" + + - name: Run linter + run: yarn eslint services/microvm-lifecycle-hooks/src + + - name: Run tests + run: yarn nx test @aws-github-runner/microvm-lifecycle-hooks + + - name: Build distribution + run: yarn workspace @aws-github-runner/microvm-lifecycle-hooks build + + - name: Verify server distribution + run: | + test -s services/microvm-lifecycle-hooks/dist/server.js + test -f services/microvm-lifecycle-hooks/dist/package.json diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 15516fce1f..6fd6e41d95 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -51,15 +51,13 @@ jobs: - prebuilt - default - ephemeral - - microvm - microvm-foundation - migration-test - multi-runner - - multi-runner-v2 - termination-watcher services: ministack: - image: ghcr.io/ministackorg/ministack:1.5.13@sha256:ce3c906f2866ff953ce4c56f06b1fa3e453bc32e41c00de17b5f5a8672c5a42c + image: ghcr.io/ministackorg/ministack:1.5.16@sha256:9813da34285a0760477c761c1c03717e0290d259213c0ca97f551fefd87b292d ports: - 4566:4566 env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68f5a38341..b5789d50dd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,21 @@ jobs: persist-credentials: false - name: Build dist working-directory: lambdas - run: yarn install --frozen-lockfile && yarn run test && yarn dist + run: yarn install --immutable --mode=skip-build && yarn run test && yarn dist + + - name: Build MicroVM lifecycle hook + working-directory: lambdas + run: yarn workspace @aws-github-runner/microvm-lifecycle-hooks build + + - name: Verify MicroVM lifecycle hook distribution + working-directory: lambdas + run: | + test -s services/microvm-lifecycle-hooks/dist/server.js + test -f services/microvm-lifecycle-hooks/dist/package.json + + - name: Package MicroVM lifecycle hook + working-directory: lambdas/services/microvm-lifecycle-hooks + run: (cd dist && zip -r ../microvm-lifecycle-hooks.zip .) - name: Get installation token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 id: token @@ -61,24 +75,34 @@ jobs: uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: '${{ github.workspace }}/lambdas/functions/**/*.zip' + + - name: Attest MicroVM lifecycle hook + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: lifecycle-hook-attest + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: '${{ github.workspace }}/lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip' + - name: Update release notes with attestation if: ${{ steps.release.outputs.releases_created == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ github.event.inputs.version }} TAG_NAME: ${{ steps.release.outputs.tag_name }} ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }} + LIFECYCLE_HOOK_ATTESTATION_URL: ${{ steps.lifecycle-hook-attest.outputs.attestation-url }} REPOSITORY: ${{ github.repository }} run: | - version="${VERSION}" tag_name="${TAG_NAME}" attestation_url="${ATTESTATION_URL}" + lifecycle_hook_attestation_url="${LIFECYCLE_HOOK_ATTESTATION_URL}" repository="${REPOSITORY}" - gh release view $version --json body -q '.body' > new-release-notes.md + gh release view "$tag_name" --json body -q '.body' > new-release-notes.md echo "## Attestation" >> new-release-notes.md - echo "Attestation url: $attestation_url" >> new-release-notes.md + echo "Lambda attestation url: $attestation_url" >> new-release-notes.md + echo "MicroVM lifecycle hook attestation url: $lifecycle_hook_attestation_url" >> new-release-notes.md echo "Verify the artifacts by running \`gh attestation verify --repo ${repository}\`" >> new-release-notes.md - gh release edit $tag_name -F new-release-notes.md -t $tag_name + gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name" + - name: Upload release assets if: ${{ steps.release.outputs.releases_created == 'true' }} env: @@ -86,10 +110,13 @@ jobs: TAG_NAME: ${{ steps.release.outputs.tag_name }} run: | tag_name="${TAG_NAME}" - for f in $(find . -name '*.zip'); do - gh release upload $tag_name $f - done - - name: Attach attestation + while IFS= read -r -d '' f; do + gh release upload "$tag_name" "$f" + done < <(find lambdas/functions -name '*.zip' -print0) + gh release upload "$tag_name" \ + "lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip" + + - name: Attach Lambda attestation if: ${{ steps.release.outputs.releases_created == 'true' }} env: ATTESTATION_BUNDLE: ${{ steps.attest.outputs.bundle-path }} @@ -99,13 +126,26 @@ jobs: run: | # rename attest bundle to github-aws-runners-terraform-aws-github-runner-attestation-$attestation-id.sigstore # OpenSSF expects the attestation bundle to be named in this format (*.sigstore) - SIGSTORE_BUNDLE=$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.sigstore - INTOTO_BUNDLE=$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.intoto.jsonl - mv ${ATTESTATION_BUNDLE} $SIGSTORE_BUNDLE - if [ -z "$SIGSTORE_BUNDLE" ]; then - echo "No attestation bundle found, skipping attachment." - exit 0 - fi - gh release upload $TAG_NAME "$SIGSTORE_BUNDLE" - cat ${SIGSTORE_BUNDLE} | jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' | base64 -d | jq .> ${INTOTO_BUNDLE} - gh release upload $TAG_NAME "${INTOTO_BUNDLE}" + sigstore_bundle="$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.sigstore" + intoto_bundle="$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.intoto.jsonl" + cp "$ATTESTATION_BUNDLE" "$sigstore_bundle" + gh release upload "$TAG_NAME" "$sigstore_bundle" + jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' "$sigstore_bundle" \ + | base64 --decode > "$intoto_bundle" + gh release upload "$TAG_NAME" "$intoto_bundle" + + - name: Attach MicroVM lifecycle hook attestation + if: ${{ steps.release.outputs.releases_created == 'true' }} + env: + ATTESTATION_BUNDLE: ${{ steps.lifecycle-hook-attest.outputs.bundle-path }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ steps.release.outputs.tag_name }} + ATTESTATION_ID: ${{ steps.lifecycle-hook-attest.outputs.attestation-id }} + run: | + sigstore_bundle="$RUNNER_TEMP/github-aws-runners-microvm-lifecycle-hooks-attestation-${ATTESTATION_ID}.sigstore" + intoto_bundle="$RUNNER_TEMP/github-aws-runners-microvm-lifecycle-hooks-attestation-${ATTESTATION_ID}.intoto.jsonl" + cp "$ATTESTATION_BUNDLE" "$sigstore_bundle" + gh release upload "$TAG_NAME" "$sigstore_bundle" + jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' "$sigstore_bundle" \ + | base64 --decode > "$intoto_bundle" + gh release upload "$TAG_NAME" "$intoto_bundle" diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index 5b519480c3..85535f1036 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -2,7 +2,13 @@ name: "Smoke Tests" on: pull_request: - paths: ["**/*.tf", "**/*.hcl", ".github/workflows/smoke-tests.yml"] + paths: + - "**/*.tf" + - "**/*.hcl" + - "images/microvm-ubuntu/**" + - "lambdas/**" + - "tests/ministack/**" + - ".github/workflows/smoke-tests.yml" workflow_dispatch: concurrency: @@ -26,10 +32,10 @@ jobs: control_plane_smoke: name: Run webhook and pool lifecycle smoke test against MiniStack runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 120 services: ministack: - image: ghcr.io/ministackorg/ministack:1.5.13@sha256:ce3c906f2866ff953ce4c56f06b1fa3e453bc32e41c00de17b5f5a8672c5a42c + image: ghcr.io/ministackorg/ministack:1.5.16@sha256:9813da34285a0760477c761c1c03717e0290d259213c0ca97f551fefd87b292d ports: - 4566:4566 options: --add-host=host.docker.internal:host-gateway @@ -59,15 +65,9 @@ jobs: terraform_version: latest terraform_wrapper: false - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas + - name: Build Lambda distributions for smoke tests run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist + ./.ci/build.sh - name: Start MockServer id: mockserver @@ -77,9 +77,17 @@ jobs: port: '1080' startup-timeout: '60' + - name: Install boto3 + run: python3 -m pip install --upgrade boto3 botocore + + - name: Set up ARM64 emulation + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 + - name: Run webhook and pool lifecycle smoke test env: MINISTACK_GITHUB_MOCK_HOST: host.docker.internal MINISTACK_GITHUB_MOCK_PORT: "1080" MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-smoke.sh + run: python3 tests/ministack/run-webhook-smoke.py diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index a3da1f17d9..ab73258140 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -50,10 +50,8 @@ env: prebuilt ephemeral termination-watcher - microvm microvm-foundation multi-runner - multi-runner-v2 external-managed-ssm-secrets TEST_MODULES: | modules/runners diff --git a/.gitignore b/.gitignore index 276fe10733..e95da29855 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ secrets.auto.tfvars node_modules/ site/ + +__pycache__/ +ministack-smoke-checklist.txt +ministack-smoke.log diff --git a/docs/examples/index.md b/docs/examples/index.md index b7bdf60811..526eb6a112 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -5,11 +5,10 @@ Examples are located in the [examples](https://github.com/github-aws-runners/ter - _[Default](default.md)_: The default example of the module - _[Ephemeral](ephemeral.md)_: Example usages of ephemeral runners based on the default example. - _[Multi Runner](multi-runner.md)_ : Example usage of creating a multi runner which creates multiple runners/ configurations with a single deployment. The examples including: "arm64", "windows", and "ubuntu" runners. -- _[Multi Runner v2](multi-runner-v2.md)_ : Example usage of the experimental v2 multi-runner configuration interface with shared defaults and per-lane overrides. +- _[Multi Runner Webhook](multi-runner-webhook.md)_: Example usage of one webhook deployment serving EC2 and Lambda MicroVM runner lanes. - _[Permissions boundary](permissions-boundary.md)_: Example usages of permissions boundaries. - _[Prebuilt Images](prebuilt.md)_: Example usages of deploying runners with a custom prebuilt image. - _[Termination watcher](termination-watcher.md)_: Example usages of termination watcher. - _[Dedicated Mac Hosts](dedicated-mac-hosts.md)_: Example usage of setting up dedicated hosts for macOS runners. - _[Externally managed SSM secrets](external-managed-ssm-secrets.md)_: Example usage of externally managed SSM secrets for the GitHub App credentials. - _[MicroVM foundation](microvm-foundation.md)_: Example usage of the regional Lambda MicroVM image-build and Network Connector prerequisites. -- _[Lambda MicroVM](microvm.md)_: Example usage of Linux ARM64 ephemeral runners backed by Lambda MicroVM images. diff --git a/docs/examples/microvm.md b/docs/examples/microvm.md deleted file mode 100644 index 4014781114..0000000000 --- a/docs/examples/microvm.md +++ /dev/null @@ -1,3 +0,0 @@ -# Lambda MicroVM - ---8<-- "examples/microvm/README.md" diff --git a/docs/examples/multi-runner-v2.md b/docs/examples/multi-runner-v2.md deleted file mode 100644 index 565b601ecb..0000000000 --- a/docs/examples/multi-runner-v2.md +++ /dev/null @@ -1 +0,0 @@ ---8<-- "examples/multi-runner-v2/README.md" diff --git a/docs/examples/multi-runner-webhook.md b/docs/examples/multi-runner-webhook.md new file mode 100644 index 0000000000..19d24d5994 --- /dev/null +++ b/docs/examples/multi-runner-webhook.md @@ -0,0 +1 @@ +--8<-- "examples/multi-runner-webhook/README.md" diff --git a/docs/microvm-runners.md b/docs/microvm-runners.md new file mode 100644 index 0000000000..872dd52de0 --- /dev/null +++ b/docs/microvm-runners.md @@ -0,0 +1,129 @@ +# Lambda MicroVM Runners (Experimental) + +!!! warning + Lambda MicroVM runner support is experimental. The image build, lifecycle-hook server, control-plane integration, and AWS MicroVM APIs must be configured together. Validate the complete flow in a non-production environment before relying on it for workloads. + +## Overview + +Lambda MicroVM runners provide ephemeral GitHub Actions runners backed by +Lambda MicroVMs. The runner control plane receives demand, obtains the +one-time runner configuration, starts a MicroVM from a published image, and +passes the runtime execution role to the MicroVM. + +The repository includes a combined [multi-runner webhook example](examples/multi-runner-webhook.md) +that places EC2 and Lambda MicroVM lanes behind one webhook endpoint. The +provider-specific lifecycle checks are shared where possible, so the same +deployment can validate both providers. + +## Prerequisites + +Before deploying the MicroVM runner lane, prepare all of the following in the +target AWS Region: + +1. **MicroVM foundation.** Apply the + [MicroVM foundation example](examples/microvm-foundation.md). It creates the + regional artifact bucket, Lambda Network Connectors, the image-build role, + and the reusable MicroVM usage policy. +2. **Lifecycle-hook artifact.** Build and release + `lambdas/services/microvm-lifecycle-hooks` through the same workspace + artifact process used for the repository's Lambda services. The resulting + ZIP is embedded in the MicroVM image. +3. **Published MicroVM image.** Use the + [MicroVM Ubuntu image instructions](https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/images/microvm-ubuntu/README.md) + to build and publish an image with Packer. The image must contain the + compatible lifecycle-hook server and runner entrypoint. +4. **Runner execution role.** Configure the runner role through the runner + configuration. This is different from the foundation's build role. The + control-plane TypeScript passes the execution role to `RunMicrovm`, so the + Lambda that starts the MicroVM must have permission to pass it. +5. **Runner control plane and artifacts.** Deploy the runner control plane with + the published image ARN/version, Network Connector ARNs, GitHub App + configuration, and the runner-control and webhook Lambda ZIPs. + +The foundation does not create the image or the runner execution role. The +image build does not choose the runtime role. These are separate dependencies +owned by the image build and runner-control-plane stages respectively. + +## IAM roles + +MicroVM deployments use two roles for two different operations: + +| Role | Used by | Responsibility | +| --- | --- | --- | +| Build role (`build_role_arn`) | Packer/image publisher | Creates and publishes the MicroVM image and accesses the foundation build artifacts. | +| Execution role | Runner control plane and the MicroVM | Is passed to `RunMicrovm` and provides the permissions used by the ephemeral runner at runtime. | + +Do not use the build role as the runner execution role. The control-plane +Lambda needs `iam:PassRole` for the configured execution role, and the +execution role must contain the runtime permissions required by the selected +runner lane. + +## Deployment order + +The complete dependency chain is: + +```text +MicroVM foundation + | + v +Build/release lifecycle-hook server + | + v +Packer builds and publishes image + | + v +Runner control plane resolves execution role + | + v +RunMicrovm starts an ephemeral runner +``` + +The lifecycle-hook server is part of the image artifact. Updating the hook +server therefore requires building/releasing the artifact and publishing a +new compatible image before deploying that image version to the runner lane. + +## Combined EC2 and MicroVM deployment + +The [multi-runner webhook example](examples/multi-runner-webhook.md) accepts +explicit `runners_lambda_zip` and `webhook_lambda_zip` inputs and configures +both compute providers behind one webhook. Its MicroVM settings require a +published image: + +```hcl +compute_provider = { + aws = { + microvm = { + image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:gha-ubuntu-arm64" + image_version = null + ingress_network_connectors = [] + egress_network_connectors = ["arn:aws:lambda:eu-west-1:123456789012:network-connector:example"] + } + } +} +``` + +Use the example's complete Terraform configuration as the source of truth for +the current input shape. The example deploys the control plane; it does not +build the foundation, lifecycle-hook artifact, or MicroVM image for you. + +## Known limitations + +- This integration is experimental and depends on AWS Lambda MicroVM APIs and + the lifecycle-hook protocol. +- A compatible lifecycle-hook server must be present in every image used by + the MicroVM provider. +- Image publication and activation are separate from Terraform deployment; + wait for the image version to become active before starting jobs. +- The build role and execution role are intentionally separate. Changes to + either role can affect a different stage of the lifecycle. +- The combined webhook example is useful for integration testing, but a real + deployment still needs a real MicroVM image and the network/runtime IAM + configuration described above. + +## Repository examples + +- [MicroVM foundation](examples/microvm-foundation.md) +- [MicroVM image build README](https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/images/microvm-ubuntu/README.md) +- [Lifecycle-hook service README](https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/services/microvm-lifecycle-hooks/README.md) +- [Multi-runner webhook](examples/multi-runner-webhook.md) +- [MicroVM foundation module](modules/public/microvm-foundation.md) diff --git a/examples/microvm-foundation/README.md b/examples/microvm-foundation/README.md index 7417f3c70e..c52c872d47 100644 --- a/examples/microvm-foundation/README.md +++ b/examples/microvm-foundation/README.md @@ -21,10 +21,26 @@ documented in `../../images/microvm-ubuntu/README.md`. Use the outputs as the bu - `connector_arns.ministack` -> `MICROVM_EGRESS_NETWORK_CONNECTOR_ARN` - `usage_policy_arn` -> attach to the control-plane role used by the runner example +The deployment order is: + +1. Apply this foundation to create the regional bucket, Network Connectors, + build role, and reusable runtime policy. +2. Build and release the lifecycle-hook service from + `lambdas/services/microvm-lifecycle-hooks` using the repository's normal + Lambda artifact process. +3. Build and publish the MicroVM image with Packer, passing the foundation + outputs and the released lifecycle-hook ZIP. The image builder uses the + **build role**. +4. Deploy the runner control plane, such as + `examples/multi-runner-webhook`, with the published image ARN/version. The + control plane resolves the **execution role** from the runner configuration + and passes it to `RunMicrovm` when it starts a job. + +The two roles must not be conflated: the build role creates the image, while +the execution role runs the ephemeral GitHub Actions runner inside that image. The foundation module owns regional storage, build IAM, Network Connectors, -and the reusable runtime policy. It does not publish an image or create the -runner control plane; those steps remain explicit and can be performed after -the foundation is available. +and the reusable runtime policy. It does not publish an image, create the +execution role, or create the runner control plane. ## Requirements diff --git a/examples/microvm-foundation/variables.tf b/examples/microvm-foundation/variables.tf index 9163e2e81f..0f05980124 100644 --- a/examples/microvm-foundation/variables.tf +++ b/examples/microvm-foundation/variables.tf @@ -64,4 +64,4 @@ variable "ecr_repository_arns" { type = set(string) description = "Optional private ECR repository ARNs used by the image build." default = [] -} +} \ No newline at end of file diff --git a/examples/microvm/.terraform.lock.hcl.tofu b/examples/microvm/.terraform.lock.hcl.tofu deleted file mode 100644 index 7aa235531d..0000000000 --- a/examples/microvm/.terraform.lock.hcl.tofu +++ /dev/null @@ -1,113 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "6.63.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:1jhQJPHOPu2mzDG/ke3tK8PNcEqQHA4vhF05WWlM/yg=", - "h1:3+pvT0KN/bkJ6TBuExj+gxptEozhnpo80Ztblwq85eo=", - "h1:5aTequ87wZS7Mh4dEIayDGKcFdaFgHtw74NtqY5Idi0=", - "h1:AMRlrrM3z1SmrslOtotqKq02zapxLKtXaSN9Jbs0Oho=", - "h1:OTjECFWTDxsjcUfOKCNBp75Z5lGrW/KplRDsjTZYT2g=", - "h1:b8LORLOKMOOl+nK1M2UhCjELSjjziClJuAv6hYuySHs=", - "h1:bUfTX1giRLOyfDbBvsDbwR3tJmsTFRWcOTQdj2npDWA=", - "h1:dzs4kwx+itVGAH7yEOyeoWcE3LNRMnWtlt4ROgyAa0M=", - "h1:lnjou+SiwpYJ+j9PXWozXPHSPlhxIZb0RqpsSEBzfGw=", - "h1:pqzUeHAQj9NctgkwaynaF2aB+3QiZXcoslzMGjT743w=", - "h1:qTXEWOWxA6sfUpC29UXrsbHnNzWH7+j1RTUVG4YCm+U=", - "h1:qdHKOKt/ISn9RLjUe22OZBpN3F7H2DFeHJL/CSc2x8E=", - "h1:tpNzIZBzzUW7/kLU3BhYf3jhdO5uNwYfNmgC9B8kvMM=", - "h1:uVVlFgjg6GyxJLbCsTO1+R5fTNbZ73mLpVpSd0mMrFk=", - "h1:xGJsV5IFf7c11cXzJrsY40hiJCghp4odT0eJyTyAUYY=", - "zh:039a03e920e55f14a691feb67216a2d142bfee603128e15f9c5138f9ecd85016", - "zh:14e060b7f46ca7b0fa009b91aef419c58cbdff854de96e9a1d853166f8d902fd", - "zh:18803e8fe2c291c8db5526c71b3287ff7c81453f10ca6d8e69cdf9c535b00783", - "zh:1b83fce6e31a6095e932d80a7c3f47ac04252653a2de2b98ec6204563310fcba", - "zh:2add7bc976ceebb1a94d84598762c9b9cf281ca52ec83deeb4e95e90aa200a12", - "zh:2f22cd5372408f11937fa5513a7b960d3cebc334c5ec65fc5322c3bac1c1f664", - "zh:41c5e857dacfd83b7ca12a435204957ff6ca8830b9efefd0d381ad4d63b19779", - "zh:4eace6246e46999782d219bc4f50f83d19ef9156bacf5ca1528da12da4918015", - "zh:5e1c1281c3f929399e2ed3dbdce03426fd57a9ec55cd36e04acf1712aa5954ba", - "zh:608272b1f5d75ead123c9d933aa1fed7dc832cedd1506019046b4c8fdcc91dce", - "zh:6b3680f8a2f7be2c171953aba89d639fb2624b9cf52ec304e16434874566601d", - "zh:99aa1006f2141f3341a02020e1c91abfb02280e57c77e0415c98b8d900353d88", - "zh:9ad235bef34a89a8dd9943f9fa9f05cc729bb52a4e0dc926a31bb13cb0ae2418", - "zh:e0e3ac361e04748a4ca0c1cdbb6abab2aa817f4ad67e1692817d16e370161d59", - "zh:f60962c982a41fde956e796425e7194b4311741c179c060c1c8b5e16a557d635", - ] -} - -provider "registry.opentofu.org/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:2wld81FnmHW0WVgy081sIfokCr2+NuatS8yjeLEet7Y=", - "h1:AClQjJ6X22V4qcRgcYSxiXCMmp2pz0G8WVQC7wAx66o=", - "h1:AY3XQbuviNd2X5VhHYEbhNta1m/CG3JD2BKFKhCt1Y4=", - "h1:CUOZUd7H11lsU+4tISlnYIiP5BqnX8IDwFCVqfLJyAg=", - "h1:JIfV0nA/pLWnIFGscvTfuavQCn2NeHxJBeb6UUg/joA=", - "h1:RejAh+nyCwqDGExGln2Kb4Ro5LyHak0eJe0P9g8CHPc=", - "h1:SHOuTZjYymsmy4asuRq6NC3yW+zdVZOOt4f5nrb+EPM=", - "h1:WwPat/gT4gO8GvvKNdSkkXWVD65JppLJfqKOt9HhOqQ=", - "h1:Z3hXVLrOyaRiiLmmL5UCOdcRMguwjN1x5TYNdmBDgls=", - "h1:dd78Ad5HdfPzPts7A9qIxfitXhAriV/qza38fr2ukjk=", - "h1:dyVb++KwDdybzLTE6bf7GZiVQ31iWsgKPWmhTQ8G42k=", - "h1:gD8ZH6WWe+5gg5+y8SpLWGPUDzSxcQ3HKP8IDM/wW3I=", - "h1:juXCww0zRQKFTDZoKqYR0+Sn1lu99oeL6pr0Jh6LWx0=", - "h1:kFAySmtsshyNV7IhIrEdASzVcvwy68eeZCVC66P7yNk=", - "h1:nS5azDopRisB2NInwDx3Hrfg2FdVt8Gw0gTQzC0rd70=", - "zh:164eb061d84e01759f391265865fb31828083d0a06b25f7af7e094cbdb18c799", - "zh:1bb9b669a82b52c0cba2860c71e9ee6699ef302f28cb8ed06f572d39bc6c7c4f", - "zh:1ea9b31a8f29302122c1e8d673693f3ac270336dae560af803cd1117265a469a", - "zh:238bd463cb0154fb935dc331da40c0a9cbe5db9cee615ae5f35ccad5eed7dc41", - "zh:30ef2b7384cf7e20f33fe75754b54cf669d59816f3ad4fc73bfb2b26fb6735e9", - "zh:35b5cded16e4b57c207d03ee0979b14baf486fa520e6edb7a2eecf18f1b85471", - "zh:3dc840d13a50cd215c7540573f27e2b61f739ba90aee5b7c3846079aa0ab5534", - "zh:3f9309a18db608f975d5691fcb47a6e14d77199156a52e9c39dcafe3737f2b07", - "zh:44263a219f7dbd1848b545d080110b4f7d0495e77b71cd3c7a0b5ec52a09accb", - "zh:4dec54aa5f445eeea035bbd4839bcded5e47ecd07cba0e70c5a09e9272cb592f", - "zh:5e8fb319d7c6d6c4566a18b9d0c91580b4901a96acd7fdc476bfc79f074368e2", - "zh:b0e8b6d41834b57fcfbb5ca00da52ccb757e1a95b6a2d546c0dae8bfbeca1cdf", - "zh:bbde4c3a1dcc1718027a61a4cdf661619d17af1b58df1038fe27bcf43c3dc29b", - "zh:c4140fff9f692baf29236557f706f9515f93229413438527d764023a82301da3", - "zh:f8e9d83184e4bbeb97c6f0d569833007c48ba5a7ff334def201df4991d03a962", - ] -} - -provider "registry.opentofu.org/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", - "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", - "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", - "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", - "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", - "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", - "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", - "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", - "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", - "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", - "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", - "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", - "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", - "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", - "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", - "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", - "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", - "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", - "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", - "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", - "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", - "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", - "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", - "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", - "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", - "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", - "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", - "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", - "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", - "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", - ] -} diff --git a/examples/microvm/README.md b/examples/microvm/README.md deleted file mode 100644 index a5eb0b48d5..0000000000 --- a/examples/microvm/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Lambda MicroVM runner example - -This example creates the VPC and GitHub Actions runner control plane for one -Linux ARM64 Lambda MicroVM lane. The lane uses ephemeral runners and -just-in-time configuration, which are required by the MicroVM provider. - -The regional MicroVM foundation is provisioned separately by the -[`microvm-foundation`](../microvm-foundation) example. Apply that example -first and provide its artifact bucket, build role, and egress Network Connector -outputs to the image build script. The image ARN produced by that build is then -supplied to this example. - -The GitHub App credentials must already exist in SSM Parameter Store. The -example outputs the webhook endpoint; configure that endpoint on the GitHub -App with the same secret stored in the referenced SSM parameter. - -## Usage - -Build or download the Lambda archives into an S3 bucket, then create a -`terraform.tfvars` file. The parameter references below are examples only: - -```hcl -aws_region = "eu-west-1" -lambda_artifact_bucket = "my-runner-lambda-artifacts" -microvm_image_arn = "arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-arm64" -egress_network_connector_arn = "arn:aws:lambda:eu-west-1:123456789012:network-connector:example" - -github_app = { - key_base64_ssm = { - arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-key" - name = "/github-runner/app-key" - } - id_ssm = { - arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" - name = "/github-runner/app-id" - } - webhook_secret_ssm = { - arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/webhook-secret" - name = "/github-runner/webhook-secret" - } -} -``` - -Run Terraform from this directory: - -```bash -terraform init -terraform apply -terraform output -raw webhook_endpoint -``` - -The MicroVM image must be built for Linux ARM64 and should use a versioned image -ARN in production. Network connector egress remains bounded by the VPC route -tables and network ACLs configured by the helper module. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [aws](#requirement\_aws) | >= 6.33 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [random](#provider\_random) | 3.9.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [base](#module\_base) | ../base | n/a | -| [runners](#module\_runners) | ../../modules/multi-runner | n/a | - -## Resources - -| Name | Type | -|------|------| -| [random_id.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [aws\_region](#input\_aws\_region) | AWS Region where the runner control plane and MicroVM resources are deployed. | `string` | `"eu-west-1"` | no | -| [egress\_network\_connector\_arn](#input\_egress\_network\_connector\_arn) | Regional Lambda Network Connector ARN used by MicroVMs and the image build. | `string` | n/a | yes | -| [environment](#input\_environment) | Name prefix for the example resources. | `string` | `null` | no | -| [github\_app](#input\_github\_app) | GitHub for API usages. | 
object({
    id         = string
    key_base64 = string
  })
| n/a | yes | -| [ingress\_network\_connector\_arns](#input\_ingress\_network\_connector\_arns) | Optional regional Lambda Network Connector ARNs exposed to MicroVMs. | `list(string)` | `[]` | no | -| [lambda\_artifact\_bucket](#input\_lambda\_artifact\_bucket) | S3 bucket containing the runner-control Lambda artifacts. | `string` | n/a | yes | -| [microvm\_image\_arn](#input\_microvm\_image\_arn) | Lambda MicroVM image ARN produced by the MicroVM image build. | `string` | n/a | yes | -| [microvm\_image\_version](#input\_microvm\_image\_version) | Optional immutable version of the Lambda MicroVM image. | `string` | `null` | no | -| [organization\_runners](#input\_organization\_runners) | Register the MicroVM runners at organization scope when true. | `bool` | `false` | no | -| [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for the runners Lambda archive. | `string` | `"runners.zip"` | no | -| [runners\_maximum\_count](#input\_runners\_maximum\_count) | Maximum number of concurrent MicroVM runners. | `number` | `10` | no | -| [webhook\_lambda\_s3\_key](#input\_webhook\_lambda\_s3\_key) | S3 key for the webhook Lambda archive. | `string` | `"webhook.zip"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [microvm\_image\_arn](#output\_microvm\_image\_arn) | The MicroVM image ARN consumed by this runner configuration. | -| [webhook\_endpoint](#output\_webhook\_endpoint) | Webhook endpoint to configure on the GitHub App. | - diff --git a/examples/microvm/main.tf b/examples/microvm/main.tf deleted file mode 100644 index 0c3da60da0..0000000000 --- a/examples/microvm/main.tf +++ /dev/null @@ -1,113 +0,0 @@ -locals { - environment = coalesce(var.environment, "microvm") - aws_region = var.aws_region -} - -module "base" { - source = "../base" - - prefix = local.environment - aws_region = local.aws_region -} - -resource "random_id" "random" { - byte_length = 20 -} - -module "runners" { - source = "../../modules/multi-runner" - - aws_region = local.aws_region - prefix = local.environment - - experimental_features = ["multi-runner-v2"] - - global_config_github = { - app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - } - - global_config_lambda = { - artifact = { - s3 = { - bucket = var.lambda_artifact_bucket - } - } - } - - global_config_orchestration_provider = { - webhook = { - runner = { - ephemeral = true - jit_config_enabled = true - maximum_count = var.runners_maximum_count - boot_time_in_minutes = 5 - } - github = { - organization_runners = var.organization_runners - } - lambda = { - artifact = { - s3 = { - key = var.runners_lambda_s3_key - } - } - webhook = { - artifact = { - s3 = { - key = var.webhook_lambda_s3_key - } - } - } - } - } - } - - global_config_storage_provider = { - aws = { - ssm = { - paths = { - root = "/github-action-runners/${local.environment}" - } - } - } - } - - global_config_compute_provider = { - aws = { - microvm = { - image_arn = var.microvm_image_arn - image_version = var.microvm_image_version - ingress_network_connectors = var.ingress_network_connector_arns - egress_network_connectors = [var.egress_network_connector_arn] - } - } - } - - multi_runner_config = { - microvm = { - runner = { - os = "linux" - architecture = "arm64" - name_prefix = "microvm-" - extra_labels = ["microvm"] - } - orchestration_provider = { - webhook = { - matcherConfig = { - labelMatchers = [["self-hosted", "linux", "arm64", "microvm"]] - bidirectionalLabelMatch = true - } - } - } - compute_provider = { - aws = { - microvm = {} - } - } - } - } -} diff --git a/examples/microvm/outputs.tf b/examples/microvm/outputs.tf deleted file mode 100644 index 87ad4c924c..0000000000 --- a/examples/microvm/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "webhook_endpoint" { - description = "Webhook endpoint to configure on the GitHub App." - value = module.runners.webhook.endpoint -} - -output "microvm_image_arn" { - description = "The MicroVM image ARN consumed by this runner configuration." - value = var.microvm_image_arn -} diff --git a/examples/microvm/providers.tf b/examples/microvm/providers.tf deleted file mode 100644 index eca2fe96a7..0000000000 --- a/examples/microvm/providers.tf +++ /dev/null @@ -1,9 +0,0 @@ -provider "aws" { - region = local.aws_region - - default_tags { - tags = { - Example = local.environment - } - } -} diff --git a/examples/microvm/variables.tf b/examples/microvm/variables.tf deleted file mode 100644 index 4f8eaa3bc9..0000000000 --- a/examples/microvm/variables.tf +++ /dev/null @@ -1,72 +0,0 @@ -variable "aws_region" { - description = "AWS Region where the runner control plane and MicroVM resources are deployed." - type = string - default = "eu-west-1" -} - -variable "environment" { - description = "Name prefix for the example resources." - type = string - default = null -} - -variable "github_app" { - description = "GitHub for API usages." - - type = object({ - id = string - key_base64 = string - }) -} - - -variable "lambda_artifact_bucket" { - description = "S3 bucket containing the runner-control Lambda artifacts." - type = string -} - -variable "runners_lambda_s3_key" { - description = "S3 key for the runners Lambda archive." - type = string - default = "runners.zip" -} - -variable "webhook_lambda_s3_key" { - description = "S3 key for the webhook Lambda archive." - type = string - default = "webhook.zip" -} - -variable "microvm_image_arn" { - description = "Lambda MicroVM image ARN produced by the MicroVM image build." - type = string -} - -variable "microvm_image_version" { - description = "Optional immutable version of the Lambda MicroVM image." - type = string - default = null -} - -variable "egress_network_connector_arn" { - description = "Regional Lambda Network Connector ARN used by MicroVMs and the image build." - type = string -} - -variable "ingress_network_connector_arns" { - description = "Optional regional Lambda Network Connector ARNs exposed to MicroVMs." - type = list(string) - default = [] -} - -variable "organization_runners" { - description = "Register the MicroVM runners at organization scope when true." - type = bool - default = false -} - -variable "runners_maximum_count" { - description = "Maximum number of concurrent MicroVM runners." - type = number - default = 10 -} diff --git a/examples/microvm/versions.tf b/examples/microvm/versions.tf deleted file mode 100644 index 8ace4cfd44..0000000000 --- a/examples/microvm/versions.tf +++ /dev/null @@ -1,13 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 6.33" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - required_version = ">= 1.3.0" -} diff --git a/examples/multi-runner-v2/.terraform.lock.hcl b/examples/multi-runner-v2/.terraform.lock.hcl deleted file mode 100644 index 62c535d49e..0000000000 --- a/examples/multi-runner-v2/.terraform.lock.hcl +++ /dev/null @@ -1,93 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "6.64.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:2fTLxzUDmp/KVIHbIeLTB4bIzWHx8E6Dw+1ALLUi+Yw=", - "h1:wXARLY+IeQ7ufYxCLTPCwToWGMRvOpiOTfJS97iwUzI=", - "zh:07172315d67bc9781240272759cdfc7bd32b7e72384a56862c2c1da3cca99a81", - "zh:154ce7d2659de9a59ddfe96d7cab41a9ddc2cb267a7d4bcdf4e737ff2ffdec06", - "zh:17324d4335a7a7ac01cc23eded530775606680ff53b47cb74a3cb95d1121f836", - "zh:307ab92324ec5a61b124881ab8cac1d9e316f4527dfd0e1b59794c229407eb4e", - "zh:31e25f1903661332e36a95283042dd3ec50b47c186db00663fbd976a11e6a6b2", - "zh:3311d9f3bd12a24886027dbe73859dcd1e67bd0e3046227a338cf2c7ca04d18e", - "zh:37916156a3aac3b29be3acebd15d53145ea4ab5d4aaa825eaebe75481fa00500", - "zh:4158cb8c38b3ac6aa98eb15935ec6bd7c30838d85d2b00acc9812df8382ae908", - "zh:5bfb9499c66d9db5b34dc5c60f426a1ab1baa5457ce2aefebca826a9c3f92fb0", - "zh:6eb29ead5a4aca3b1f35812e7e8c75419180e1928e479b458f206861277736db", - "zh:7a82b6dd0c0cdef8045a4adfbddd36acb86b6b23fcbed8e189c2d71f7dc4a502", - "zh:9556bd792032c3f7e73ea4dd08cec88dc1327f5a4a57d79c30ba844ae2b9a3c0", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:c5234180464cb800c83a41f57462742b802c150ad7d4417626fcd9cb511c01d2", - "zh:cd776b83b1f7b36635957350afe7ce28ba4e4ea3a5e2deb00d13dbd3b35d9d40", - "zh:fb583a7b791c6f915b86573d04f05ddbf7f1a5e4120c5d8a7450a3086c1225c4", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:9rBZCMNpxKwMlRbWH2QpwD3kqUCAejdOZQ/aiiDObXQ=", - "h1:m24fjcInWvTVZ1XSo2MaNuKPe+X/gfG8SIi09rA7a7M=", - "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", - "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", - "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", - "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", - "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", - "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", - "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", - "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", - "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", - "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", - "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:TuxJq10DVnRP7c5HBZPyyvQGcckNVfijyU1eXEu5e4M=", - "h1:m5FqidbIgh+E9OigiZh8/xbkvpUQFSj3hZo/jqNLCLQ=", - "zh:08c59776542ea16e5a8545752787b17ff412922182b4cfabe16139197be8ac44", - "zh:123109cc7e5ed6d515787fbc212f2a3fd5e75647bb24ab7c801ccd4d4ed42451", - "zh:14b3fa4372754b54844b41d5dbd4671a292d8d6828b90169061feb4d7b15dd05", - "zh:56a4daaa3212f57b764bf3d1f333141c6610c5f21abb240e0111221f7c7fa4d4", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7e888a026dbacd2474a42264227ae35f639780f0f0c613529d10a95cd61988b3", - "zh:85a53646267e87d600df7124e4767ffde9bba3b6356d45d961618bdd68131cc7", - "zh:8ffa0e9c7c39b2ab0905b472465d6e35ef0b776b3f6273bb34c150340b61bff1", - "zh:9846510a1841530d4403f4818e233f91e3b3bade7441047599fbf800742f65be", - "zh:afa98d44860875f037c6def0a7e6ff208e042712ba771f620482b143cd336891", - "zh:bdca130d9ef27488ae0b13bc8fd8019e8bbdd4f2ceff29da066bd333165d68c5", - "zh:cb3b94cbca88210dd0d1f11e2b8a89333f48c3857faf8f70f589072ce7c28610", - "zh:f0c0ba87925fe32f84b80f7513b1efb1b0866f51f899ba825e95ad59ff09b018", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", - "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", - "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", - "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", - "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", - "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", - "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", - "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", - "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", - "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", - "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", - "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", - "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", - ] -} diff --git a/examples/multi-runner-v2/.terraform.lock.hcl.tofu b/examples/multi-runner-v2/.terraform.lock.hcl.tofu deleted file mode 100644 index e8fb9a5cf8..0000000000 --- a/examples/multi-runner-v2/.terraform.lock.hcl.tofu +++ /dev/null @@ -1,150 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "6.64.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:/G38+XhC1mBVkmeWdtk/wk7lX2BxviJ2XZ70dpoaKKQ=", - "h1:7BzHdGCBG5usqOIhfBq89dkdUopnSo+qe9qRCKDSRHc=", - "h1:8AgY9Hc5/R5j97WgCcDSlbuKk0pjk3vkp7oz4mtGVY8=", - "h1:DKdOy/0RfYLxpzAXBPWTO5Eusvqx5UoGxiq2J0E6DY4=", - "h1:KSwetpR4S2eUsKmHftt73Cbx72lPWYET4V+Ej05rnkI=", - "h1:MEi5Ecge1Uwx/DRGfdVDzV5Q/soRxKh6dBHxUjGdaDQ=", - "h1:VqjWicgPZW32+YnSe0Lo78qq8/24I8XNV+E9d/lBz/4=", - "h1:WBgbFHdg/3ekWoAH6UeKiwfk6iqLr1f7TX9R/mJUK8M=", - "h1:YisB3zMV5Kh6p5/eVuPAAPEmudD/UqGN4C/V3zRtAq4=", - "h1:bG5dXqR4mSlcebUG+anerOWYDyeaScZJeLSJk0cYBfE=", - "h1:iosW/imG2pc4La7qdeM/rK6ldMXhcU6YVW7tjqwNXtI=", - "h1:nKE1gnLZxIoqukQ1YI9EUdmrQIUeAN4PWb5ecN8U9K8=", - "h1:x0hJO5+On8FaKExr4p2cNJhWsNWFZq1EiDD6CfVwy2E=", - "h1:yPH75sRH+f3aJlJAloOL/BikeZV6/0GP8VQvnJoMRKM=", - "h1:zCWB5ZD98/ZC0a50HTGoC/fTAseh189xxCFEL5Mt7r4=", - "zh:06e09ced9480ae12578122f7a25758a15d8fe684da0f6a0a61b9bc2f4a4918ad", - "zh:2035805f0ed8bf81d493e7a52f22965b3d5d402687a95d1caa8c4b1b348c1264", - "zh:25fe72a3d6a330eab6c8957f9e6bdf297ffdce95fa059fef30b80da764bee6b2", - "zh:49df644d19e39b9947e84609260028687057191ddd941783c0211386ade53040", - "zh:4d8438a5d25f18eb376c8375c70f81afb79d0fc1e63ebb6df1d0e02287964dde", - "zh:5cd9717e819506132126a896e959cd4cf1bb213c033c37777c9d01a593937e2c", - "zh:6955caa4f435373ae870de31bdda85e51c60b68c51a4206df5a21b853bcefe21", - "zh:82a413500c35241745e097797610d2bff57c26e29f34ca711182fdde5c265d13", - "zh:831f78acce42a759a977769b0409387ec13ff64b4f46c24eb7e7662e0f352525", - "zh:88648a159119a0435bf86c6cd1f7482dc43dfa2eb742f9b29053da1ee9fdabd8", - "zh:9fc745d71a2e36dbdae0ee69be70675509e5a9dec1a3c5a9be6007e568d78c07", - "zh:bf6d11d6ed1655f61f70eb2906e5d1f7ff5e78b6539dcfb3116ed6f8960c9e20", - "zh:ca17a6ca363afe930ad3474966d39cb549b7f1e5efdca909972dd26f90eefc89", - "zh:d7e9cc87ada1314e6d8ecc5849385a8f8f45757bd2c145b8657f015c65e5078d", - "zh:eddb4d6d86700788d132ba2a83d306646ccb2a3a0cec0a0ab3e215307c61d8f2", - ] -} - -provider "registry.opentofu.org/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=", - "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=", - "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=", - "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=", - "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=", - "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=", - "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=", - "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=", - "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=", - "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=", - "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=", - "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=", - "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=", - "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=", - "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=", - "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b", - "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1", - "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d", - "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252", - "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542", - "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586", - "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5", - "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84", - "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a", - "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d", - "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07", - "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703", - "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29", - "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5", - "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09", - ] -} - -provider "registry.opentofu.org/hashicorp/null" { - version = "3.3.2" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:1T+00cjQNmRAHAz9xjEBFpf5wRRb0IBuXS/W8ke5BWs=", - "h1:46gmIYe+klib6TlHKSqEkMLjvnzVWiCB2NYA2zR8MX8=", - "h1:7WQ3wjfaeqnXxq+a8cYiYeWUnMTgY1JcuX+z7sZd72s=", - "h1:MVM+vkVtW/YyKfn111pyho0y87I4TekaNMbBLkn0/C8=", - "h1:QBcIbI2Dp4v6Iui37pn4qmw8YeiFLbSWcJuzZVl/65Y=", - "h1:SsVKTUR+vgLaC1YnoDa2fnYpzREcgNgWRcu5x+vwjHA=", - "h1:WUaeuTNn9w6UXZ9cMq4+qZy4ZAr71B9NcUDEXjqfdKs=", - "h1:WtEaA7alasNwEQ4L3+KyQtbkSOPsexzJ4LUZ7PKaycI=", - "h1:WxS7rjYIZ1WQc4GkICch8XrbxoSY8TjUfLbPDo6oEcQ=", - "h1:Ysvc/FPvcwk+iMg7IcLkqZhT/KhtZTYji+UBqMlcTs4=", - "h1:ZLjbXnfVcRvS/DAN3BcNebOsnOcs3Nx6mJpFCj4dZ2c=", - "h1:fAmvQjIGyqdGMc+v/fUEINCyuU4iaKSzsV8PWsOnmAc=", - "h1:jwkbEtf3S7W+Bl4soynNkUHFfK/4I/H74urHY758XVw=", - "h1:qY1sKzlxNTp/dqZR23bM4egmVMRlaQudLlBYraMt1pw=", - "h1:t8H1KNwJQwKE/GqpHeRxOWgMk0Yv35qbBTBzqB/rhr0=", - "zh:09e94b0b7dfc0c6450c247517b5410546039c758e513d89b588af6df70c3d57d", - "zh:0b72497b6fd79a2b04785b64890a565a8cc7b06ded95da05e6dab2f3b8a02d58", - "zh:1c0ee6f81f7bcdec8d568a145a450eb57a6f1cfe5e48943375d1af22ed54151e", - "zh:1c6899b475f035d352af1e7f33dc30beab8b8e3784f8cb55a2cc4a11997fbd66", - "zh:43e57a2a56e9874604501bdebe431bb573fb77d2c5f4d7598ab30727dc0e90ea", - "zh:49cf2f36298a5ac3ac8d80ceb87466e6a99d2c021005bcd9e3a79f2314fd0a13", - "zh:665be40d2c7f3d768b8f39a041371526d4b7b396b4c11e162a1886212da176c9", - "zh:6bb1583d88ddb38b1c6b4624e25ad414ddd8bf65b0dd9c074580847311f83924", - "zh:71d64453bdc795667e9841d7c90e3fef6ff157e0598d51dac4bb1e4583b85407", - "zh:73ac02bc3b680e1ea75aab24ec2a359c8f0a021f73d43b2feae0a899e75a93ad", - "zh:b2b777ee07b910e7345df85321fab6c9a25c33ecb56129758dda8fadaba09fe4", - "zh:bb984d52880749a49e509e3b243804869e1af40ee2d34322a30f5f340f8d8dbd", - "zh:e0724bc083527343b4a4099fd4f95511e49a0e113416cdba58a64446742b68b1", - "zh:e391c14e367cd64d986ddc8d81f2db76d49600ce0521720618ff1ecc0decde7f", - "zh:e95c1af8e8e9967d678cfa7c77ca229c863a68f7c9a85318bf08f26628afe338", - ] -} - -provider "registry.opentofu.org/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", - "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", - "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", - "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", - "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", - "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", - "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", - "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", - "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", - "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", - "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", - "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", - "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", - "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", - "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", - "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", - "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", - "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", - "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", - "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", - "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", - "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", - "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", - "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", - "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", - "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", - "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", - "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", - "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", - "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", - ] -} diff --git a/examples/multi-runner-v2/README.md b/examples/multi-runner-v2/README.md deleted file mode 100644 index 2755c8fcf5..0000000000 --- a/examples/multi-runner-v2/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Multi-runner v2 example - -This example demonstrates the experimental multi-runner v2 interface. Shared -defaults are configured with `global_config*` variables, while -each runner lane uses `multi_runner_config` for its matcher, -runner lifecycle, and compute-provider settings. - -The example creates three lanes from one deployment: - -- Linux ARM64 Amazon Linux runners. -- Ephemeral Linux x64 Amazon Linux runners with job retry enabled. -- Windows x64 Server Core 2022 runners. - -The v2 interface keeps provider-owned settings inside the selected provider -configuration. For example, VPC and subnet settings are under -`global_config_compute_provider.aws.ec2`, while the per-lane -instance types and AMI configuration are under each lane's compute provider -block. The optional `ami` variable can provide per-lane AMI filters and owners, -which is useful for test environments with locally registered images. - -Configure the GitHub App variables before applying: - -```bash -terraform init -terraform apply \ - -var='github_app={id="123456",key_base64="..."}' -``` - -The `github_app` value is sensitive and should be supplied through a secure -variable source in real deployments rather than committed to configuration. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.6 | -| [aws](#requirement\_aws) | >= 6.33 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [random](#provider\_random) | 3.9.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [base](#module\_base) | ../base | n/a | -| [runners](#module\_runners) | ../../modules/multi-runner | n/a | -| [webhook\_github\_app](#module\_webhook\_github\_app) | ../../modules/webhook-github-app | n/a | - -## Resources - -| Name | Type | -|------|------| -| [random_id.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | -| [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | -| [github\_app](#input\_github\_app) | GitHub App ID and base64-encoded private key. | 
object({
    id         = string
    key_base64 = string
  })
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | -| [webhook\_secret](#output\_webhook\_secret) | n/a | - diff --git a/examples/multi-runner-v2/main.tf b/examples/multi-runner-v2/main.tf deleted file mode 100644 index 4f63974f11..0000000000 --- a/examples/multi-runner-v2/main.tf +++ /dev/null @@ -1,180 +0,0 @@ -locals { - environment = var.environment != null ? var.environment : "multi-runner-v2" - aws_region = var.aws_region -} - -resource "random_id" "random" { - byte_length = 20 -} - -module "base" { - source = "../base" - - prefix = local.environment - aws_region = local.aws_region -} - -module "runners" { - source = "../../modules/multi-runner" - - prefix = local.environment - aws_region = local.aws_region - - experimental_features = ["multi-runner-v2"] - - global_config = { - tags = { - Example = local.environment - Project = "ProjectX" - } - runner = { - os = "linux" - architecture = "x64" - extra_labels = ["v2"] - } - } - - global_config_github = { - app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - } - - global_config_lambda = { - architecture = "arm64" - } - - global_config_orchestration_provider = { - webhook = { - eventbridge = { - enabled = true - accept_events = ["workflow_job"] - } - } - } - - global_config_compute_provider = { - aws = { - ec2 = { - vpc_id = module.base.vpc.vpc_id - subnet_ids = module.base.vpc.private_subnets - ssm_enabled = true - runner_binaries = { - enabled = true - } - } - } - } - - multi_runner_config = { - linux-arm64 = { - runner = { - architecture = "arm64" - name_prefix = "amazon-arm64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "linux", "arm64", "amazon"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["t4g.large", "c6g.large"] - ami = lookup(var.ami, "linux-arm64", null) - } - } - } - } - - linux-x64 = { - runner = { - name_prefix = "amazon-x64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - ephemeral = true - maximum_count = 1 - } - matcherConfig = { - labelMatchers = [["self-hosted", "linux", "x64", "amazon"]] - exactMatch = false - priority = 1 - } - queue = { - delay_webhook_event = 0 - } - job_retry = { - enabled = true - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5a.large", "m5ad.large"] - ami = lookup(var.ami, "linux-x64", null) - } - } - } - } - - windows-x64 = { - runner = { - os = "windows" - name_prefix = "windows-x64-" - } - orchestration_provider = { - webhook = { - runner = { - boot_time_in_minutes = 20 - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "windows", "x64", "servercore-2022"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5.large", "c5.large"] - ami = lookup(var.ami, "windows-x64", { - filter = { - name = ["Windows_Server-2022-English-Full-ECS_Optimized-*"] - state = ["available"] - } - owners = ["amazon"] - id_ssm_parameter = null - kms_key = null - }) - } - } - } - } - } -} - -module "webhook_github_app" { - source = "../../modules/webhook-github-app" - depends_on = [module.runners] - - github_app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - webhook_endpoint = module.runners.webhook.endpoint -} diff --git a/examples/multi-runner-v2/providers.tf b/examples/multi-runner-v2/providers.tf deleted file mode 100644 index eca2fe96a7..0000000000 --- a/examples/multi-runner-v2/providers.tf +++ /dev/null @@ -1,9 +0,0 @@ -provider "aws" { - region = local.aws_region - - default_tags { - tags = { - Example = local.environment - } - } -} diff --git a/examples/multi-runner-v2/variables.tf b/examples/multi-runner-v2/variables.tf deleted file mode 100644 index fe104758b9..0000000000 --- a/examples/multi-runner-v2/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -variable "github_app" { - description = "GitHub App ID and base64-encoded private key." - - type = object({ - id = string - key_base64 = string - }) - sensitive = true -} - -variable "environment" { - description = "Environment name, used as prefix." - - type = string - default = null -} - -variable "aws_region" { - description = "AWS region to deploy to." - - type = string - default = "eu-west-1" -} - -variable "ami" { - description = "Optional AMI configuration keyed by runner lane." - - type = map(object({ - filter = optional(map(list(string)), { state = ["available"] }) - owners = optional(list(string), ["amazon"]) - id_ssm_parameter = optional(object({ - arn = string - }), null) - kms_key = optional(object({ - arn = string - }), null) - })) - default = {} -} diff --git a/examples/microvm/.terraform.lock.hcl b/examples/multi-runner-webhook/.terraform.lock.hcl similarity index 75% rename from examples/microvm/.terraform.lock.hcl rename to examples/multi-runner-webhook/.terraform.lock.hcl index 252cc1596b..c1f4433ff3 100644 --- a/examples/microvm/.terraform.lock.hcl +++ b/examples/multi-runner-webhook/.terraform.lock.hcl @@ -3,7 +3,7 @@ provider "registry.terraform.io/hashicorp/aws" { version = "6.66.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" + constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0, >= 6.61.0" hashes = [ "h1:OnLj4nhqJnEcUzyyRKUjp1FgWG00Y8maikJEYSf9Zjw=", "h1:hBEaeBm9nm7A/u1nnD0nfolTPP55/BoKRFWk8zG8/fk=", @@ -69,3 +69,25 @@ provider "registry.terraform.io/hashicorp/random" { "zh:8c2b8c6a7ccdec16b73e2fb9f3700ea097f58c592571e4c5de60c93d2301732c", ] } + +provider "registry.terraform.io/hashicorp/time" { + version = "0.14.2" + constraints = ">= 0.13.0" + hashes = [ + "h1:eQRXh8mZFlUJfzYXKdaYRHRMhiS2cFyCfgP1mjkrtuI=", + "h1:gnP2hptiFIHSHUFBvAFKhE/Yh5u5yVEx+P7XSB58A/E=", + "zh:0aa1028d91041f4dceba193e3707dac57358d0063d97e20700e554758b67baca", + "zh:32bee9f2b2678e2a0789ad86e716d09ca1d5450180b3cd8033ee7a251bfd352e", + "zh:3aded9ef4dc6f4aec202a50c68a08b40013d325f9947f10168ebc8bee54109fc", + "zh:4d924637f3115ffa4ffc7f16d3f366bc472594f7447d400adb9def7ac92e3fc8", + "zh:5c35008e1363deafaa440ab43519409866dd7ec72aabeb2317bc16cd82756784", + "zh:6b30d97c9827501d7010fe49c8889d7a6cc8b45b77cdb1668c65af7f28a27d73", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:851e0f0e9c4de395e42220de51e7fb20d0e845e629f3cf37056da19e597304e7", + "zh:85622b4779b3ba7424780f7f8efcfa585227e245201cdb68303f2118b388e971", + "zh:9039153e3d45147183804188a1ce36c3811db9a4dca80f36fa382408d5f50b72", + "zh:a3d385413dc258a53fe8f4ded5d1fcd1eba715c65a015d74e70911bab2919207", + "zh:b9310a2327f7c8ad2aa3266f89c4d026b4bf18b09f7c4257c0ecd71a32f32db3", + "zh:cee7f2143da0c494115da94984bf3630e3e5855e27dae720a7bc29bb6acd9be2", + ] +} diff --git a/examples/multi-runner-webhook/.terraform.lock.hcl.tofu b/examples/multi-runner-webhook/.terraform.lock.hcl.tofu new file mode 100644 index 0000000000..8737446421 --- /dev/null +++ b/examples/multi-runner-webhook/.terraform.lock.hcl.tofu @@ -0,0 +1,150 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "6.65.0" + constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0, >= 6.61.0" + hashes = [ + "h1:/D/KChJMHHi6N2Ae8pDT2CoZ1ZVPgmO4DvT3TM1uYdA=", + "h1:0KBMNN4G86DISLGy8e7PdtgcjgLWmFM0Tu/+PlcI6Xc=", + "h1:0xke8tvUJrFES4mVvTaPvBQAad6XfV1kOE7p8i+xN+4=", + "h1:1Ra6ZrgNEnjkReeeNpOEml91kbhCHwoSzV/ZI3yVeNw=", + "h1:1jrxTTKLGfTEPR9BZ6hkGMYcK0ph5bx1dzhSSYtGgsQ=", + "h1:AYFFxtquEyqmRHuAEPx1mOnNzIEGPh5WvKIY+u4HSoA=", + "h1:CXFxyNowi2BGGcMxHPk6+X5wuo7ZAwuK2/OGX9wRve8=", + "h1:UTMijAbC6R9HYu8x6QoQjOKP5EhDB2885v+GjVbOfuk=", + "h1:UUQbMjGJufv6KVfAN1uMVoDkJJV7gL5+qXt8pA82W2w=", + "h1:XOuZUW+/aP5FeEMd25136uGu4Yd3scANUQ3khIePxpw=", + "h1:XkTODDfiyRIzHYCSQX/TxT5A28OJpimgbJvddYSVl0M=", + "h1:bJ3Hx/OsgpaMGLAyu7U3x+IJOOEpynsrp5SAa68/xdQ=", + "h1:e8xYPOcVYj/ke4NwRyOTdy2AH/g1n6I6STENon69LvM=", + "h1:pYM2NuBZ9Yml+TdLZfuTDSjpRR2ZDW2lgljMrSYhbS4=", + "h1:q041n/UFMg+1rQ8ydIoVaHSOYE7qzV/dnTii/eQl2HA=", + "zh:079c8d8adae825fcd81180979a7c87f66eb4c18825dde91790eac178ffa5b506", + "zh:30912d1497c8a5dfe2d26eeab68b1ceb05621a0cc5205a9623fc048c2a34987d", + "zh:33cbbc1fa2df3c80ca026b4073cff1013fd5e648b32ae40d2e2dd7a8892cd6f4", + "zh:6ba73e9aff1762c2ec9f3d4184b49ae7c219bc302fe38aa9c47a3267617c7c7d", + "zh:8581d0ae4ab4bb14b24fe3d4145900cbcbb373271fa327da10f107bff947ee62", + "zh:8d4528d906ebba03c857d2a30cfd75d3f847ff6c1781a57b535fd0edd659f7db", + "zh:9617db8d86e5ec3c4be76c163d3fd0de013cbd804b337368e21d7916aac686bf", + "zh:bbd497e2859a5a09962b1209529a07c8d921493758cd6c8d01200b2ec1f6eba7", + "zh:c446d97456cc9e8adf2a409e0f4ad8ab5c4a54d6b36e3d985a52f6ea7c2603e7", + "zh:c8b005e981e6e8fbb01e9ea2281df977401c236da24e676d98e8c46c2042463b", + "zh:d3c2ecb7647f865b17e93ae6f44fedd50e3d3aa1e7d67336731dc38d6b46a9de", + "zh:d4b0a5ab8625c787e9e706eaa3843a104ac4edaadd4cc46c3f13490c2003fd69", + "zh:da81af555be23b26b6f82352b29bdb904456429b3752b5a574fb7e636a62784b", + "zh:e4d7efff69897583bcf7451f631ae027da528f9355ff72339a5460837856e41b", + "zh:e76114d3e0b20893ce22bd50d7c813842f0747570101e4389b8c7e57cf2a1019", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.3.2" + constraints = "~> 3.0, ~> 3.2" + hashes = [ + "h1:1T+00cjQNmRAHAz9xjEBFpf5wRRb0IBuXS/W8ke5BWs=", + "h1:46gmIYe+klib6TlHKSqEkMLjvnzVWiCB2NYA2zR8MX8=", + "h1:7WQ3wjfaeqnXxq+a8cYiYeWUnMTgY1JcuX+z7sZd72s=", + "h1:MVM+vkVtW/YyKfn111pyho0y87I4TekaNMbBLkn0/C8=", + "h1:QBcIbI2Dp4v6Iui37pn4qmw8YeiFLbSWcJuzZVl/65Y=", + "h1:SsVKTUR+vgLaC1YnoDa2fnYpzREcgNgWRcu5x+vwjHA=", + "h1:WUaeuTNn9w6UXZ9cMq4+qZy4ZAr71B9NcUDEXjqfdKs=", + "h1:WtEaA7alasNwEQ4L3+KyQtbkSOPsexzJ4LUZ7PKaycI=", + "h1:WxS7rjYIZ1WQc4GkICch8XrbxoSY8TjUfLbPDo6oEcQ=", + "h1:Ysvc/FPvcwk+iMg7IcLkqZhT/KhtZTYji+UBqMlcTs4=", + "h1:ZLjbXnfVcRvS/DAN3BcNebOsnOcs3Nx6mJpFCj4dZ2c=", + "h1:fAmvQjIGyqdGMc+v/fUEINCyuU4iaKSzsV8PWsOnmAc=", + "h1:jwkbEtf3S7W+Bl4soynNkUHFfK/4I/H74urHY758XVw=", + "h1:qY1sKzlxNTp/dqZR23bM4egmVMRlaQudLlBYraMt1pw=", + "h1:t8H1KNwJQwKE/GqpHeRxOWgMk0Yv35qbBTBzqB/rhr0=", + "zh:09e94b0b7dfc0c6450c247517b5410546039c758e513d89b588af6df70c3d57d", + "zh:0b72497b6fd79a2b04785b64890a565a8cc7b06ded95da05e6dab2f3b8a02d58", + "zh:1c0ee6f81f7bcdec8d568a145a450eb57a6f1cfe5e48943375d1af22ed54151e", + "zh:1c6899b475f035d352af1e7f33dc30beab8b8e3784f8cb55a2cc4a11997fbd66", + "zh:43e57a2a56e9874604501bdebe431bb573fb77d2c5f4d7598ab30727dc0e90ea", + "zh:49cf2f36298a5ac3ac8d80ceb87466e6a99d2c021005bcd9e3a79f2314fd0a13", + "zh:665be40d2c7f3d768b8f39a041371526d4b7b396b4c11e162a1886212da176c9", + "zh:6bb1583d88ddb38b1c6b4624e25ad414ddd8bf65b0dd9c074580847311f83924", + "zh:71d64453bdc795667e9841d7c90e3fef6ff157e0598d51dac4bb1e4583b85407", + "zh:73ac02bc3b680e1ea75aab24ec2a359c8f0a021f73d43b2feae0a899e75a93ad", + "zh:b2b777ee07b910e7345df85321fab6c9a25c33ecb56129758dda8fadaba09fe4", + "zh:bb984d52880749a49e509e3b243804869e1af40ee2d34322a30f5f340f8d8dbd", + "zh:e0724bc083527343b4a4099fd4f95511e49a0e113416cdba58a64446742b68b1", + "zh:e391c14e367cd64d986ddc8d81f2db76d49600ce0521720618ff1ecc0decde7f", + "zh:e95c1af8e8e9967d678cfa7c77ca229c863a68f7c9a85318bf08f26628afe338", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.9.1" + constraints = "~> 3.0" + hashes = [ + "h1:38E2VQmQDhws/3AL3D/EzBGuCseepyZRIswAOx8CqoQ=", + "h1:7+qv9kpOpBC9EUPCubnPxh603tu3l9EIMMBkpbt1H1Y=", + "h1:CEQeHfnUDB3uqAkKoEWfWgbj+kpoQHgcuPbAjPzbh+U=", + "h1:HPYO9tf8KUSHqSdz1uOL97MLeaVHPaWPY1JW6tKU19E=", + "h1:KYXiC06Pr3WJcIUbDq9MgdAbInO5zcRyHFqV1x5UcJg=", + "h1:MygjbYH8CrPv8RUe75tZAFmrFNIzQLT45fiyYx7u2tI=", + "h1:RMSARNOw4qZx+VmHYnVMGljsFVfCV1P+nJjKrN2XIOI=", + "h1:U/71jbSbfsfVLxWpSlhyVHh/DnQXQhjoJCGkyongkBA=", + "h1:WwLLvRE1q95CTGxyTjKpctXJ0ooVs9d22JAQS9fC3uo=", + "h1:ZtRBSqoyfQAhngjUjM0NRPtj6NdSJ/JBENFwT8D276s=", + "h1:cfxedZLduhHD1UtqQDjAQZNAEhr/bWDAZ4nU9rSdySg=", + "h1:i45mo4de0QKOreStMqUQ7qyZL3ucFq42l178Fm5/hMU=", + "h1:v3SAJKN4D3dOM95xKwgKGIWELe5nUBbyXdb5HNz7icw=", + "h1:v3vTk/STekrzNc6NG3jL9/05zhvF1QVCgyRRbvSHPcI=", + "h1:zHgFWtRBgOycqhw8HdLSvMVNWGJtIjOAPYOcAdE7cL0=", + "zh:09aaf19b0d22726d2378e0e89fbbefc183494d7bd585759d6c4e69ba50951a2f", + "zh:31575ca9bc0db20337096d178ea73bce3ebca343ed071c67f78cf39f800c9ec6", + "zh:624fb6ed552abc34a5aaac41e76a373da65ac08e524b09b672f29c60e6ac896a", + "zh:6a4760d55132b9750ac1a04f6fc32e247034daa999f71452dba9cbca225a529a", + "zh:768a6047cfb8958e7b0b120c580aa3de6624a7fbb2c56ad6df85cd559ed26ec7", + "zh:8983c788ba660bcb587e64ff9c3e4323515caf78facbe0abe6432e7aff8df893", + "zh:8d570eb026a4f00b58a1d36be0ce3c13adf4d973efcd4162b05cb295bbc14257", + "zh:a2259540854d5f699c36b89244fb202ebb2c219b64669a51072687d04fb47152", + "zh:aaa51d905b0e80a28e02f9bee2cf6c91ffade7389d77ab9198aa12809ed04955", + "zh:afb60995e98573facddfb47baedf7e288408680eb00b5d3df570611758947c72", + "zh:b9a46d852ce53fa037f47537a7de53f37b759ccf211600b7ba44c66ba4b616b7", + "zh:bafcfeeefcd0dfefeff120b655b45edb0497c4717534ffe5201b3cb556d1ffe6", + "zh:c3ac24d397eae054aca2290e20943e0c767592cc661c890850c25ac01829308d", + "zh:eafba4127ebadcc5ed0e427935c66fb5e2da7cfdaae39a66d52f4a50d51faf1e", + "zh:f39d4bce213ed9bba3474bad468136af08ff6c4c33adaafcc10c1f78067adfe3", + ] +} + +provider "registry.opentofu.org/hashicorp/time" { + version = "0.14.2" + constraints = ">= 0.13.0" + hashes = [ + "h1:0lkmuDlyUBEK2vAxb9r8jY8kMpqYbMoSb4GWnSbA9iY=", + "h1:4ccOXW03+ENKJieXGwTfMvRlkpT9o+ra6dw240C1UFE=", + "h1:8+b7rm7aVI0cNDoegUPuEKNAQeEC9jYHaaqgVWGvGag=", + "h1:BzLQYmKbF3aM81kS9GZQ0mnJPU/bVFa3Jgk9ZIFJP80=", + "h1:EHyMvebIwqieUVzq6WbbheNWUHRgy8B5hhRv++V10qA=", + "h1:ICdTbU+IeBH2xihtoYENxmLIGfObCbcb8l2XK2abJGw=", + "h1:JKIAzWzVxRY6Q+ybCjmZ6DnMfkyp/zdZmWkTIB1JzOo=", + "h1:JkTAWz5bbrSgrnkkF5XhoMbSLSihtDDb5V+SYAZYOes=", + "h1:TNzAoSy5lcv/8pjzlb4nz+m92N3G/osbU+FV+uh0SQs=", + "h1:XJLK9UX0/LxUM7Z4pxB1tf6TP8iCz0bxM87ot4wq1ns=", + "h1:bkdzFk//GNj0iHkXgupa0XqwNjYx3O6+czgIi+IkXjA=", + "h1:cbpg0fadPwbhtL0EMnPj+AkwpPUvflGGeYB31SKTHdw=", + "h1:hv6Fp4zk1JQ7Dj+cmUJZmuI3aLEGjw4VSFyqgUvisDg=", + "h1:oMHgtUYEDs1DJrTXNPgSNoQ0f+0FZ+nAYJWJYpHBtAI=", + "h1:qdSn+kIg2bcZkgGLJ5zQ1K03qoRWYYBxGv6J7SWlShY=", + "zh:0c5caf61f978612c78eead85fac256ce42a62e5a04616afedf41065a98639c9f", + "zh:1356743176e6522d6f0a998cf04edc9c5b3e3a99893562fc3a9bc7c4f7ac738e", + "zh:1aae0a419092d5e20d6c38199b6c406a73abe12f7f665fa1bd9fed07f454d427", + "zh:4fc26faa672af3806d5760c88c7ad223ae8a04002ae77372d7744fa44e56b3f6", + "zh:503189a08d6468a1c24099f3aa9b22aa9f6fd82bdab118ee6a2af665441fc8a7", + "zh:53f1ca144377f8c42a1fc71cff22e6c3ad5e2ae9d9eaaf0a95cbe70c896b30cb", + "zh:64ac1844933a18767bb1b9ea19b5432f83acf682fcff1e49680e50d341420e78", + "zh:6819c08b9228375f56fd2d33881fb7fbb0a59581acabfcd662b4eb002aae3917", + "zh:c7e0df3698ad0f75114ce72049d8d8be242e6510806e13bdf8ca8f416e3d9f1c", + "zh:c8a7d8601dcc700efdfc65194bb8450ca28f04c0e42aac8a355a7b758c82e252", + "zh:d311cd2ef10b5b9762f248e54c9d9211c8f6c24a5efc80fcddeba95d0347713d", + "zh:d34865e994a91b764bd1283dfa25ee3d84571d65959987d4a7f17405dcf27e43", + "zh:ee5ff12288f160969dc6a2764b634295cfe73f5705172f0d2f5d32a358cf29f7", + "zh:eec9f3327f2cd180af6fa5e16370c7d5cf0c79874f03d2c1a48e22c7c1af4951", + "zh:f0fc458693aec12dcf84bb7d7b2f49283a818282072729a26765d679fb2984ed", + ] +} diff --git a/examples/multi-runner-webhook/README.md b/examples/multi-runner-webhook/README.md new file mode 100644 index 0000000000..1d63205c7c --- /dev/null +++ b/examples/multi-runner-webhook/README.md @@ -0,0 +1,100 @@ +# Multi-runner webhook example + +This example exercises the shared experimental multi-runner v2 webhook path +with EC2 and Lambda MicroVM compute. The runner lanes, webhook orchestration, +Lambda artifacts, and GitHub configuration are common; provider-owned inputs +are grouped under `compute_provider`. + +The example creates both an EC2 lane and a Lambda MicroVM lane behind the same +webhook endpoint. The MiniStack smoke test sends matching jobs to each lane in +sequence, so adding another provider means adding another lane and provider +specific lifecycle assertions to the same deployment. + +The runner-control and webhook Lambda archives are explicit inputs: + +```sh +terraform apply \ + -var='runners_lambda_zip=/path/to/runners.zip' \ + -var='webhook_lambda_zip=/path/to/webhook.zip' +``` + +## MicroVM prerequisites + +The MicroVM lane expects an image that has already been built and published in +the target Region. The image is not created by this example. Prepare it in +this order: + +1. Apply `examples/microvm-foundation`. +2. Build and release the lifecycle-hook service from + `lambdas/services/microvm-lifecycle-hooks` using the same artifact process + used for the repository's Lambda services. +3. Build the image with Packer from `images/microvm-ubuntu`, passing the + foundation's bucket, connector, build-role, and lifecycle-hook artifact. +4. Set `compute_provider.aws.microvm.image_arn` (and, when applicable, + `image_version`) to the published image. + +The foundation's build role is used to create the image. It is different from +the execution role used by the runner job. The runner configuration owns that +execution role; the control-plane TypeScript passes it to `RunMicrovm` when it +starts an ephemeral runner. The control-plane Lambda therefore needs +permission to pass the configured execution role, and the role needs the +runtime permissions required by the selected runner lane. + +This example deploys both EC2 and MicroVM lanes behind one webhook endpoint, +but it does not replace the foundation, image build, lifecycle-hook release, +or execution-role setup steps. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.6 | +| [aws](#requirement\_aws) | >= 6.33 | +| [null](#requirement\_null) | ~> 3.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | 6.66.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [base](#module\_base) | ../base | n/a | +| [microvm\_foundation](#module\_microvm\_foundation) | ../../modules/microvm-foundation | n/a | +| [runners](#module\_runners) | ../../modules/multi-runner | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_log_group.aws_cloudwatch_log_group_microvm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_ecr_repository.base_ubuntu24](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository) | resource | +| [aws_ecr_repository_policy.repository_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository_policy) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.ecr_repository_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_region](#input\_aws\_region) | AWS Region where the runner control plane and compute provider resources are deployed. | `string` | `"eu-west-1"` | no | +| [compute\_provider](#input\_compute\_provider) | Provider-specific settings for the EC2 and MicroVM runner lanes. | 
object({
    aws = object({
      ec2 = object({
        instance_types = list(string)
        ami = object({
          filter = optional(map(list(string)), { state = ["available"] })
          owners = optional(list(string), ["amazon"])
          id_ssm_parameter = optional(object({
            arn = string
          }), null)
          kms_key = optional(object({
            arn = string
          }), null)
        })
      })
      microvm = object({
        image_arn                  = string
        image_version              = optional(string, null)
        ingress_network_connectors = optional(list(string), [])
        egress_network_connectors  = list(string)
      })
    })
  })
| n/a | yes | +| [environment](#input\_environment) | Name prefix for the example resources. | `string` | n/a | yes | +| [github\_app](#input\_github\_app) | GitHub App credentials used by the webhook orchestration provider. | 
object({
    id             = string
    key_base64     = string
    webhook_secret = string
  })
| n/a | yes | +| [github\_enterprise\_server](#input\_github\_enterprise\_server) | Optional GitHub Enterprise Server endpoint used by the smoke-test API mock. | 
object({
    url        = string
    ssl_verify = bool
  })
| `null` | no | +| [runners\_lambda\_zip](#input\_runners\_lambda\_zip) | Local ZIP file for the runner-control Lambda. | `string` | n/a | yes | +| [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | Local ZIP file for the webhook Lambda. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [microvm](#output\_microvm) | n/a | +| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | +| [webhook\_secret](#output\_webhook\_secret) | n/a | + diff --git a/examples/multi-runner-webhook/main.tf b/examples/multi-runner-webhook/main.tf new file mode 100644 index 0000000000..5b1b8b94c6 --- /dev/null +++ b/examples/multi-runner-webhook/main.tf @@ -0,0 +1,183 @@ +module "base" { + source = "../base" + + prefix = var.environment + aws_region = var.aws_region +} + +module "runners" { + source = "../../modules/multi-runner" + + prefix = var.environment + aws_region = var.aws_region + + experimental_features = ["multi-runner-v2"] + + global_config = { + tags = { + Example = var.environment + Project = "MiniStack" + } + runner = { + os = "linux" + architecture = "x64" + } + } + + global_config_github = { + app = { + key_base64 = var.github_app.key_base64 + id = var.github_app.id + webhook_secret = var.github_app.webhook_secret + } + enterprise_server = var.github_enterprise_server + } + + global_config_lambda = { + architecture = "x86_64" + } + + global_config_observability = { + logs = { + level = "debug" + } + } + + global_config_orchestration_provider = { + webhook = { + runner = { + ephemeral = true + jit_config_enabled = true + # The smoke test keeps the ephemeral resources alive until each provider's + # scale-down phase, so it needs capacity for standard, dynamic, and pool runners. + maximum_count = 3 + boot_time_in_minutes = 0 + } + lambda = { + artifact = { + zip = var.runners_lambda_zip + } + scale = { + up = { + job_queued_check_enabled = true + } + down = { + # Smoke scenarios create and remove runners immediately; do not + # wait for the Linux five-minute minimum runtime before checking + # the GitHub runner state. + minimum_running_time_in_minutes = 0 + } + } + pool = { + config = [{ + schedule_expression = "cron(0 0 1 1 ? 2099)" + schedule_expression_timezone = "UTC" + size = 1 + }] + runner_owner = "test-owner" + } + webhook = { + artifact = { + zip = var.webhook_lambda_zip + } + } + } + } + } + + global_config_storage_provider = { + aws = { + ssm = { + paths = { + root = "/github-action-runners/${var.environment}" + } + } + } + } + + global_config_compute_provider = { + aws = { + ec2 = { + vpc_id = module.base.vpc.vpc_id + subnet_ids = module.base.vpc.private_subnets + ssm_enabled = true + binaries_syncer = { + enabled = false + } + } + microvm = { + image_arn = var.compute_provider.aws.microvm.image_arn + image_version = var.compute_provider.aws.microvm.image_version + ingress_network_connectors = var.compute_provider.aws.microvm.ingress_network_connectors + egress_network_connectors = var.compute_provider.aws.microvm.egress_network_connectors + } + } + } + + multi_runner_config = { + ec2 = { + runner = { + os = "linux" + architecture = "x64" + name_prefix = "ec2-" + extra_labels = ["ec2"] + } + orchestration_provider = { + webhook = { + github = { + organization_runners = true + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "x64", "ec2"]] + bidirectionalLabelMatch = true + dynamic_labels_enabled = true + awsDynamicLabelsPolicy = { + restricted_keys = { + "instance-type" = { allowed = ["m5.*"] } + } + } + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = var.compute_provider.aws.ec2.instance_types + ami = var.compute_provider.aws.ec2.ami + } + } + } + } + + microvm = { + runner = { + os = "linux" + architecture = "arm64" + name_prefix = "microvm-" + extra_labels = ["microvm"] + } + orchestration_provider = { + webhook = { + github = { + organization_runners = true + } + matcherConfig = { + labelMatchers = [["self-hosted", "linux", "arm64", "microvm"]] + bidirectionalLabelMatch = true + dynamic_labels_enabled = true + awsDynamicLabelsPolicy = { + restricted_keys = { + "image-version" = { allowed = ["3.0"] } + } + } + } + } + } + compute_provider = { + aws = { + microvm = {} + } + } + } + } +} diff --git a/examples/multi-runner-webhook/microvm.tf b/examples/multi-runner-webhook/microvm.tf new file mode 100644 index 0000000000..fa2a39e758 --- /dev/null +++ b/examples/multi-runner-webhook/microvm.tf @@ -0,0 +1,74 @@ +resource "aws_cloudwatch_log_group" "aws_cloudwatch_log_group_microvm" { + name = "/aws/lambda/microvms/ubuntu24" +} + +resource "aws_ecr_repository" "base_ubuntu24" { + name = "base-ubuntu24" + force_delete = true +} + + +data "aws_iam_policy_document" "ecr_repository_policy" { + + statement { + effect = "Allow" + actions = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:DescribeImages", + "ecr:GetAuthorizationToken", + "ecr:ListImages" + ] + + principals { + type = "AWS" + identifiers = [data.aws_caller_identity.current.account_id] + } + } +} + +resource "aws_ecr_repository_policy" "repository_policy" { + repository = "base-ubuntu24" + policy = data.aws_iam_policy_document.ecr_repository_policy.json +} + + +locals { + network_connectors = { + ministack = { + name = "ministack" + vpc_id = module.base.vpc.vpc_id + subnet_ids = module.base.vpc.private_subnets + } + } +} + +data "aws_caller_identity" "current" {} + +module "microvm_foundation" { + source = "../../modules/microvm-foundation" + + aws_region = var.aws_region + tags = { + Component = "microvm-foundation" + } + build_policy_name_prefix = "gha-microvm-build-policy-" + build_role_name_prefix = "gha-microvm-build-" + network_connector_operator_role_name_prefix = "gha-microvm-network-operator-" + usage_policy_name_prefix = "gha-microvm-runtime-usage-policy-" + artifact_bucket_name = "ministack-microvm-artifacts-${var.aws_region}" + artifact_retention_days = 30 + image_name_prefix = "gha-ubuntu-arm64" + ecr_repository_arns = ["arn:aws:ecr:${var.aws_region}:${data.aws_caller_identity.current.account_id}:repository/base-ubuntu24"] + network_connectors = local.network_connectors + force_destroy_artifact_bucket = true +} + +locals { + microvm = { + ecr_repo = aws_ecr_repository.base_ubuntu24.repository_url + log_group = aws_cloudwatch_log_group.aws_cloudwatch_log_group_microvm.name + microvm_foundation = module.microvm_foundation + } +} \ No newline at end of file diff --git a/examples/multi-runner-v2/outputs.tf b/examples/multi-runner-webhook/outputs.tf similarity index 57% rename from examples/multi-runner-v2/outputs.tf rename to examples/multi-runner-webhook/outputs.tf index 1feaf2e671..333ef57ea6 100644 --- a/examples/multi-runner-v2/outputs.tf +++ b/examples/multi-runner-webhook/outputs.tf @@ -1,8 +1,11 @@ output "webhook_endpoint" { value = module.runners.webhook.endpoint } - output "webhook_secret" { sensitive = true - value = random_id.random.hex + value = var.github_app.webhook_secret } + +output "microvm" { + value = local.microvm +} \ No newline at end of file diff --git a/examples/multi-runner-webhook/providers.tf b/examples/multi-runner-webhook/providers.tf new file mode 100644 index 0000000000..f24f950b27 --- /dev/null +++ b/examples/multi-runner-webhook/providers.tf @@ -0,0 +1,9 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Example = var.environment + } + } +} diff --git a/examples/multi-runner-webhook/variables.tf b/examples/multi-runner-webhook/variables.tf new file mode 100644 index 0000000000..7e19d64c1a --- /dev/null +++ b/examples/multi-runner-webhook/variables.tf @@ -0,0 +1,68 @@ +variable "aws_region" { + description = "AWS Region where the runner control plane and compute provider resources are deployed." + type = string + default = "eu-west-1" +} + +variable "environment" { + description = "Name prefix for the example resources." + type = string +} + +variable "github_app" { + description = "GitHub App credentials used by the webhook orchestration provider." + sensitive = true + + type = object({ + id = string + key_base64 = string + webhook_secret = string + }) +} + +variable "github_enterprise_server" { + description = "Optional GitHub Enterprise Server endpoint used by the smoke-test API mock." + type = object({ + url = string + ssl_verify = bool + }) + default = null +} + +variable "runners_lambda_zip" { + description = "Local ZIP file for the runner-control Lambda." + type = string +} + +variable "webhook_lambda_zip" { + description = "Local ZIP file for the webhook Lambda." + type = string +} + +variable "compute_provider" { + description = "Provider-specific settings for the EC2 and MicroVM runner lanes." + + type = object({ + aws = object({ + ec2 = object({ + instance_types = list(string) + ami = object({ + filter = optional(map(list(string)), { state = ["available"] }) + owners = optional(list(string), ["amazon"]) + id_ssm_parameter = optional(object({ + arn = string + }), null) + kms_key = optional(object({ + arn = string + }), null) + }) + }) + microvm = object({ + image_arn = string + image_version = optional(string, null) + ingress_network_connectors = optional(list(string), []) + egress_network_connectors = list(string) + }) + }) + }) +} diff --git a/examples/multi-runner-v2/versions.tf b/examples/multi-runner-webhook/versions.tf similarity index 76% rename from examples/multi-runner-v2/versions.tf rename to examples/multi-runner-webhook/versions.tf index 6af69ab915..6883c62423 100644 --- a/examples/multi-runner-v2/versions.tf +++ b/examples/multi-runner-webhook/versions.tf @@ -4,14 +4,15 @@ terraform { source = "hashicorp/aws" version = ">= 6.33" } - local = { - source = "hashicorp/local" - version = "~> 2.0" + null = { + source = "hashicorp/null" + version = "~> 3.0" } random = { source = "hashicorp/random" version = "~> 3.0" } } + required_version = ">= 1.5.6" } diff --git a/images/microvm-ubuntu/README.md b/images/microvm-ubuntu/README.md index a744265329..af1ce6b729 100644 --- a/images/microvm-ubuntu/README.md +++ b/images/microvm-ubuntu/README.md @@ -7,8 +7,11 @@ Dockerfile, compiled lifecycle-hook ZIP contract, and image entrypoint. Before building the image: 1. Apply `examples/microvm-foundation` in the target AWS Region. -2. Install Packer and set the required AWS, S3, IAM, connector, and - lifecycle-hook variables. +2. Build and release the lifecycle-hook service from + `lambdas/services/microvm-lifecycle-hooks`, then set + `MICROVM_LIFECYCLE_HOOK_ZIP` to the released artifact. +3. Install Packer and set the required AWS, S3, IAM, connector, and image + variables. The image intentionally excludes the source repository's optional external telemetry and Teleport services. It contains only the Actions runner, @@ -39,7 +42,13 @@ packer build -color=false github_agent.microvm.ubuntu.pkr.hcl ``` The build role, artifact bucket, and network connector are created by the -foundation module. Keep the bucket private and versioned, use the module's -least-privilege policies, and do not put credentials in checked-in files. The -lifecycle-hook ZIP must contain the compiled `server.js` at its archive root; -any bundled dependencies must use safe relative paths. +foundation module. The build role is used only while Packer creates and +publishes the image; it is not baked into the image and is not the role used +by runner jobs. The execution role is selected by the runner control plane and +passed to `RunMicrovm` at launch time, so it is not configured by this image +build. + +Keep the bucket private and versioned, use the module's least-privilege +policies, and do not put credentials in checked-in files. The lifecycle-hook +ZIP must contain the compiled `server.js` at its archive root; any bundled +dependencies must use safe relative paths. diff --git a/images/microvm-ubuntu/github_agent.microvm.ubuntu.pkr.hcl b/images/microvm-ubuntu/github_agent.microvm.ubuntu.pkr.hcl index f8d0638d94..7e4965f7d0 100644 --- a/images/microvm-ubuntu/github_agent.microvm.ubuntu.pkr.hcl +++ b/images/microvm-ubuntu/github_agent.microvm.ubuntu.pkr.hcl @@ -3,13 +3,6 @@ # while delegating packaging, regional publication, and polling to boto3. # The null builder and shell-local provisioner are Packer built-ins, so this # template intentionally has no required_plugins entry for them. - -variable "aws_data_path" { - description = "Botocore data path containing the Lambda MicroVM service model." - type = string - default = env("AWS_DATA_PATH") -} - variable "aws_region" { description = "AWS Region for the S3 artifact, Ubuntu ECR mirror, and Lambda MicroVM image." type = string @@ -96,7 +89,6 @@ build { # MICROVM_ENVIRONMENT_VARIABLES is inherited from the build step. Do not # add it here: shell-local renders environment_vars into the shell argv. environment_vars = [ - "AWS_DATA_PATH=${var.aws_data_path}", "AWS_REGION=${var.aws_region}", "MICROVM_ARTIFACT_BUCKET=${var.artifact_bucket}", "MICROVM_BUILD_ROLE_ARN=${var.build_role_arn}", diff --git a/images/microvm-ubuntu/packer/scripts/microvm/build-microvm-image.py b/images/microvm-ubuntu/packer/scripts/microvm/build-microvm-image.py index 81c13beb2e..c8fb9e5b6d 100755 --- a/images/microvm-ubuntu/packer/scripts/microvm/build-microvm-image.py +++ b/images/microvm-ubuntu/packer/scripts/microvm/build-microvm-image.py @@ -256,7 +256,7 @@ def microvm_client(session: Any, region: str) -> Any: except Exception as error: if type(error).__name__ == 'UnknownServiceError': raise BuildError( - 'AWS_DATA_PATH must contain the Lambda MicroVM service model' + 'Version of boto3 must contain the Lambda MicroVM service model' ) from error raise diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts index e58d73093c..84a0c5e0fd 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -7,10 +7,10 @@ const cleanEnv = process.env; beforeEach(() => { process.env = { ...cleanEnv }; process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; + process.env.MICROVM_IMAGE_VERSION = '2.0'; process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; process.env.SSM_TOKEN_PATH = '/github-action-runners/unit-test/token/'; - delete process.env.MICROVM_IMAGE_VERSION; delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; delete process.env.MICROVM_LOG_GROUP; @@ -20,7 +20,7 @@ describe('loadMicrovmProviderConfig', () => { it('loads required values and applies optional defaults', () => { expect(loadMicrovmProviderConfig()).toEqual({ imageIdentifier: process.env.MICROVM_IMAGE_ARN, - imageVersion: undefined, + imageVersion: '2.0', executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, ingressNetworkConnectors: undefined, egressNetworkConnectors: undefined, @@ -46,6 +46,7 @@ describe('loadMicrovmProviderConfig', () => { it.each([ ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], + ['MICROVM_IMAGE_VERSION', 'MICROVM_IMAGE_VERSION'], ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], ['SSM_TOKEN_PATH', 'SSM_TOKEN_PATH'], diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts index b86331967b..9baf16c1d7 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -4,7 +4,7 @@ export interface MicrovmProviderConfig { egressNetworkConnectors?: string[]; executionRoleArn: string; imageIdentifier: string; - imageVersion?: string; + imageVersion: string; ingressNetworkConnectors?: string[]; logging?: Logging; metadataSsmPath: string; @@ -61,7 +61,7 @@ export function loadMicrovmProviderConfig(): MicrovmProviderConfig { return { imageIdentifier: requiredEnvironmentValue('MICROVM_IMAGE_ARN', process.env.MICROVM_IMAGE_ARN), - imageVersion: optionalEnvironmentValue(process.env.MICROVM_IMAGE_VERSION), + imageVersion: requiredEnvironmentValue('MICROVM_IMAGE_VERSION', process.env.MICROVM_IMAGE_VERSION), executionRoleArn: requiredEnvironmentValue('MICROVM_EXECUTION_ROLE_ARN', process.env.MICROVM_EXECUTION_ROLE_ARN), ingressNetworkConnectors: parseNetworkConnectors( 'MICROVM_INGRESS_NETWORK_CONNECTORS', diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts index ace08450f3..ced9b18a4f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -88,7 +88,7 @@ export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { runnerTokenSsmPath: '/runner/token', }); }); - - it('requires the image ARN and version to be provided together', () => { - expect(() => - createMicrovmRunHookPayload({ - imageArn, - runnerConfigSsmPath, - runnerTokenSsmPath, - }), - ).toThrow('MicroVM hook payload image ARN and version must be provided together'); - }); - - it('omits image metadata when no explicit image version is selected', () => { - expect(JSON.parse(createMicrovmRunHookPayload({ runnerConfigSsmPath, runnerTokenSsmPath }))).toEqual({ - version: 1, - runnerConfigSsmPath, - runnerTokenSsmPath, - }); - }); }); describe('createMicrovmRunners', () => { @@ -150,6 +132,7 @@ describe('createMicrovmRunners', () => { it('rejects a metadata path that overlaps the JIT token path', async () => { vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ imageIdentifier: imageArn, + imageVersion: '2.0', executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', metadataSsmPath: '/github-action-runners/unit-test/token/metadata', runnerTokenSsmPath, diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts index 9d7154707b..76c1d9fe8f 100644 --- a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -27,28 +27,18 @@ const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ ]); export interface MicrovmRunHookPayloadV1 { - imageArn?: string; - imageVersion?: string; + imageArn: string; + imageVersion: string; runnerConfigSsmPath: string; runnerTokenSsmPath: string; version: 1; } export function createMicrovmRunHookPayload(payload: Omit): string { - const hasImageArn = payload.imageArn !== undefined; - const hasImageVersion = payload.imageVersion !== undefined; - if (hasImageArn !== hasImageVersion) { - throw new Error('MicroVM hook payload image ARN and version must be provided together'); - } - return JSON.stringify({ version: 1, - ...(hasImageArn - ? { - imageArn: payload.imageArn, - imageVersion: payload.imageVersion, - } - : {}), + imageArn: payload.imageArn, + imageVersion: payload.imageVersion, runnerConfigSsmPath: payload.runnerConfigSsmPath, runnerTokenSsmPath: payload.runnerTokenSsmPath, } satisfies MicrovmRunHookPayloadV1); @@ -140,12 +130,8 @@ export async function createMicrovmRunners( nonRetryableErrorCount: 0, }; const runHookPayload = createMicrovmRunHookPayload({ - ...(config.imageVersion !== undefined - ? { - imageArn: config.imageIdentifier, - imageVersion: config.imageVersion, - } - : {}), + imageArn: config.imageIdentifier, + imageVersion: config.imageVersion, runnerConfigSsmPath: normalizedRunnerConfigPath, runnerTokenSsmPath: normalizedRunnerTokenPath, }); diff --git a/lambdas/libs/storage-providers/aws/ssm/logger.test.ts b/lambdas/libs/storage-providers/aws/ssm/logger.test.ts index 9fcd42af98..e954406b46 100644 --- a/lambdas/libs/storage-providers/aws/ssm/logger.test.ts +++ b/lambdas/libs/storage-providers/aws/ssm/logger.test.ts @@ -28,4 +28,11 @@ describe('AWS SSM storage logger', () => { expect(getErrorNames(error)).toEqual(['GetParameterError', 'ParameterNotFound']); }); + + it('includes the AWS service error type from a wrapped cause', () => { + const cause = Object.assign(new Error('ParameterNotFound'), { __type: 'ParameterNotFound' }); + const error = Object.assign(new Error('wrapped'), { name: 'GetParameterError', cause }); + + expect(getErrorNames(error)).toEqual(['GetParameterError', 'Error', 'ParameterNotFound']); + }); }); diff --git a/lambdas/libs/storage-providers/aws/ssm/logger.ts b/lambdas/libs/storage-providers/aws/ssm/logger.ts index 0e3633eb71..309a5c38ff 100644 --- a/lambdas/libs/storage-providers/aws/ssm/logger.ts +++ b/lambdas/libs/storage-providers/aws/ssm/logger.ts @@ -20,6 +20,9 @@ export function getErrorNames(error: unknown): string[] { if ('name' in current && typeof current.name === 'string') { names.push(current.name); } + if ('__type' in current && typeof current.__type === 'string' && !names.includes(current.__type)) { + names.push(current.__type); + } current = 'cause' in current ? current.cause : undefined; } diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts index 4fc13926d9..20f4373619 100644 --- a/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts +++ b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts @@ -53,7 +53,7 @@ describe('aws_ssm runner group cache store', () => { }); it('returns undefined when ParameterNotFound is wrapped by the SSM provider', async () => { - const cause = Object.assign(new Error('missing'), { name: 'ParameterNotFound' }); + const cause = Object.assign(new Error('ParameterNotFound'), { __type: 'ParameterNotFound' }); getParameterMock.mockRejectedValue( Object.assign(new Error('failed to get parameter'), { name: 'GetParameterError', cause }), ); @@ -64,7 +64,7 @@ describe('aws_ssm runner group cache store', () => { expect.objectContaining({ runnerGroupName: 'Default', parameterName: '/runner/config/runner-group/Default', - errorNames: ['GetParameterError', 'ParameterNotFound'], + errorNames: ['GetParameterError', 'Error', 'ParameterNotFound'], }), ); }); diff --git a/lambdas/package.json b/lambdas/package.json index b223239520..e070302b3a 100644 --- a/lambdas/package.json +++ b/lambdas/package.json @@ -3,7 +3,8 @@ "private": true, "workspaces": [ "functions/*", - "libs/*" + "libs/*", + "services/*" ], "scripts": { "build": "nx run-many --target=build --all", diff --git a/lambdas/services/microvm-lifecycle-hooks/README.md b/lambdas/services/microvm-lifecycle-hooks/README.md new file mode 100644 index 0000000000..075de5295c --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/README.md @@ -0,0 +1,137 @@ +# Lambda MicroVM lifecycle hooks + +This service implements the lifecycle-hook HTTP server used to start one ephemeral GitHub Actions runner inside an AWS Lambda MicroVM. Storage-specific reads and one-time consumption are delegated to `@aws-github-runner/storage-providers`; this package owns only payload validation, lifecycle state, and the runner process boundary. + +## Build and run + +From `lambdas/`: + +```bash +yarn nx test @aws-github-runner/microvm-lifecycle-hooks +yarn workspace @aws-github-runner/microvm-lifecycle-hooks build +yarn workspace @aws-github-runner/microvm-lifecycle-hooks start +``` + +`build` uses esbuild to create the self-contained CommonJS server bundle `dist/server.js`. It also writes `dist/package.json` with `type: commonjs` so the bundle remains executable after it is copied outside the Yarn workspace. + +## Build and release with the Lambda artifacts + +The lifecycle-hook server is a deployable image-build artifact, not a service +that is installed separately beside the runner control plane. Build and test it +through the normal Lambda workspace/release process, then provide the resulting +ZIP to the MicroVM image build as `MICROVM_LIFECYCLE_HOOK_ZIP`. Packer embeds +that artifact in the image; every published image used by the MicroVM provider +must contain a compatible hook server. + +The hook server's release lifecycle is therefore separate from the MicroVM +execution role. The image build uses the foundation's build role. When a job +starts, the runner control plane supplies the runtime execution role to +`RunMicrovm`. + +To build before invoking Docker, run the workspace build above. In the existing MicroVM runner Dockerfile, which already installs s6-overlay and the GitHub runner's Node 24 runtime, copy the complete artifact and replace the old hook command with: + +```dockerfile +COPY lambdas/services/microvm-lifecycle-hooks/dist/ /opt/microvm-lifecycle-hooks/ +ENTRYPOINT ["/init"] +CMD ["/command/with-contenv", "/opt/actions-runner/externals/node24/bin/node", "/opt/microvm-lifecycle-hooks/server.js"] +``` + +Alternatively, build the service inside Docker with the repository root as the build context. Add this pinned builder stage: + +```dockerfile +ARG NODE_BUILDER_IMAGE=node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 +FROM ${NODE_BUILDER_IMAGE} AS lifecycle-build +WORKDIR /source +COPY lambdas/ ./lambdas/ +RUN corepack enable \ + && cd lambdas \ + && yarn install --immutable \ + && yarn workspace @aws-github-runner/microvm-lifecycle-hooks build +``` + +Use a clean checkout for that build context, or exclude local `node_modules/`, `coverage/`, and `dist/` directories with `.dockerignore`, so host-built dependencies are not copied into the Linux builder. + +Then copy the builder output into the existing final runner stage and use its supervisor and Node 24 runtime: + +```dockerfile +COPY --from=lifecycle-build \ + /source/lambdas/services/microvm-lifecycle-hooks/dist/ \ + /opt/microvm-lifecycle-hooks/ +ENTRYPOINT ["/init"] +CMD ["/command/with-contenv", "/opt/actions-runner/externals/node24/bin/node", "/opt/microvm-lifecycle-hooks/server.js"] +``` + +For an image without s6-overlay, start the artifact with `node /opt/microvm-lifecycle-hooks/server.js` under that image's process supervisor. The hook binds to `0.0.0.0:8080` by default. Restrict the port to the Lambda MicroVM lifecycle network; the protocol does not add a separate application authentication layer. + +## Run payloads + +AWS sends an outer JSON object whose `runHookPayload` is itself a JSON string. Version 1 remains strict and SSM-specific for backwards compatibility: + +```json +{ + "microvmId": "microvm-bdd2d536-3d87-35e4-8b40-18664608ebc1", + "runHookPayload": "{\"version\":1,\"imageArn\":\"arn:aws:lambda:eu-west-1:166060576821:function:microvm-image\",\"imageVersion\":\"8.0\",\"runnerConfigSsmPath\":\"/github-action-runners/example/config\",\"runnerTokenSsmPath\":\"/github-action-runners/example/token\"}" +} +``` + +Version 1 is translated to the shared allowlisted SSM environment. Version 2 carries the exact environment-variable map under `context.storage`. SSM example: + +```json +{ + "version": 2, + "context": { + "storage": { + "RUNNER_CONFIG_STORAGE_PROVIDER": "aws_ssm", + "SSM_TOKEN_PATH": "/github-action-runners/example/token" + } + } +} +``` + +Both versions reject missing, unknown, or provider-incompatible fields. The SSM storage context accepts only its two keys; AWS credentials, timeout overrides, and arbitrary environment names are rejected. The validated storage map is exported once before the consumer is resolved. A retry may reuse the identical map, but it cannot change storage configuration after initialization. + +`microvmId` is an opaque path-safe `[A-Za-z0-9_.-]{1,256}` value. The resolved storage provider uses it to consume the one-time JIT configuration. Storage context variables are removed from the runner child environment. + +For a rolling upgrade, keep emitting version 1 SSM payloads until every deployed image contains this service. Old images do not understand version 2. + +## Entrypoint contract + +On `/run`, the hook starts `${RUNNER_ROOT:-/opt/actions-runner}/run.sh --jitconfig ` directly without a shell. The JIT configuration is passed only as the `--jitconfig` argument to the runner process: + +```text +run.sh --jitconfig +``` + +The hook waits for a short process-launch handoff before acknowledging `/run`; it does not require a custom readiness pipe. The JIT configuration, storage context, and AWS credential environment variables are not inherited by the runner process. `/terminate` sends `SIGTERM` to the detached process group and escalates to `SIGKILL` after the grace period. + +After the runner entrypoint exits on its own, the hook closes its HTTP server and exits with status `0` only when the runner exited cleanly. In the documented s6-overlay image layout above, that makes the foreground container command exit so s6 can stop the remaining image services and shut down the application container's PID 1. This path does not require `lambda:TerminateMicrovm` in the runner role. AWS documents only explicit termination and maximum duration as MicroVM termination triggers, so retain trusted control-plane cleanup and the maximum duration as failure backstops, and verify the container-exit behavior against a restored MicroVM before relying on it operationally. + +Useful environment variables are: + +| Variable | Default | Purpose | +| --------------------------------- | --------------------- | --------------------------------------------- | +| `HOOK_PORT` | `8080` | Lifecycle-hook HTTP port | +| `RUNNER_ROOT` | `/opt/actions-runner` | GitHub Actions runner installation | +| `RUNNER_USER` | `runner` | Runner process user name | +| `RUNNER_UID` | `1000` | Runner UID when the hook runs as root | +| `RUNNER_GID` | `1000` | Runner GID when the hook runs as root | +| `RUN_HOOK_TIMEOUT_SECONDS` | `55` | Total `/run` budget, bounded to 40–55 seconds | +| `HOOK_HEADERS_TIMEOUT_SECONDS` | `5` | HTTP header receive timeout | +| `HOOK_REQUEST_TIMEOUT_SECONDS` | `10` | HTTP request receive timeout | +| `HOOK_KEEP_ALIVE_TIMEOUT_SECONDS` | `5` | Idle keep-alive timeout | +| `AWS_SDK_CALL_TIMEOUT_SECONDS` | `5` | Individual storage-provider call timeout | +| `RUNNER_CONFIG_TIMEOUT_SECONDS` | `20` | Total runner-configuration polling timeout | +| `RUNNER_CONFIG_POLL_SECONDS` | `2` | Delay between provider polling attempts | +| `RUNNER_CONFIG_DELETE_ATTEMPTS` | `3` | SSM one-time configuration delete attempts | + +The request body is capped at 20 KiB and HTTP headers at 16 KiB. Internal errors are returned generically and secret-bearing provider errors are never logged. + +## Runtime security + +Removing AWS credential and storage variables from the runner child prevents accidental environment inheritance; it is not an IAM boundary. A job can still obtain credentials made available to the runtime role, so scope that role to each lane and treat job code as untrusted. + +- For SSM, grant only `ssm:GetParameter` and `ssm:DeleteParameter` on the lane's token path. Add `kms:Decrypt` only for the customer-managed key that encrypts those parameters. + +## TypeScript API + +The workspace service root is import-safe; importing it does not start the server. It exports the parser, lifecycle, process launcher, storage adapter, and server factories for composition and testing. `src/index.ts` is the executable-only NCC entrypoint. A producer can call `loadRunnerConfigStorageContextFromEnvironment` from `@aws-github-runner/storage-providers/runner-config-consumer` to copy only the selected provider and locator into `context.storage`. diff --git a/lambdas/services/microvm-lifecycle-hooks/build.mjs b/lambdas/services/microvm-lifecycle-hooks/build.mjs new file mode 100644 index 0000000000..4f26c0dcb8 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/build.mjs @@ -0,0 +1,21 @@ +import { mkdir, rm, writeFile } from 'node:fs/promises'; + +import { build } from 'esbuild'; + +await rm('dist', { force: true, recursive: true }); +await mkdir('dist', { recursive: true }); + +await build({ + bundle: true, + entryPoints: ['src/index.ts'], + format: 'cjs', + legalComments: 'eof', + minify: false, + packages: 'bundle', + platform: 'node', + sourcemap: false, + target: 'node24', + outfile: 'dist/server.js', +}); + +await writeFile('dist/package.json', '{\n "type": "commonjs"\n}\n'); diff --git a/lambdas/services/microvm-lifecycle-hooks/package.json b/lambdas/services/microvm-lifecycle-hooks/package.json new file mode 100644 index 0000000000..663194be7e --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/package.json @@ -0,0 +1,45 @@ +{ + "name": "@aws-github-runner/microvm-lifecycle-hooks", + "version": "1.0.0", + "private": true, + "description": "AWS Lambda MicroVM lifecycle hook server for ephemeral GitHub Actions runners", + "main": "src/public.ts", + "exports": { + ".": "./src/public.ts" + }, + "type": "module", + "license": "MIT", + "engines": { + "node": ">=24" + }, + "scripts": { + "start": "node dist/server.js", + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "build": "node build.mjs", + "dist": "yarn build && cd dist && zip ../microvm-lifecycle-hooks.zip *", + "format": "prettier --write \"**/*.{ts,json,md}\"", + "format-check": "prettier --check \"**/*.{ts,json,md}\"", + "all": "yarn build && yarn format && yarn lint && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "esbuild": "^0.27.0" + }, + "dependencies": { + "@aws-github-runner/storage-providers": "*" + }, + "nx": { + "includedScripts": [ + "build", + "dist", + "format", + "format-check", + "lint", + "start", + "watch", + "all" + ] + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/contracts.ts b/lambdas/services/microvm-lifecycle-hooks/src/contracts.ts new file mode 100644 index 0000000000..fbaadeacd1 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/contracts.ts @@ -0,0 +1,41 @@ +import type { RunnerConfigStorageContext } from '@aws-github-runner/storage-providers/runner-config-consumer'; + +export interface RunContext { + imageArn?: string; + imageVersion?: string; + microvmId: string; + storage: RunnerConfigStorageContext; +} + +export interface ConsumeOptions { + deadlineMs: number; + signal: AbortSignal; +} + +export interface RunnerBootstrap { + jitConfig: string; +} + +/** Resolves and consumes a one-time runner configuration without exposing provider details. */ +export interface JitConfigSource { + consume(context: RunContext, options: ConsumeOptions): Promise; +} + +export interface ManagedProcess { + readonly ready: Promise; + readonly exit: Promise; + readonly exited: boolean; + stop(graceMs?: number): Promise; +} + +export interface RunnerLauncher { + launch(bootstrap: RunnerBootstrap, microvmId: string): ManagedProcess; +} + +export interface Logger { + info(message: string, ...values: unknown[]): void; + warn(message: string, ...values: unknown[]): void; + error(message: string, ...values: unknown[]): void; +} + +export const consoleLogger: Logger = console; diff --git a/lambdas/services/microvm-lifecycle-hooks/src/index.ts b/lambdas/services/microvm-lifecycle-hooks/src/index.ts new file mode 100644 index 0000000000..30cde16d7b --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/index.ts @@ -0,0 +1,7 @@ +import { consoleLogger } from './contracts'; +import { main } from './server'; + +void main().catch(() => { + consoleLogger.error('Lambda MicroVM lifecycle hook server failed to start'); + process.exitCode = 1; +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.test.ts new file mode 100644 index 0000000000..9bdf48eadc --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.test.ts @@ -0,0 +1,357 @@ +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; +import { arch, tmpdir, type } from 'node:os'; +import { join } from 'node:path'; + +import type { JitConfigSource, Logger, ManagedProcess, RunContext, RunnerBootstrap, RunnerLauncher } from './contracts'; +import { RunnerLifecycle } from './lifecycle'; + +const quietLogger: Logger = { + error: () => undefined, + info: () => undefined, + warn: () => undefined, +}; + +const MICROVM_ID = 'microvm-bdd2d536-3d87-35e4-8b40-18664608ebc1'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function overrideEnvironment(overrides: Record): () => void { + const previous = new Map(); + for (const [name, value] of Object.entries(overrides)) { + previous.set(name, process.env[name]); + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + return (): void => { + for (const [name, value] of previous) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + }; +} + +function runRequest( + payload: object = { + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + runnerConfigSsmPath: '/runner/config', + runnerTokenSsmPath: '/runner/token', + version: 1, + }, +): string { + return JSON.stringify({ + microvmId: MICROVM_ID, + runHookPayload: JSON.stringify(payload), + }); +} + +class DeferredProcess implements ManagedProcess { + public readonly ready = Promise.resolve(); + public readonly exit: Promise; + public exited = false; + private resolveExit!: (code: number | null) => void; + + public constructor() { + this.exit = new Promise((resolve) => { + this.resolveExit = resolve; + }); + } + + public finish(code: number | null): void { + this.exited = true; + this.resolveExit(code); + } + + public async stop(): Promise { + if (!this.exited) { + this.finish(null); + } + } +} + +describe('RunnerLifecycle', () => { + it('writes setup information before launching the runner', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-lifecycle-setup-info-')); + const restoreEnvironment = overrideEnvironment({ ACTIONS_RUNNER_ROOT: directory }); + let setupInfoAtLaunch: unknown; + const launcher: RunnerLauncher = { + launch(): ManagedProcess { + setupInfoAtLaunch = JSON.parse(readFileSync(join(directory, '.setup_info'), 'utf8')); + return new DeferredProcess(); + }, + }; + + try { + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + launcher, + quietLogger, + ); + + await lifecycle.start(runRequest()); + + expect(setupInfoAtLaunch).toEqual([ + { + group: 'Operating System', + detail: `Platform: ${type()}\nArchitecture: ${arch()}`, + }, + { + group: 'Runner Image', + detail: + 'MicroVM image ARN: arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner\nMicroVM image version: 8.0', + }, + { + group: 'Lambda MicroVM', + detail: `MicroVM id: ${MICROVM_ID}`, + }, + ]); + expect((await stat(join(directory, '.setup_info'))).mode & 0o777).toBe(0o644); + } finally { + restoreEnvironment(); + await rm(directory, { force: true, recursive: true }); + } + }); + + it('writes available setup information without image metadata', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-lifecycle-setup-info-')); + const restoreEnvironment = overrideEnvironment({ ACTIONS_RUNNER_ROOT: directory }); + let setupInfoAtLaunch: unknown; + const launcher: RunnerLauncher = { + launch(): ManagedProcess { + setupInfoAtLaunch = JSON.parse(readFileSync(join(directory, '.setup_info'), 'utf8')); + return new DeferredProcess(); + }, + }; + + try { + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + launcher, + quietLogger, + ); + + await lifecycle.start( + runRequest({ + runnerConfigSsmPath: '/runner/config', + runnerTokenSsmPath: '/runner/token', + version: 1, + }), + ); + + expect(setupInfoAtLaunch).toEqual([ + { + group: 'Operating System', + detail: `Platform: ${type()}\nArchitecture: ${arch()}`, + }, + { + group: 'Lambda MicroVM', + detail: `MicroVM id: ${MICROVM_ID}`, + }, + ]); + } finally { + restoreEnvironment(); + await rm(directory, { force: true, recursive: true }); + } + }); + + it('starts only once and waits for terminate cleanup after the runner exits', async () => { + const events: string[] = []; + const processHandle = new DeferredProcess(); + const source: JitConfigSource = { + async consume(context: RunContext): Promise { + events.push(`consume:${context.storage.RUNNER_CONFIG_STORAGE_PROVIDER}:${context.microvmId}`); + return { jitConfig: 'encoded-jit' }; + }, + }; + const launcher: RunnerLauncher = { + launch(bootstrap, id): ManagedProcess { + events.push(`launch:${id}:${bootstrap.jitConfig}`); + return processHandle; + }, + }; + const lifecycle = new RunnerLifecycle(source, launcher, quietLogger); + + await expect(lifecycle.start(runRequest())).resolves.toBe(true); + await expect(lifecycle.start(runRequest())).resolves.toBe(false); + expect(events).toEqual([`consume:aws_ssm:${MICROVM_ID}`, `launch:${MICROVM_ID}:encoded-jit`]); + + processHandle.finish(0); + await expect(lifecycle.completion).resolves.toBe(0); + await lifecycle.stop(); + expect(processHandle.exited).toBe(true); + }); + + it('does not report an externally requested stop as runner self-completion', async () => { + const processHandle = new DeferredProcess(); + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + { launch: () => processHandle }, + quietLogger, + ); + + await lifecycle.start(runRequest()); + await lifecycle.stop(); + + await expect( + Promise.race([ + lifecycle.completion.then(() => 'completed'), + new Promise((resolve) => setImmediate(() => resolve('pending'))), + ]), + ).resolves.toBe('pending'); + }); + + it('reserves the runner startup budget before consuming configuration', async () => { + let consumeDeadline = 0; + const processHandle = new DeferredProcess(); + const lifecycle = new RunnerLifecycle( + { + async consume(_context, options): Promise { + consumeDeadline = options.deadlineMs; + return { jitConfig: 'encoded-jit' }; + }, + }, + { launch: () => processHandle }, + quietLogger, + ); + vi.spyOn(Date, 'now').mockReturnValue(1_000); + + await lifecycle.start(runRequest()); + + expect(consumeDeadline).toBe(21_000); + await lifecycle.stop(); + }); + + it('returns to idle if the GitHub Actions runner cannot launch', async () => { + const consume = vi.fn().mockResolvedValue({ jitConfig: 'encoded-jit' }); + const lifecycle = new RunnerLifecycle( + { consume }, + { + launch(): ManagedProcess { + throw new Error('spawn failed'); + }, + }, + quietLogger, + ); + + await expect(lifecycle.start(runRequest())).rejects.toThrow('spawn failed'); + await expect(lifecycle.start(runRequest())).rejects.toThrow('spawn failed'); + expect(consume).toHaveBeenCalledTimes(2); + }); + + it('logs the startup stage and safe error details without exposing internal messages', async () => { + const messages: unknown[] = []; + const logger: Logger = { + error: (...values) => messages.push(...values), + info: () => undefined, + warn: () => undefined, + }; + const error = new Error('encoded-jit-secret'); + error.name = 'encoded-jit-secret-name'; + Object.assign(error, { code: 'encoded-jit-secret-code' }); + const lifecycle = new RunnerLifecycle( + { + consume: async () => { + throw error; + }, + }, + { launch: () => new DeferredProcess() }, + logger, + ); + + await expect(lifecycle.start(runRequest())).rejects.toBe(error); + + const serializedMessages = JSON.stringify(messages); + expect(serializedMessages).toContain('consume runner configuration'); + expect(serializedMessages).toContain('unknown-error'); + expect(serializedMessages).not.toContain('encoded-jit-secret'); + }); + + it('aborts in-flight consumption when the run-hook deadline elapses', async () => { + let consumedSignal: AbortSignal | undefined; + let launched = false; + let releaseConsume = (): void => undefined; + const consumption = new Promise((resolve) => { + releaseConsume = resolve; + }); + const lifecycle = new RunnerLifecycle( + { + async consume(_context, options): Promise { + consumedSignal = options.signal; + await consumption; + return { jitConfig: 'encoded-jit' }; + }, + }, + { + launch(): ManagedProcess { + launched = true; + return new DeferredProcess(); + }, + }, + quietLogger, + ); + let calls = 0; + vi.spyOn(Date, 'now').mockImplementation(() => (calls++ === 0 ? 1_000 : 61_000)); + + await expect(lifecycle.start(runRequest())).rejects.toThrow('run-hook deadline elapsed'); + releaseConsume(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(consumedSignal?.aborted).toBe(true); + expect(launched).toBe(false); + }); + + it('waits for cleanup when terminate races with the runner launch handoff', async () => { + let finishCleanup = (): void => undefined; + let reportLaunched = (): void => undefined; + let stopCalled = false; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + const launched = new Promise((resolve) => { + reportLaunched = resolve; + }); + const processHandle: ManagedProcess = { + ready: new Promise(() => undefined), + exit: new Promise(() => undefined), + exited: false, + async stop(): Promise { + stopCalled = true; + await cleanup; + }, + }; + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + { + launch(): ManagedProcess { + reportLaunched(); + return processHandle; + }, + }, + quietLogger, + ); + + const rejectedStart = expect(lifecycle.start(runRequest())).rejects.toThrow('runner start was cancelled'); + await launched; + let terminateSettled = false; + const terminate = lifecycle.stop().then(() => { + terminateSettled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(stopCalled).toBe(true); + expect(terminateSettled).toBe(false); + + finishCleanup(); + await terminate; + await rejectedStart; + expect(terminateSettled).toBe(true); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.ts b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.ts new file mode 100644 index 0000000000..87fe7c02df --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.ts @@ -0,0 +1,235 @@ +import type { JitConfigSource, Logger, ManagedProcess, RunnerBootstrap, RunnerLauncher } from './contracts'; +import { consoleLogger } from './contracts'; +import { parseRunRequest } from './payload'; +import { writeRunnerSetupInfo } from './processes'; +import { beforeDeadline, beforeDeadlineOrAbort } from './timing'; + +type LifecycleState = 'idle' | 'starting' | 'running' | 'stopping' | 'stopped'; + +const SAFE_ERROR_NAMES = new Set([ + 'AccessDeniedException', + 'AbortError', + 'ExpiredTokenException', + 'InternalServerError', + 'InvalidKeyId', + 'KMSInvalidStateException', + 'ParameterNotFound', + 'ResourceNotFoundException', + 'TimeoutError', + 'ThrottlingException', +]); + +const SAFE_ERROR_CODES = new Set(['EACCES', 'EINVAL', 'ENOENT', 'EPERM', 'ETIMEDOUT']); + +const SAFE_ERROR_MESSAGES = new Set([ + 'operation was cancelled', + 'run-hook deadline elapsed', + 'GitHub Actions runner exited before the launch handoff', + 'runner launch was cancelled', + 'runner start was cancelled', +]); + +interface DiagnosticError extends Error { + code?: unknown; + $metadata?: unknown; +} + +function safeErrorDetails(error: unknown): Record { + if (!(error instanceof Error)) { + return { errorType: typeof error }; + } + + const diagnosticError = error as DiagnosticError; + const details: Record = { + errorName: SAFE_ERROR_NAMES.has(error.name) ? error.name : 'unknown-error', + }; + if (SAFE_ERROR_CODES.has(diagnosticError.code as string)) { + details.errorCode = diagnosticError.code; + } + if (SAFE_ERROR_MESSAGES.has(error.message)) { + details.errorMessage = error.message; + } + + if (diagnosticError.$metadata !== null && typeof diagnosticError.$metadata === 'object') { + const metadata = diagnosticError.$metadata as Record; + if (typeof metadata.httpStatusCode === 'number' && Number.isInteger(metadata.httpStatusCode)) { + details.httpStatusCode = metadata.httpStatusCode; + } + if (typeof metadata.attempts === 'number' && Number.isInteger(metadata.attempts)) { + details.awsAttempts = metadata.attempts; + } + if (typeof metadata.totalRetryDelay === 'number' && Number.isInteger(metadata.totalRetryDelay)) { + details.awsRetryDelayMs = metadata.totalRetryDelay; + } + } + + return details; +} + +function boundedNumber(value: string | undefined, fallback: number, minimum: number, maximum: number): number { + const parsed = Number(value); + return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback; +} + +export class RunnerLifecycle { + private readonly runHookBudgetMs = boundedNumber(process.env.RUN_HOOK_TIMEOUT_SECONDS, 55, 40, 55) * 1_000; + // Reserve Lambda's 30-second service readiness window plus five seconds of local margin. + private readonly launchReserveMs = 35_000; + private state: LifecycleState = 'idle'; + private microvmId?: string; + private startAbort?: AbortController; + private startPromise?: Promise; + private runner?: ManagedProcess; + private resolveCompletion!: (exitCode: number | null) => void; + public readonly completion = new Promise((resolve) => { + this.resolveCompletion = resolve; + }); + + public constructor( + private readonly jitConfigSource: JitConfigSource, + private readonly launcher: RunnerLauncher, + private readonly logger: Logger = consoleLogger, + ) {} + + private currentState(): LifecycleState { + return this.state; + } + + public async start(body: string): Promise { + const context = parseRunRequest(body); + const deadlineMs = Date.now() + this.runHookBudgetMs; + + if (this.microvmId === context.microvmId && this.state === 'running') { + return false; + } + if (this.microvmId === context.microvmId && this.state === 'starting') { + if (this.startPromise === undefined) { + throw new Error('runner start state is inconsistent'); + } + await beforeDeadline(this.startPromise, deadlineMs); + if (this.currentState() === 'running') { + return false; + } + throw new Error('the preceding runner start did not succeed'); + } + if (this.state !== 'idle') { + throw new Error('another runner lifecycle is already active in this MicroVM'); + } + + const abort = new AbortController(); + this.state = 'starting'; + this.microvmId = context.microvmId; + this.startAbort = abort; + const startOperation = this.startRunner(context, deadlineMs, abort); + this.startPromise = startOperation; + const clearStartPromise = (): void => { + if (this.startPromise === startOperation) { + this.startPromise = undefined; + } + }; + void startOperation.then(clearStartPromise, clearStartPromise); + try { + await beforeDeadline(startOperation, deadlineMs); + return true; + } catch (error) { + // Cancel the underlying work so a timed-out hook cannot register a runner later. + abort.abort(); + throw error; + } + } + + private async startRunner( + context: ReturnType, + deadlineMs: number, + abort: AbortController, + ): Promise { + let bootstrap: RunnerBootstrap | undefined; + let processHandle: ManagedProcess | undefined; + let stage = 'consume runner configuration'; + try { + this.logger.info('Lifecycle hook consuming runner configuration for MicroVM %s', context.microvmId); + bootstrap = await this.jitConfigSource.consume(context, { + deadlineMs: deadlineMs - this.launchReserveMs, + signal: abort.signal, + }); + if (abort.signal.aborted) { + throw new Error('runner start was cancelled'); + } + + await writeRunnerSetupInfo(context, this.logger); + stage = 'launch GitHub Actions runner'; + this.logger.info('Lifecycle hook launching GitHub Actions runner for MicroVM %s', context.microvmId); + processHandle = this.launcher.launch(bootstrap, context.microvmId); + + stage = 'wait for runner launch handoff'; + this.logger.info('Lifecycle hook waiting for runner launch handoff for MicroVM %s', context.microvmId); + await beforeDeadlineOrAbort(processHandle.ready, deadlineMs, abort.signal); + if (abort.signal.aborted || this.state !== 'starting') { + throw new Error('runner start was cancelled'); + } + + this.runner = processHandle; + this.startAbort = undefined; + this.state = 'running'; + this.logger.info('GitHub Actions runner launch handed off for MicroVM %s', context.microvmId); + void this.monitorRunner(processHandle); + } catch (error) { + this.logger.error('Lifecycle hook runner startup failed', { + microvmId: context.microvmId, + stage, + ...safeErrorDetails(error), + }); + if (processHandle !== undefined) { + await processHandle.stop(); + } + if (this.state === 'stopping') { + this.state = 'stopped'; + } else { + this.state = 'idle'; + this.microvmId = undefined; + } + this.startAbort = undefined; + throw error; + } finally { + // JavaScript strings cannot be zeroized, but release the retained credential promptly. + if (bootstrap !== undefined) { + bootstrap.jitConfig = ''; + } + } + } + + private async monitorRunner(processHandle: ManagedProcess): Promise { + const exitCode = await processHandle.exit; + if (this.runner === processHandle) { + this.runner = undefined; + this.state = 'stopped'; + this.resolveCompletion(exitCode); + } + } + + public async stop(): Promise { + if (this.state === 'idle') { + this.state = 'stopped'; + } else if (this.state === 'starting' || this.state === 'running') { + this.state = 'stopping'; + } + this.startAbort?.abort(); + const starting = this.startPromise; + if (starting !== undefined) { + try { + await starting; + } catch { + // Cancellation is expected when terminate races with /run. + } + } + const running = this.runner; + this.runner = undefined; + await (running?.stop() ?? Promise.resolve()); + this.state = 'stopped'; + } + + public async resume(): Promise { + // Never re-consume a one-time runner configuration on resume. + return true; + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/payload.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/payload.test.ts new file mode 100644 index 0000000000..d357686ffc --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/payload.test.ts @@ -0,0 +1,136 @@ +import { HookRequestError, parseRunRequest } from './payload'; + +const MICROVM_ID = 'microvm-bdd2d536-3d87-35e4-8b40-18664608ebc1'; +const SSM_STORAGE = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', +} as const; +function request( + payload: object = { + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + runnerConfigSsmPath: '/github-action-runners/tenant/config', + runnerTokenSsmPath: '/github-action-runners/tenant/token', + version: 1, + }, + microvmId = MICROVM_ID, +): string { + return JSON.stringify({ + microvmId, + runHookPayload: JSON.stringify(payload), + }); +} + +describe('parseRunRequest', () => { + it('maps the producer version 1 payload to the allowlisted SSM storage environment', () => { + expect(parseRunRequest(request())).toEqual({ + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + microvmId: MICROVM_ID, + storage: SSM_STORAGE, + }); + }); + + it('accepts version 1 payloads without image metadata', () => { + expect( + parseRunRequest( + request({ + runnerConfigSsmPath: '/github-action-runners/tenant/config', + runnerTokenSsmPath: '/github-action-runners/tenant/token', + version: 1, + }), + ), + ).toEqual({ + microvmId: MICROVM_ID, + storage: SSM_STORAGE, + }); + }); + + it('preserves version 1 trailing-slash normalization', () => { + expect( + parseRunRequest( + request({ + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + runnerConfigSsmPath: '/github-action-runners/tenant/config/', + runnerTokenSsmPath: '/github-action-runners/tenant/token/', + version: 1, + }), + ).storage, + ).toEqual({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', + }); + }); + + it.each([SSM_STORAGE])('accepts a strict version 2 $RUNNER_CONFIG_STORAGE_PROVIDER storage context', (storage) => { + expect( + parseRunRequest( + request({ + context: { storage }, + version: 2, + }), + ), + ).toEqual({ microvmId: MICROVM_ID, storage }); + }); + + it('accepts opaque path-safe MicroVM identifiers up to 256 characters', () => { + expect(parseRunRequest(request(undefined, 'a'.repeat(256))).microvmId).toHaveLength(256); + expect(parseRunRequest(request(undefined, 'future_id.example-01')).microvmId).toBe('future_id.example-01'); + }); + + it.each([ + ['invalid outer JSON', '{'], + ['an invalid MicroVM identifier', request(undefined, '../vm')], + ['an overlong MicroVM identifier', request(undefined, 'a'.repeat(257))], + ['an unversioned payload', request({ runnerConfigSsmPath: '/runner/config', runnerTokenSsmPath: '/runner/token' })], + [ + 'partial image metadata', + request({ + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + runnerConfigSsmPath: '/runner/config', + runnerTokenSsmPath: '/runner/token', + version: 1, + }), + ], + ['a relative legacy SSM path', request({ runnerConfigSsmPath: 'runner/token', version: 1 })], + ['a root legacy SSM path', request({ runnerConfigSsmPath: '/', version: 1 })], + ['repeated legacy SSM slashes', request({ runnerConfigSsmPath: '/runner//token', version: 1 })], + ['legacy SSM traversal', request({ runnerConfigSsmPath: '/runner/../token', version: 1 })], + [ + 'extra version 1 fields', + request({ encodedJitConfig: 'not-a-real-secret', runnerConfigSsmPath: '/runner/token', version: 1 }), + ], + [ + 'missing version 1 fields', + request({ context: { storage: SSM_STORAGE }, runnerConfigSsmPath: '/runner/token', version: 1 }), + ], + ['missing version 2 context', request({ version: 2 })], + ['missing version 2 storage', request({ context: {}, version: 2 })], + ['extra version 2 fields', request({ context: { storage: SSM_STORAGE }, unexpected: true, version: 2 })], + ['extra version 2 context fields', request({ context: { storage: SSM_STORAGE, unexpected: true }, version: 2 })], + [ + 'an unknown storage provider', + request({ + context: { + storage: { RUNNER_CONFIG_STORAGE_PROVIDER: 'unknown', SSM_TOKEN_PATH: '/runner/token' }, + }, + version: 2, + }), + ], + [ + 'typed provider fields in the environment map', + request({ context: { storage: { provider: 'aws_ssm', tokenPath: '/runner/token' } }, version: 2 }), + ], + [ + 'AWS credential injection', + request({ context: { storage: { ...SSM_STORAGE, AWS_ACCESS_KEY_ID: 'not-a-real-key' } }, version: 2 }), + ], + [ + 'timeout override injection', + request({ context: { storage: { ...SSM_STORAGE, RUNNER_CONFIG_TIMEOUT_SECONDS: '60' } }, version: 2 }), + ], + ])('rejects %s', (_name, body) => { + expect(() => parseRunRequest(body)).toThrow(HookRequestError); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/payload.ts b/lambdas/services/microvm-lifecycle-hooks/src/payload.ts new file mode 100644 index 0000000000..c0f22aeb4c --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/payload.ts @@ -0,0 +1,131 @@ +import { + parseRunnerConfigStorageContext, + type RunnerConfigStorageContext, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; + +import type { RunContext } from './contracts'; + +const MICROVM_ID_PATTERN = /^[A-Za-z0-9_.-]{1,256}$/; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:aws[a-z-]*:lambda:[A-Za-z0-9-]+:[0-9]{12}:microvm-image:[A-Za-z0-9_.-]+$/; +const MICROVM_IMAGE_VERSION_PATTERN = /^[A-Za-z0-9_.-]{1,128}$/; + +export const MAX_REQUEST_BYTES = 20 * 1024; + +export class HookRequestError extends Error { + public constructor(message: string) { + super(message); + this.name = 'HookRequestError'; + } +} + +interface LambdaRunRequest { + microvmId?: unknown; + runHookPayload?: unknown; +} + +interface VersionedRunPayload { + version?: unknown; + imageArn?: unknown; + imageVersion?: unknown; + runnerConfigSsmPath?: unknown; + runnerTokenSsmPath?: unknown; + context?: unknown; +} + +interface VersionTwoContext { + storage?: unknown; +} + +function parseObject(value: string, errorMessage: string): T { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new HookRequestError(errorMessage); + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new HookRequestError(errorMessage); + } + return parsed as T; +} + +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const keys = Object.keys(value); + return keys.length === expected.length && keys.every((key) => expected.includes(key)); +} + +function hasOnlyKeys(value: object, allowed: readonly string[]): boolean { + return Object.keys(value).every((key) => allowed.includes(key)); +} + +function isObject(value: unknown): value is object { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseStorageContext(value: unknown): RunnerConfigStorageContext { + try { + return parseRunnerConfigStorageContext(value); + } catch { + // Storage validation details are deliberately not reflected to the hook caller. + throw new HookRequestError('runner configuration storage context is missing or invalid'); + } +} + +export function parseRunRequest(body: string): RunContext { + const request = parseObject(body, 'request body must be a JSON object'); + if (typeof request.microvmId !== 'string' || !MICROVM_ID_PATTERN.test(request.microvmId)) { + throw new HookRequestError('microvmId is missing or invalid'); + } + if (typeof request.runHookPayload !== 'string') { + throw new HookRequestError('runHookPayload must be a JSON string'); + } + + const payload = parseObject(request.runHookPayload, 'runHookPayload must contain valid JSON'); + if (payload.version === 1) { + if ( + !hasOnlyKeys(payload, ['version', 'imageArn', 'imageVersion', 'runnerConfigSsmPath', 'runnerTokenSsmPath']) || + typeof payload.runnerConfigSsmPath !== 'string' || + typeof payload.runnerTokenSsmPath !== 'string' + ) { + throw new HookRequestError('version 1 runHookPayload contains unsupported or missing fields'); + } + const hasImageMetadata = payload.imageArn !== undefined || payload.imageVersion !== undefined; + if ( + hasImageMetadata && + (typeof payload.imageArn !== 'string' || + payload.imageArn.length > 2_048 || + !MICROVM_IMAGE_ARN_PATTERN.test(payload.imageArn) || + typeof payload.imageVersion !== 'string' || + !MICROVM_IMAGE_VERSION_PATTERN.test(payload.imageVersion)) + ) { + throw new HookRequestError('imageArn and imageVersion must be valid when provided'); + } + return { + ...(hasImageMetadata + ? { + imageArn: payload.imageArn as string, + imageVersion: payload.imageVersion as string, + } + : {}), + microvmId: request.microvmId, + storage: parseStorageContext({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: payload.runnerTokenSsmPath, + }), + }; + } + if (payload.version === 2) { + if (!hasExactKeys(payload, ['version', 'context'])) { + throw new HookRequestError('version 2 runHookPayload contains unsupported or missing fields'); + } + if (!isObject(payload.context) || !hasExactKeys(payload.context, ['storage'])) { + throw new HookRequestError('version 2 context contains unsupported or missing fields'); + } + const context = payload.context as VersionTwoContext; + return { + microvmId: request.microvmId, + storage: parseStorageContext(context.storage), + }; + } + throw new HookRequestError('runHookPayload version must be 1 or 2'); +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/processes.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/processes.test.ts new file mode 100644 index 0000000000..46b46dafce --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/processes.test.ts @@ -0,0 +1,98 @@ +import { chown, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { GitHubRunnerLauncher } from './processes'; + +async function prepareRunnerFixture(directory: string, runner: string): Promise { + if (process.getuid?.() !== 0) { + return; + } + + const uid = Number(process.env.RUNNER_UID ?? 1_000); + const gid = Number(process.env.RUNNER_GID ?? 1_000); + await chown(directory, uid, gid); + await chown(runner, uid, gid); +} + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('GitHubRunnerLauncher', () => { + it('launches run.sh directly with the JIT config and a sanitized environment', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-runner-')); + const output = join(directory, 'output'); + const environmentOutput = join(directory, 'environment-output'); + const runner = join(directory, 'run.sh'); + + await writeFile( + runner, + `#!/bin/sh +set -eu +printf '%s|%s|%s' "$1" "$2" "$MICROVM_ID" > "$TEST_RUNNER_OUTPUT" +printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + "\${ENCODED_JIT_CONFIG-unset}" \ + "\${AWS_ACCESS_KEY_ID-unset}" \ + "\${AWS_SESSION_TOKEN-unset}" \ + "\${AWS_CONTAINER_CREDENTIALS_FULL_URI-unset}" \ + "\${AWS_PROFILE-unset}" \ + "\${AWS_DEFAULT_PROFILE-unset}" \ + "\${AWS_CONFIG_FILE-unset}" \ + "\${AWS_SHARED_CREDENTIALS_FILE-unset}" \ + "\${AWS_CREDENTIAL_EXPIRATION-unset}" \ + "\${RUNNER_CONFIG_STORAGE_PROVIDER-unset}" \ + "\${SSM_TOKEN_PATH-unset}" \ + "\${RUNNER_ALLOW_RUNASROOT-unset}" > "$TEST_RUNNER_ENV_OUTPUT" +sleep 0.2 +`, + { mode: 0o700 }, + ); + await prepareRunnerFixture(directory, runner); + + vi.stubEnv('RUNNER_ROOT', directory); + vi.stubEnv('TEST_RUNNER_OUTPUT', output); + vi.stubEnv('TEST_RUNNER_ENV_OUTPUT', environmentOutput); + vi.stubEnv('ENCODED_JIT_CONFIG', 'test-value'); + vi.stubEnv('AWS_ACCESS_KEY_ID', 'test-value'); + vi.stubEnv('AWS_SESSION_TOKEN', 'test-value'); + vi.stubEnv('AWS_CONTAINER_CREDENTIALS_FULL_URI', 'http://127.0.0.1/credentials'); + vi.stubEnv('AWS_PROFILE', 'test-profile'); + vi.stubEnv('AWS_DEFAULT_PROFILE', 'test-profile'); + vi.stubEnv('AWS_CONFIG_FILE', '/tmp/test-config'); + vi.stubEnv('AWS_SHARED_CREDENTIALS_FILE', '/tmp/test-credentials'); + vi.stubEnv('AWS_CREDENTIAL_EXPIRATION', '2099-01-01T00:00:00Z'); + vi.stubEnv('RUNNER_CONFIG_STORAGE_PROVIDER', 'aws_ssm'); + vi.stubEnv('SSM_TOKEN_PATH', '/runner/token'); + vi.stubEnv('RUNNER_ALLOW_RUNASROOT', '1'); + try { + const processHandle = new GitHubRunnerLauncher(30_000, 10).launch({ jitConfig: 'encoded-jit' }, 'mvm-1234'); + + await processHandle.ready; + await expect(processHandle.exit).resolves.toBe(0); + expect(await readFile(output, 'utf8')).toBe('--jitconfig|encoded-jit|mvm-1234'); + expect(await readFile(environmentOutput, 'utf8')).toBe( + 'unset|unset|unset|unset|unset|unset|unset|unset|unset|unset|unset|unset', + ); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); + + it('rejects readiness when run.sh exits before the launch handoff', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-runner-')); + const runner = join(directory, 'run.sh'); + + await writeFile(runner, '#!/bin/sh\nexit 7\n', { mode: 0o700 }); + await prepareRunnerFixture(directory, runner); + vi.stubEnv('RUNNER_ROOT', directory); + try { + const processHandle = new GitHubRunnerLauncher().launch({ jitConfig: 'encoded-jit' }, 'mvm-1234'); + + await expect(processHandle.ready).rejects.toThrow('exited before the launch handoff'); + await expect(processHandle.exit).resolves.toBe(7); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/processes.ts b/lambdas/services/microvm-lifecycle-hooks/src/processes.ts new file mode 100644 index 0000000000..b523e3ff57 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/processes.ts @@ -0,0 +1,274 @@ +import { type ChildProcess, spawn } from 'node:child_process'; +import { chmod, writeFile } from 'node:fs/promises'; +import { arch, type } from 'node:os'; +import { isAbsolute, join } from 'node:path'; + +import type { Logger, ManagedProcess, RunContext, RunnerBootstrap, RunnerLauncher } from './contracts'; +import { delay } from './timing'; + +function safeErrorName(error: unknown): string { + return error instanceof Error && error.name ? error.name : 'UnknownError'; +} + +/** Writes the runner-visible machine information consumed during job setup. */ +export async function writeRunnerSetupInfo(context: RunContext, logger: Logger): Promise { + const runnerRoot = process.env.ACTIONS_RUNNER_ROOT ?? '/opt/actions-runner'; + const setupInfoPath = join(runnerRoot, '.setup_info'); + const setupInfo = [ + { + group: 'Operating System', + detail: `Platform: ${type()}\nArchitecture: ${arch()}`, + }, + ]; + if (context.imageArn !== undefined && context.imageVersion !== undefined) { + setupInfo.push({ + group: 'Runner Image', + detail: `MicroVM image ARN: ${context.imageArn}\nMicroVM image version: ${context.imageVersion}`, + }); + } + setupInfo.push({ + group: 'Lambda MicroVM', + detail: `MicroVM id: ${context.microvmId}`, + }); + + try { + await writeFile(setupInfoPath, `${JSON.stringify(setupInfo, null, 2)}\n`, { + encoding: 'utf8', + mode: 0o644, + }); + await chmod(setupInfoPath, 0o644); + } catch (error) { + // Setup information is informational and must not strand consumed JIT. + logger.warn('GitHub Actions runner setup information could not be written (%s)', safeErrorName(error)); + } +} + +const LAUNCH_HANDOFF_DELAY_MS = 1_000; +const MAX_POSIX_ID = 2_147_483_647; +const ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const ALWAYS_DENIED_RUNNER_ENVIRONMENT = new Set([ + 'ACTIONS_RUNNER_INPUT_JITCONFIG', + 'AWS_ACCESS_KEY_ID', + 'AWS_CONFIG_FILE', + 'AWS_CONTAINER_AUTHORIZATION_TOKEN', + 'AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE', + 'AWS_CONTAINER_CREDENTIALS_FULL_URI', + 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI', + 'AWS_CREDENTIAL_EXPIRATION', + 'AWS_DEFAULT_PROFILE', + 'AWS_PROFILE', + 'AWS_ROLE_ARN', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SECURITY_TOKEN', + 'AWS_SESSION_TOKEN', + 'AWS_SHARED_CREDENTIALS_FILE', + 'AWS_WEB_IDENTITY_TOKEN_FILE', + 'ENCODED_JIT_CONFIG', + 'INTERNAL_SERVICES', + 'JIT_CONFIG', + 'MICROVM_RUNNER_ENV_DENYLIST', + 'RUNNER_ALLOW_RUNASROOT', + 'RUNNER_CONFIG_SSM_ARN', + 'RUNNER_CONFIG_SSM_PATH', + 'RUNNER_CONFIG_STORAGE_PROVIDER', + 'RUNNER_TOKEN_SSM_PATH', + 'SSM_TOKEN_PATH', + 'bootstrap_payload', + 'encoded_jit_config', + 'jit_config', +]); + +function signalProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void { + if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) { + return; + } + try { + process.kill(-child.pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { + child.kill(signal); + } + } +} + +function parsePosixId(variable: string, fallback: number): number { + const value = process.env[variable] ?? String(fallback); + if (!/^\d+$/.test(value)) { + throw new Error(`${variable} must be a positive integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > MAX_POSIX_ID) { + throw new Error(`${variable} must be a positive integer`); + } + return parsed; +} + +function runnerIdentity(): { gid?: number; uid?: number } { + if (process.getuid?.() !== 0) { + return {}; + } + return { + gid: parsePosixId('RUNNER_GID', 1_000), + uid: parsePosixId('RUNNER_UID', 1_000), + }; +} + +function runnerEnvironmentDenylist(): Set { + const configured = process.env.MICROVM_RUNNER_ENV_DENYLIST; + if (configured === undefined || configured.trim() === '') { + return new Set(ALWAYS_DENIED_RUNNER_ENVIRONMENT); + } + + const denylist = new Set(ALWAYS_DENIED_RUNNER_ENVIRONMENT); + for (const name of configured.split(',')) { + const normalized = name.trim(); + if (!ENVIRONMENT_NAME_PATTERN.test(normalized)) { + throw new Error('MICROVM_RUNNER_ENV_DENYLIST contains an invalid environment name'); + } + denylist.add(normalized); + } + return denylist; +} + +function runnerEnvironment(microvmId: string, denylist: ReadonlySet): NodeJS.ProcessEnv { + const environment = { ...process.env }; + for (const name of denylist) { + delete environment[name]; + } + return { + ...environment, + HOME: process.env.RUNNER_HOME ?? '/home/runner', + LOGNAME: process.env.RUNNER_USER ?? 'runner', + MICROVM_ID: microvmId, + USER: process.env.RUNNER_USER ?? 'runner', + }; +} + +function redactSpawnArguments(child: ChildProcess, arguments_: string[], sensitiveValue: string): void { + for (let index = 0; index < arguments_.length; index += 1) { + if (arguments_[index] === sensitiveValue) { + arguments_[index] = '[redacted]'; + } + } + for (let index = 0; index < child.spawnargs.length; index += 1) { + if (child.spawnargs[index] === sensitiveValue) { + child.spawnargs[index] = '[redacted]'; + } + } +} + +function waitForLaunchHandoff(child: ChildProcess, handoffDelayMs: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + let handoffTimer: NodeJS.Timeout | undefined; + + const cleanup = (): void => { + child.off('error', onError); + child.off('spawn', onSpawn); + child.off('exit', onExit); + if (handoffTimer !== undefined) { + clearTimeout(handoffTimer); + } + }; + + const fail = (error: Error): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + reject(error); + }; + + const commit = (): void => { + if (settled) { + return; + } + if (child.exitCode !== null || child.signalCode !== null) { + fail(new Error('GitHub Actions runner exited before the launch handoff')); + return; + } + settled = true; + cleanup(); + child.unref(); + resolve(); + }; + + const onError = (error: Error): void => fail(error); + const onExit = (): void => fail(new Error('GitHub Actions runner exited before the launch handoff')); + const onSpawn = (): void => { + handoffTimer = setTimeout(commit, handoffDelayMs); + }; + + child.once('error', onError); + child.once('spawn', onSpawn); + child.once('exit', onExit); + }); +} + +export class NodeManagedProcess implements ManagedProcess { + public readonly ready: Promise; + public readonly exit: Promise; + + public constructor( + private readonly child: ChildProcess, + readiness: Promise, + private readonly defaultStopGraceMs: number, + ) { + this.ready = readiness; + this.exit = new Promise((resolve) => { + child.once('exit', (code) => resolve(code)); + child.once('error', () => resolve(null)); + }); + } + + public get exited(): boolean { + return this.child.exitCode !== null || this.child.signalCode !== null; + } + + public async stop(graceMs = this.defaultStopGraceMs): Promise { + if (this.exited) { + return; + } + signalProcessGroup(this.child, 'SIGTERM'); + const exitedGracefully = await Promise.race([this.exit.then(() => true), delay(graceMs).then(() => false)]); + if (!exitedGracefully && !this.exited) { + signalProcessGroup(this.child, 'SIGKILL'); + await Promise.race([this.exit, delay(5_000)]); + } + } +} + +/** Launches the GitHub Actions runner directly from the image's runner installation. */ +export class GitHubRunnerLauncher implements RunnerLauncher { + private readonly denylist = runnerEnvironmentDenylist(); + private readonly runnerRoot = process.env.RUNNER_ROOT ?? '/opt/actions-runner'; + private readonly identity = runnerIdentity(); + + public constructor( + private readonly stopGraceMs = 30_000, + private readonly handoffDelayMs = LAUNCH_HANDOFF_DELAY_MS, + ) { + if (!isAbsolute(this.runnerRoot)) { + throw new Error('RUNNER_ROOT must be an absolute path'); + } + } + + public launch(bootstrap: RunnerBootstrap, microvmId: string): ManagedProcess { + const runner = join(this.runnerRoot, 'run.sh'); + const arguments_ = ['--jitconfig', bootstrap.jitConfig]; + const child = spawn(runner, arguments_, { + cwd: this.runnerRoot, + detached: true, + env: runnerEnvironment(microvmId, this.denylist), + shell: false, + stdio: ['ignore', 'inherit', 'inherit'], + ...this.identity, + }); + redactSpawnArguments(child, arguments_, bootstrap.jitConfig); + + const ready = waitForLaunchHandoff(child, this.handoffDelayMs); + return new NodeManagedProcess(child, ready, this.stopGraceMs); + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/public.ts b/lambdas/services/microvm-lifecycle-hooks/src/public.ts new file mode 100644 index 0000000000..a82479fecd --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/public.ts @@ -0,0 +1,26 @@ +export type { + ConsumeOptions, + JitConfigSource, + Logger, + ManagedProcess, + RunContext, + RunnerBootstrap, + RunnerLauncher, +} from './contracts'; +export { consoleLogger } from './contracts'; +export { RunnerLifecycle } from './lifecycle'; +export { HookRequestError, MAX_REQUEST_BYTES, parseRunRequest } from './payload'; +export { GitHubRunnerLauncher, NodeManagedProcess } from './processes'; +export { + createHookExitRequester, + createDefaultLifecycle, + createHookServer, + HOOK_PREFIX, + main, + parsePositiveInteger, + shutdownHookServer, + watchRunnerCompletion, +} from './server'; +export type { HookLifecycle, HookServerOptions } from './server'; +export { StorageJitConfigSource } from './storage'; +export type { StorageJitConfigSourceOptions } from './storage'; diff --git a/lambdas/services/microvm-lifecycle-hooks/src/server.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/server.test.ts new file mode 100644 index 0000000000..823955d755 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/server.test.ts @@ -0,0 +1,210 @@ +import type { AddressInfo } from 'node:net'; + +import type { Logger } from './contracts'; +import { + createHookExitRequester, + createHookServer, + type HookLifecycle, + HOOK_PREFIX, + parsePositiveInteger, + shutdownHookServer, + watchRunnerCompletion, +} from './server'; + +const quietLogger: Logger = { + error: () => undefined, + info: () => undefined, + warn: () => undefined, +}; + +const idleLifecycle: HookLifecycle = { + resume: async () => true, + start: async () => true, + stop: async () => undefined, +}; + +async function listen(server: ReturnType): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +async function close(server: ReturnType): Promise { + await new Promise((resolve, reject) => { + server.close((error) => (error === undefined ? resolve() : reject(error))); + server.closeAllConnections(); + }); +} + +describe('hook server', () => { + it('rejects invalid and out-of-range positive integer values', () => { + expect(parsePositiveInteger(undefined, 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('0', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('-1', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('1.5', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('8080http', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('65536', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('9007199254740992', 8080)).toBe(8080); + expect(parsePositiveInteger('9090', 8080, 65_535)).toBe(9090); + }); + + it('configures bounded request, header, connection, and socket limits', () => { + const server = createHookServer(idleLifecycle, quietLogger, { + headersTimeoutMs: 2_000, + keepAliveTimeoutMs: 3_000, + requestTimeoutMs: 4_000, + }); + + expect(server.headersTimeout).toBe(2_000); + expect(server.keepAliveTimeout).toBe(3_000); + expect(server.requestTimeout).toBe(4_000); + expect(server.maxConnections).toBe(128); + expect(server.maxHeadersCount).toBe(64); + expect(server.maxRequestsPerSocket).toBe(100); + }); + + it('acknowledges build hooks without starting a runner', async () => { + const lifecycle: HookLifecycle = { + resume: vi.fn(), + start: vi.fn(), + stop: vi.fn(), + }; + const server = createHookServer(lifecycle, quietLogger); + const baseUrl = await listen(server); + try { + const ready = await fetch(`${baseUrl}${HOOK_PREFIX}/ready`, { method: 'POST' }); + const validate = await fetch(`${baseUrl}${HOOK_PREFIX}/validate`, { method: 'POST' }); + + expect(ready.status).toBe(200); + await expect(ready.json()).resolves.toEqual({ status: 'ready' }); + expect(validate.status).toBe(200); + await expect(validate.json()).resolves.toEqual({ status: 'validated' }); + expect(lifecycle.start).not.toHaveBeenCalled(); + expect(lifecycle.stop).not.toHaveBeenCalled(); + } finally { + await close(server); + } + }); + + it('rejects oversized request bodies before invoking the lifecycle', async () => { + const lifecycle: HookLifecycle = { + ...idleLifecycle, + start: vi.fn(), + }; + const server = createHookServer(lifecycle, quietLogger); + const baseUrl = await listen(server); + try { + const response = await fetch(`${baseUrl}${HOOK_PREFIX}/run`, { + body: 'x'.repeat(20 * 1024 + 1), + method: 'POST', + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toEqual({ error: 'request body is too large' }); + expect(lifecycle.start).not.toHaveBeenCalled(); + } finally { + await close(server); + } + }); + + it('does not reflect or log secret-bearing internal errors', async () => { + const messages: unknown[] = []; + const logger: Logger = { + error: (...values) => messages.push(...values), + info: () => undefined, + warn: () => undefined, + }; + const lifecycle: HookLifecycle = { + ...idleLifecycle, + start: async () => { + const error = new Error('encoded-jit-secret'); + error.name = 'encoded-jit-secret'; + throw error; + }, + }; + const server = createHookServer(lifecycle, logger); + const baseUrl = await listen(server); + try { + const response = await fetch(`${baseUrl}${HOOK_PREFIX}/run`, { + body: '{}', + method: 'POST', + }); + + expect(response.status).toBe(500); + await expect(response.json()).resolves.toEqual({ error: 'lifecycle hook failed' }); + expect(JSON.stringify(messages)).not.toContain('encoded-jit-secret'); + } finally { + await close(server); + } + }); + + it('waits for lifecycle cleanup before closing active connections', async () => { + const events: string[] = []; + let finishCleanup = (): void => undefined; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + const server = { + close: () => events.push('stop-accepting'), + closeAllConnections: () => events.push('close-connections'), + }; + const lifecycle = { + async stop(): Promise { + events.push('cleanup-started'); + await cleanup; + events.push('cleanup-finished'); + }, + }; + + const shutdown = shutdownHookServer(server, lifecycle); + await new Promise((resolve) => setImmediate(resolve)); + expect(events).toEqual(['stop-accepting', 'cleanup-started']); + + finishCleanup(); + await shutdown; + expect(events).toEqual(['stop-accepting', 'cleanup-started', 'cleanup-finished', 'close-connections']); + }); + + it.each([ + { expectedExitCode: 0, runnerExitCode: 0 }, + { expectedExitCode: 1, runnerExitCode: 7 }, + { expectedExitCode: 1, runnerExitCode: null }, + ])('requests hook exit $expectedExitCode after runner status $runnerExitCode', async (testCase) => { + const requestExit = vi.fn(); + + watchRunnerCompletion({ completion: Promise.resolve(testCase.runnerExitCode) }, quietLogger, requestExit); + + await Promise.resolve(); + expect(requestExit).not.toHaveBeenCalled(); + await new Promise((resolve) => setImmediate(resolve)); + expect(requestExit).toHaveBeenCalledOnce(); + expect(requestExit).toHaveBeenCalledWith(testCase.expectedExitCode); + }); + + it('closes the hook exactly once before publishing its process exit code', async () => { + const events: string[] = []; + const requestExit = createHookExitRequester( + { + close: () => events.push('stop-accepting'), + closeAllConnections: () => events.push('close-connections'), + }, + { + async stop(): Promise { + events.push('stop-runner'); + }, + }, + quietLogger, + (exitCode) => events.push(`exit:${exitCode}`), + ); + + requestExit(0); + requestExit(1); + await new Promise((resolve) => setImmediate(resolve)); + + expect(events).toEqual(['stop-accepting', 'stop-runner', 'close-connections', 'exit:0']); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/server.ts b/lambdas/services/microvm-lifecycle-hooks/src/server.ts new file mode 100644 index 0000000000..c37fd39886 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/server.ts @@ -0,0 +1,275 @@ +import http, { type IncomingMessage, type ServerResponse } from 'node:http'; + +import type { Logger } from './contracts'; +import { consoleLogger } from './contracts'; +import { RunnerLifecycle } from './lifecycle'; +import { HookRequestError, MAX_REQUEST_BYTES } from './payload'; +import { GitHubRunnerLauncher } from './processes'; +import { StorageJitConfigSource } from './storage'; + +export const HOOK_PREFIX = '/aws/lambda-microvms/runtime/v1'; + +const MAX_TIMER_SECONDS = 2_147_483; + +export interface HookLifecycle { + start(body: string): Promise; + stop(): Promise; + resume(): Promise; +} + +export interface HookServerOptions { + headersTimeoutMs?: number; + keepAliveTimeoutMs?: number; + requestTimeoutMs?: number; +} + +export function parsePositiveInteger( + value: string | undefined, + fallback: number, + maximum = Number.MAX_SAFE_INTEGER, +): number { + if (value === undefined || !/^\d+$/.test(value)) { + return fallback; + } + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= maximum ? parsed : fallback; +} + +function timeoutMilliseconds(variable: string, fallbackSeconds: number, maximumSeconds = 60): number { + return ( + parsePositiveInteger(process.env[variable], fallbackSeconds, Math.min(maximumSeconds, MAX_TIMER_SECONDS)) * 1_000 + ); +} + +function respond(response: ServerResponse, status: number, payload: object): void { + const body = Buffer.from(JSON.stringify(payload)); + response.writeHead(status, { + 'Cache-Control': 'no-store', + 'Content-Length': body.length, + 'Content-Type': 'application/json', + }); + response.end(body); +} + +function readBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const contentLength = request.headers['content-length']; + let declaredLength: number | undefined; + if (contentLength !== undefined) { + declaredLength = Number(contentLength); + if (!Number.isInteger(declaredLength) || declaredLength < 0) { + reject(new HookRequestError('Content-Length is invalid')); + request.resume(); + return; + } + if (declaredLength > MAX_REQUEST_BYTES) { + reject(new HookRequestError('request body is too large')); + request.resume(); + return; + } + } + + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + + const fail = (error: Error): void => { + if (settled) { + return; + } + settled = true; + reject(error); + }; + request.on('data', (chunk: Buffer) => { + if (settled) { + return; + } + size += chunk.length; + if (size > MAX_REQUEST_BYTES) { + fail(new HookRequestError('request body is too large')); + request.destroy(); + return; + } + chunks.push(chunk); + }); + request.once('end', () => { + if (settled) { + return; + } + if (declaredLength !== undefined && declaredLength !== size) { + fail(new HookRequestError('Content-Length does not match the request body')); + return; + } + settled = true; + resolve(Buffer.concat(chunks).toString('utf8')); + }); + request.once('aborted', () => fail(new HookRequestError('request body was interrupted'))); + request.once('error', (error) => fail(error)); + }); +} + +export function createHookServer( + lifecycle: HookLifecycle, + logger: Logger = consoleLogger, + options: HookServerOptions = {}, +): http.Server { + const requestTimeout = options.requestTimeoutMs ?? timeoutMilliseconds('HOOK_REQUEST_TIMEOUT_SECONDS', 10); + const headersTimeout = Math.min( + options.headersTimeoutMs ?? timeoutMilliseconds('HOOK_HEADERS_TIMEOUT_SECONDS', 5), + requestTimeout, + ); + const keepAliveTimeout = options.keepAliveTimeoutMs ?? timeoutMilliseconds('HOOK_KEEP_ALIVE_TIMEOUT_SECONDS', 5); + + const server = http.createServer( + { + headersTimeout, + keepAliveTimeout, + maxHeaderSize: 16 * 1024, + requestTimeout, + }, + async (request, response) => { + const path = request.url ?? ''; + if (request.method !== 'POST') { + request.resume(); + respond(response, 405, { error: 'method not allowed' }); + return; + } + + try { + // Consume every POST body so all lifecycle endpoints share the same bounded request handling. + const body = await readBody(request); + if (path === `${HOOK_PREFIX}/ready`) { + respond(response, 200, { status: 'ready' }); + return; + } + if (path === `${HOOK_PREFIX}/validate`) { + respond(response, 200, { status: 'validated' }); + return; + } + if (path === `${HOOK_PREFIX}/run`) { + const started = await lifecycle.start(body); + respond(response, 200, { status: started ? 'started' : 'already-started' }); + return; + } + if (path === `${HOOK_PREFIX}/terminate`) { + await lifecycle.stop(); + respond(response, 200, { status: 'stopped' }); + return; + } + if (path === `${HOOK_PREFIX}/resume`) { + const ready = await lifecycle.resume(); + respond(response, ready ? 200 : 503, { status: ready ? 'ready' : 'not-ready' }); + return; + } + if (path === `${HOOK_PREFIX}/suspend`) { + respond(response, 200, { status: 'ok' }); + return; + } + respond(response, 404, { error: 'unknown lifecycle hook' }); + } catch (error) { + if (error instanceof HookRequestError) { + logger.warn('Rejected invalid lifecycle hook request'); + respond(response, 400, { error: error.message }); + return; + } + // Parse and provider errors can contain credentials in both message and name. + logger.error('Lifecycle hook failed'); + respond(response, 500, { error: 'lifecycle hook failed' }); + } + }, + ); + server.maxConnections = 128; + server.maxHeadersCount = 64; + server.maxRequestsPerSocket = 100; + return server; +} + +export function createDefaultLifecycle(logger: Logger = consoleLogger): RunnerLifecycle { + return new RunnerLifecycle(new StorageJitConfigSource(), new GitHubRunnerLauncher(), logger); +} + +interface ClosableServer { + close(): unknown; + closeAllConnections(): void; +} + +interface StoppableLifecycle { + stop(): Promise; +} + +export async function shutdownHookServer(server: ClosableServer, lifecycle: StoppableLifecycle): Promise { + server.close(); + try { + await lifecycle.stop(); + } finally { + server.closeAllConnections(); + } +} + +function hookExitCode(runnerExitCode: number | null): number { + return runnerExitCode === 0 ? 0 : 1; +} + +export function watchRunnerCompletion( + lifecycle: Pick, + logger: Logger, + requestExit: (exitCode: number) => void, +): void { + void lifecycle.completion.then((runnerExitCode) => { + const exitCode = hookExitCode(runnerExitCode); + if (exitCode === 0) { + logger.info('GitHub Actions runner exited with status %s', runnerExitCode); + } else { + logger.error('GitHub Actions runner exited unexpectedly with status %s', runnerExitCode ?? 'signal'); + } + // Let the /run handler flush its acknowledgement if the runner exits immediately after handoff. + setImmediate(() => requestExit(exitCode)); + }); +} + +export function createHookExitRequester( + server: ClosableServer, + lifecycle: StoppableLifecycle, + logger: Logger, + setExitCode: (exitCode: number) => void = (exitCode) => { + // Let Node exit naturally after lifecycle cleanup and log streams have drained. + process.exitCode = exitCode; + }, +): (exitCode: number) => void { + let exiting = false; + return (exitCode: number): void => { + if (exiting) { + return; + } + exiting = true; + void shutdownHookServer(server, lifecycle).then( + () => setExitCode(exitCode), + () => { + logger.error('Lifecycle hook shutdown failed'); + setExitCode(1); + }, + ); + }; +} + +export async function main(): Promise { + const logger = consoleLogger; + const lifecycle = createDefaultLifecycle(logger); + const server = createHookServer(lifecycle, logger); + const port = parsePositiveInteger(process.env.HOOK_PORT, 8080, 65_535); + + const requestExit = createHookExitRequester(server, lifecycle, logger); + process.once('SIGINT', () => requestExit(0)); + process.once('SIGTERM', () => requestExit(0)); + watchRunnerCompletion(lifecycle, logger, requestExit); + + await new Promise((resolve, reject) => { + const onError = (): void => reject(new Error('lifecycle hook server could not listen')); + server.once('error', onError); + server.listen(port, '0.0.0.0', () => { + server.off('error', onError); + logger.info('Lambda MicroVM lifecycle hooks listening on port %d', port); + resolve(); + }); + }); +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/storage.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/storage.test.ts new file mode 100644 index 0000000000..d1f2ad3400 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/storage.test.ts @@ -0,0 +1,87 @@ +import type { + RunnerConfigConsumer, + RunnerConfigStorageContext, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; + +import { StorageJitConfigSource } from './storage'; + +const SSM_STORAGE: RunnerConfigStorageContext = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', +}; + +describe('StorageJitConfigSource', () => { + it('exports the allowlisted context once before resolving and consuming from the environment', async () => { + const events: string[] = []; + const environment: NodeJS.ProcessEnv = {}; + const consumer: RunnerConfigConsumer = { + consume: vi.fn(async () => { + events.push('consume'); + return 'encoded-jit'; + }), + }; + const exportEnvironment = vi.fn((context: RunnerConfigStorageContext) => { + events.push('export'); + return context; + }); + const createConsumer = vi.fn((target: NodeJS.ProcessEnv) => { + events.push('create'); + expect(target).toBe(environment); + expect(target).toMatchObject(SSM_STORAGE); + return consumer; + }); + const source = new StorageJitConfigSource({ createConsumer, environment, exportEnvironment }); + const signal = new AbortController().signal; + + await expect( + source.consume({ microvmId: 'microvm-1234', storage: SSM_STORAGE }, { deadlineMs: 123_456, signal }), + ).resolves.toEqual({ jitConfig: 'encoded-jit' }); + await expect( + source.consume( + { + microvmId: 'microvm-1234', + storage: { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', + }, + }, + { deadlineMs: 123_457, signal }, + ), + ).resolves.toEqual({ jitConfig: 'encoded-jit' }); + + expect(events).toEqual(['export', 'create', 'consume', 'create', 'consume']); + expect(exportEnvironment).toHaveBeenCalledOnce(); + expect(exportEnvironment).toHaveBeenCalledWith(SSM_STORAGE); + expect(createConsumer).toHaveBeenCalledTimes(2); + expect(consumer.consume).toHaveBeenNthCalledWith(1, 'microvm-1234', { + deadlineMs: 123_456, + signal, + }); + }); + + it('rejects storage context changes after the one-time environment export', async () => { + const environment: NodeJS.ProcessEnv = {}; + const consumer: RunnerConfigConsumer = { consume: vi.fn().mockResolvedValue('encoded-jit') }; + const exportEnvironment = vi.fn((context: RunnerConfigStorageContext) => context); + const createConsumer = vi.fn().mockReturnValue(consumer); + const source = new StorageJitConfigSource({ createConsumer, environment, exportEnvironment }); + const options = { deadlineMs: 123_456, signal: new AbortController().signal }; + + await source.consume({ microvmId: 'microvm-1234', storage: SSM_STORAGE }, options); + await expect( + source.consume( + { + microvmId: 'microvm-1234', + storage: { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/other/token', + }, + }, + options, + ), + ).rejects.toThrow('storage context cannot change'); + + expect(exportEnvironment).toHaveBeenCalledOnce(); + expect(createConsumer).toHaveBeenCalledOnce(); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/storage.ts b/lambdas/services/microvm-lifecycle-hooks/src/storage.ts new file mode 100644 index 0000000000..528683f59e --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/storage.ts @@ -0,0 +1,49 @@ +import { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + type RunnerConfigConsumer, + type RunnerConfigStorageContext, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; + +import type { ConsumeOptions, JitConfigSource, RunContext, RunnerBootstrap } from './contracts'; + +type RunnerConfigConsumerFactory = typeof createRunnerConfigConsumerFromEnvironment; +type RunnerConfigStorageExporter = typeof exportRunnerConfigStorageEnvironment; + +export interface StorageJitConfigSourceOptions { + createConsumer?: RunnerConfigConsumerFactory; + environment?: NodeJS.ProcessEnv; + exportEnvironment?: RunnerConfigStorageExporter; +} + +function storageContextFingerprint(context: RunnerConfigStorageContext): string { + return JSON.stringify(Object.entries(context).sort(([left], [right]) => left.localeCompare(right))); +} + +/** Adapts the shared provider registry to the lifecycle's one-time bootstrap contract. */ +export class StorageJitConfigSource implements JitConfigSource { + private readonly createConsumer: RunnerConfigConsumerFactory; + private readonly environment: NodeJS.ProcessEnv; + private readonly exportEnvironment: RunnerConfigStorageExporter; + private exportedStorageFingerprint?: string; + + public constructor(options: StorageJitConfigSourceOptions = {}) { + this.createConsumer = options.createConsumer ?? createRunnerConfigConsumerFromEnvironment; + this.environment = options.environment ?? process.env; + this.exportEnvironment = options.exportEnvironment ?? exportRunnerConfigStorageEnvironment; + } + + public async consume(context: RunContext, options: ConsumeOptions): Promise { + const fingerprint = storageContextFingerprint(context.storage); + if (this.exportedStorageFingerprint === undefined) { + Object.assign(this.environment, this.exportEnvironment(context.storage)); + this.exportedStorageFingerprint = fingerprint; + } else if (this.exportedStorageFingerprint !== fingerprint) { + throw new Error('runner configuration storage context cannot change after initialization'); + } + + const consumer: RunnerConfigConsumer = this.createConsumer(this.environment); + const jitConfig = await consumer.consume(context.microvmId, options); + return { jitConfig }; + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/timing.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/timing.test.ts new file mode 100644 index 0000000000..b62d8af038 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/timing.test.ts @@ -0,0 +1,32 @@ +import { beforeDeadlineOrAbort, delay } from './timing'; + +describe('timing helpers', () => { + it('removes the delay abort listener after resolving', async () => { + const signal = new AbortController().signal; + const remove = vi.spyOn(signal, 'removeEventListener'); + + await delay(1, signal); + + expect(remove).toHaveBeenCalledOnce(); + }); + + it('removes the delay abort listener after cancellation', async () => { + const controller = new AbortController(); + const remove = vi.spyOn(controller.signal, 'removeEventListener'); + const pending = delay(1_000, controller.signal); + + controller.abort(); + + await expect(pending).rejects.toThrow('operation was cancelled'); + expect(remove).toHaveBeenCalledOnce(); + }); + + it('rejects immediately when an operation is already aborted', async () => { + const controller = new AbortController(); + controller.abort(); + + await expect( + beforeDeadlineOrAbort(Promise.resolve('unused'), Date.now() + 1_000, controller.signal), + ).rejects.toThrow('runner start was cancelled'); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/timing.ts b/lambdas/services/microvm-lifecycle-hooks/src/timing.ts new file mode 100644 index 0000000000..fc791d2e29 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/timing.ts @@ -0,0 +1,66 @@ +export function delay(milliseconds: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => signal?.removeEventListener('abort', cancel); + const finish = (): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + resolve(); + }; + const timer = setTimeout(finish, milliseconds); + const cancel = (): void => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + cleanup(); + reject(new Error('operation was cancelled')); + }; + signal?.addEventListener('abort', cancel, { once: true }); + if (signal?.aborted) { + cancel(); + } + }); +} + +export async function beforeDeadline(promise: Promise, deadlineMs: number): Promise { + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) { + throw new Error('run-hook deadline elapsed'); + } + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('run-hook deadline elapsed')), remaining); + }); + try { + return await Promise.race([promise, deadline]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } +} + +export async function beforeDeadlineOrAbort( + promise: Promise, + deadlineMs: number, + signal: AbortSignal, +): Promise { + if (signal.aborted) { + throw new Error('runner start was cancelled'); + } + let cancel = (): void => undefined; + const cancelled = new Promise((_resolve, reject) => { + cancel = (): void => reject(new Error('runner start was cancelled')); + signal.addEventListener('abort', cancel, { once: true }); + }); + try { + return await beforeDeadline(Promise.race([promise, cancelled]), deadlineMs); + } finally { + signal.removeEventListener('abort', cancel); + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/tsconfig.json b/lambdas/services/microvm-lifecycle-hooks/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/services/microvm-lifecycle-hooks/vitest.config.ts b/lambdas/services/microvm-lifecycle-hooks/vitest.config.ts new file mode 100644 index 0000000000..e3c59146ee --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/vitest.config.ts @@ -0,0 +1,12 @@ +import { mergeConfig } from 'vitest/config'; + +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts'], + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 88cf3bab55..c05df8a661 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -201,6 +201,16 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/microvm-lifecycle-hooks@workspace:services/microvm-lifecycle-hooks": + version: 0.0.0-use.local + resolution: "@aws-github-runner/microvm-lifecycle-hooks@workspace:services/microvm-lifecycle-hooks" + dependencies: + "@aws-github-runner/storage-providers": "npm:*" + "@types/node": "npm:^22.19.3" + esbuild: "npm:^0.27.0" + languageName: unknown + linkType: soft + "@aws-github-runner/storage-providers@npm:*, @aws-github-runner/storage-providers@workspace:libs/storage-providers": version: 0.0.0-use.local resolution: "@aws-github-runner/storage-providers@workspace:libs/storage-providers" diff --git a/mkdocs.yaml b/mkdocs.yaml index 81b4992a9e..289ca9b955 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -60,6 +60,7 @@ nav: - Multi-runner v1 to v2 migration: multi-runner-v1-v2-migration.md - Getting started: getting-started.md - Security: security.md + - Lambda MicroVM runners (experimental): microvm-runners.md - Architecture decisions: - MiniStack for integration tests: adr/0001-use-ministack-for-terraform-integration-tests.md - Modules: @@ -80,7 +81,7 @@ nav: - Overview: examples/index.md - Default: examples/default.md - Multi Runner: examples/multi-runner.md - - Multi Runner v2: examples/multi-runner-v2.md + - Multi Runner Webhook: examples/multi-runner-webhook.md - Ephemeral: examples/ephemeral.md - External managed secrets: examples/external-managed-ssm-secrets.md - Custom AMI: examples/prebuilt.md diff --git a/modules/microvm-foundation/README.md b/modules/microvm-foundation/README.md index d87227c89f..dde8e4a2a1 100644 --- a/modules/microvm-foundation/README.md +++ b/modules/microvm-foundation/README.md @@ -12,6 +12,24 @@ It manages: - A Lambda-trusted Network Connector operator role and propagation barrier. - An unattached runtime usage policy for the reserved image namespace and connector inventory. +## Build role and execution role + +MicroVM deployments use two different IAM roles with different lifecycles: + +- The `build_role_arn` output is the **build role**. The image builder assumes + this role while it creates and publishes a MicroVM image. It grants the + image-build permissions for the foundation artifact bucket, logs, and any + configured ECR repositories. It is not the role used by a runner job. +- The **execution role** is attached to each MicroVM when the runner control + plane launches it. The control-plane TypeScript passes this role to + `RunMicrovm`; the Lambda that calls that API must be allowed to pass the + role. The MicroVM and the ephemeral runner use this role at runtime. + +The execution role is resolved by the runner configuration and is intentionally +not created by this foundation module. The foundation creates the regional +build resources and the reusable `usage_policy_arn`; the runner/control-plane +configuration owns the runtime role and its provider-specific permissions. + The module does not create MicroVM images, runner execution roles, or the runner control plane. Attach `usage_policy_arn` to the control-plane role that owns the runtime launch operations. The caller must also grant the Terraform @@ -114,6 +132,7 @@ No modules. | [build\_policy\_name\_prefix](#input\_build\_policy\_name\_prefix) | Name prefix for the Lambda MicroVM build policy. | `string` | n/a | yes | | [build\_role\_name\_prefix](#input\_build\_role\_name\_prefix) | Name prefix for the Lambda MicroVM build role. | `string` | n/a | yes | | [ecr\_repository\_arns](#input\_ecr\_repository\_arns) | Optional regional ECR repository ARNs from which MicroVM image builds can pull runner base images. | `set(string)` | `[]` | no | +| [force\_destroy\_artifact\_bucket](#input\_force\_destroy\_artifact\_bucket) | Whether to force destroy the S3 bucket containing Lambda MicroVM image source artifacts. | `bool` | `false` | no | | [image\_name\_prefix](#input\_image\_name\_prefix) | IAM namespace prefix reserved for externally published Lambda MicroVM image names. This module does not create or enumerate images. | `string` | n/a | yes | | [network\_connector\_operator\_role\_name\_prefix](#input\_network\_connector\_operator\_role\_name\_prefix) | Name prefix for the Lambda Network Connector operator role. | `string` | n/a | yes | | [network\_connectors](#input\_network\_connectors) | Regional Lambda MicroVM Network Connectors keyed by a stable consumer-defined identity. | 
map(object({
    name             = string
    vpc_id           = string
    subnet_ids       = list(string)
    network_protocol = optional(string, "IPv4")
  }))
| n/a | yes | diff --git a/modules/microvm-foundation/storage.tf b/modules/microvm-foundation/storage.tf index 73ff52e172..fb717f0697 100644 --- a/modules/microvm-foundation/storage.tf +++ b/modules/microvm-foundation/storage.tf @@ -8,6 +8,8 @@ resource "aws_s3_bucket" "artifacts" { #checkov:skip=CKV2_AWS_62:The publisher uploads artifacts synchronously and no event-driven consumer requires S3 notifications. bucket = var.artifact_bucket_name tags = var.tags + + force_destroy = var.force_destroy_artifact_bucket } resource "aws_s3_bucket_ownership_controls" "artifacts" { diff --git a/modules/microvm-foundation/variables.tf b/modules/microvm-foundation/variables.tf index c21e6068c0..e2d5ae2fa5 100644 --- a/modules/microvm-foundation/variables.tf +++ b/modules/microvm-foundation/variables.tf @@ -13,8 +13,8 @@ variable "build_policy_name_prefix" { description = "Name prefix for the Lambda MicroVM build policy." validation { - condition = length(var.build_policy_name_prefix) >= 1 && length(var.build_policy_name_prefix) <= 64 && can(regex("^[a-zA-Z0-9-_]+-$", var.build_policy_name_prefix)) - error_message = "build_policy_name_prefix must be 1 to 64 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." + condition = length(var.build_policy_name_prefix) >= 1 && length(var.build_policy_name_prefix) <= 38 && can(regex("^[a-zA-Z0-9-_]+-$", var.build_policy_name_prefix)) + error_message = "build_policy_name_prefix must be 1 to 38 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." } } @@ -23,8 +23,8 @@ variable "usage_policy_name_prefix" { description = "Name prefix for the Lambda MicroVM runtime usage policy." validation { - condition = length(var.usage_policy_name_prefix) >= 1 && length(var.usage_policy_name_prefix) <= 64 && can(regex("^[a-zA-Z0-9-_]+-$", var.usage_policy_name_prefix)) - error_message = "usage_policy_name_prefix must be 1 to 64 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." + condition = length(var.usage_policy_name_prefix) >= 1 && length(var.usage_policy_name_prefix) <= 38 && can(regex("^[a-zA-Z0-9-_]+-$", var.usage_policy_name_prefix)) + error_message = "usage_policy_name_prefix must be 1 to 38 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." } } @@ -33,8 +33,8 @@ variable "build_role_name_prefix" { description = "Name prefix for the Lambda MicroVM build role." validation { - condition = length(var.build_role_name_prefix) >= 1 && length(var.build_role_name_prefix) <= 64 && can(regex("^[a-zA-Z0-9-_]+-$", var.build_role_name_prefix)) - error_message = "build_role_name_prefix must be 1 to 64 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." + condition = length(var.build_role_name_prefix) >= 1 && length(var.build_role_name_prefix) <= 38 && can(regex("^[a-zA-Z0-9-_]+-$", var.build_role_name_prefix)) + error_message = "build_role_name_prefix must be 1 to 38 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." } } @@ -43,8 +43,8 @@ variable "network_connector_operator_role_name_prefix" { description = "Name prefix for the Lambda Network Connector operator role." validation { - condition = length(var.network_connector_operator_role_name_prefix) >= 1 && length(var.network_connector_operator_role_name_prefix) <= 64 && can(regex("^[a-zA-Z0-9-_]+-$", var.network_connector_operator_role_name_prefix)) - error_message = "network_connector_operator_role_name_prefix must be 1 to 64 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." + condition = length(var.network_connector_operator_role_name_prefix) >= 1 && length(var.network_connector_operator_role_name_prefix) <= 38 && can(regex("^[a-zA-Z0-9-_]+-$", var.network_connector_operator_role_name_prefix)) + error_message = "network_connector_operator_role_name_prefix must be 1 to 38 characters, contain only letters, numbers, hyphens, or underscores, and end with a hyphen." } } @@ -111,11 +111,11 @@ variable "network_connectors" { condition = alltrue([ for connector in values(var.network_connectors) : ( length(connector.name) >= 1 - && length(connector.name) <= 64 + && length(connector.name) <= 38 && can(regex("^[a-zA-Z0-9_-]+$", connector.name)) ) ]) - error_message = "Each network connector name must contain only letters, numbers, hyphens, or underscores and be at most 64 characters." + error_message = "Each network connector name must contain only letters, numbers, hyphens, or underscores and be at most 38 characters." } validation { @@ -152,3 +152,9 @@ variable "network_connectors" { error_message = "Each network connector network_protocol must be IPv4 or DualStack." } } + +variable "force_destroy_artifact_bucket" { + type = bool + description = "Whether to force destroy the S3 bucket containing Lambda MicroVM image source artifacts." + default = false +} \ No newline at end of file diff --git a/modules/multi-runner/tests/config-resolution.tftest.hcl b/modules/multi-runner/tests/config-resolution.tftest.hcl index 99a0e6a9d0..7b5d8e229c 100644 --- a/modules/multi-runner/tests/config-resolution.tftest.hcl +++ b/modules/multi-runner/tests/config-resolution.tftest.hcl @@ -639,6 +639,7 @@ run "v2_microvm_inputs_route_to_microvm_provider" { && local.resolved_config.multi_runner_config["microvm"].compute_provider.aws.ec2 == null && local.resolved_config.multi_runner_config["microvm"].compute_provider.aws.microvm.image_arn == "arn:aws:lambda:eu-west-1:123456789012:microvm-image:global" && local.resolved_config.multi_runner_config["microvm"].compute_provider.aws.microvm.image_version == "8" + && local.runner_matcher_config["microvm"].computeProvider == "microvm" && local.effective_config.orchestration_provider.webhook.lambda.webhook.artifact.s3.key == "global-webhook.zip" ) error_message = "Experimental MicroVM lanes must resolve Linux ARM64 settings, inherit global provider values, and place the webhook artifact key under lambda.webhook.artifact." @@ -654,4 +655,4 @@ run "v2_microvm_inputs_route_to_microvm_provider" { ) error_message = "Experimental MicroVM lanes must route through module.runner_configs and expose the MicroVM provider contract without an EC2 provider." } -} \ No newline at end of file +} diff --git a/modules/multi-runner/webhook.tf b/modules/multi-runner/webhook.tf index 250af4a214..5a6abeff65 100644 --- a/modules/multi-runner/webhook.tf +++ b/modules/multi-runner/webhook.tf @@ -6,9 +6,12 @@ locals { runner_matcher_config = { for k, v in local.webhook_runner_config : k => { - id = aws_sqs_queue.queued_builds[k].id - arn = aws_sqs_queue.queued_builds[k].arn - computeProvider = "ec2" + id = aws_sqs_queue.queued_builds[k].id + arn = aws_sqs_queue.queued_builds[k].arn + computeProvider = one([ + for provider_type, provider_config in v.compute_provider.aws : provider_type + if provider_config != null + ]) matcherConfig = { labelMatchers = v.orchestration_provider.webhook.matcherConfig.labelMatchers exactMatch = v.orchestration_provider.webhook.matcherConfig.exactMatch diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-down.tf b/modules/orchestration-providers/webhook/scale-runners/scale-down.tf index efe57570f6..b5a82fbe01 100644 --- a/modules/orchestration-providers/webhook/scale-runners/scale-down.tf +++ b/modules/orchestration-providers/webhook/scale-runners/scale-down.tf @@ -36,6 +36,7 @@ resource "aws_lambda_function" "scale_down" { PARAMETER_GITHUB_APP_ID_NAME = var.config.github.app_parameters.id.name PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github.app_parameters.key_base64.name PARAMETER_GITHUB_APPS_MANIFEST_NAME = var.config.github.app_parameters.additional_apps_manifest != null ? var.config.github.app_parameters.additional_apps_manifest.name : "" + SSM_TOKEN_PATH = var.storage_provider.aws.ssm.token_path }, var.storage_provider.scale_down.environment_variables) } diff --git a/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl b/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl index 22b695aac2..9cb752cf72 100644 --- a/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl +++ b/modules/orchestration-providers/webhook/scale-runners/tests/scale-runners.tftest.hcl @@ -237,6 +237,7 @@ run "assembles_provider_neutral_scaling_control_plane" { && aws_lambda_function.scale_down.environment[0].variables["RUNNER_BOOT_TIME_IN_MINUTES"] == "12" && aws_lambda_function.scale_up.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" && aws_lambda_function.scale_down.environment[0].variables["MICROVM_CLUSTER"] == "runner-cluster" + && aws_lambda_function.scale_down.environment[0].variables["SSM_TOKEN_PATH"] == "/github-runner/tokens" && !contains(keys(aws_lambda_function.scale_up.environment[0].variables), "INSTANCE_TYPES") ) error_message = "The common scaling Lambdas must select the compute provider while injecting webhook-owned capacity and boot-time settings." diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index ff7c91dff8..344199645d 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -47,6 +47,7 @@ resource "aws_lambda_function" "scale_down" { POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error COMPUTE_PROVIDER_TYPE = "ec2" + SSM_TOKEN_PATH = local.token_path } } diff --git a/modules/webhook/README.md b/modules/webhook/README.md index b4a7d3ba28..043f0cca8e 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -88,7 +88,7 @@ yarn run dist | [repository\_white\_list](#input\_repository\_white\_list) | List of github repository full names (owner/repo\_name) that will be allowed to use the github app. Leave empty for no filtering. | `list(string)` | `[]` | no | | [role\_path](#input\_role\_path) | The path that will be added to the role; if not set, the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | -| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `allowed_keys = []`, `blocked_keys = []` (cannot be used together with `allowed_keys`), and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn             = string
    id              = string
    computeProvider = optional(string, "ec2")
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy = optional(object({
        allowed_keys = optional(list(string), [])
        blocked_keys = optional(list(string), [])
        restricted_keys = optional(map(object({
          allowed = optional(list(string), [])
          denied  = optional(list(string), [])
          max     = optional(string, null)
        })), {})
      }), null)
    })
  }))
| n/a | yes | +| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2` and identifies the compute provider that owns the queue. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type` or `image-version` for `ghr-microvm-image-version`. | 
map(object({
    arn             = string
    id              = string
    computeProvider = optional(string, "ec2")
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy = optional(object({
        allowed_keys = optional(list(string), [])
        blocked_keys = optional(list(string), [])
        restricted_keys = optional(map(object({
          allowed = optional(list(string), [])
          denied  = optional(list(string), [])
          max     = optional(string, null)
        })), {})
      }), null)
    })
  }))
| n/a | yes | | [storage\_provider](#input\_storage\_provider) | Storage-provider configuration used by the webhook resources. | 
object({
    aws = object({
      kms_key_id = optional(string, null)
      ssm = object({
        paths = object({
          root    = string
          webhook = string
        })
      })
    })
  })
| n/a | yes | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index d4b25d7d70..177bbaa0c3 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -23,7 +23,7 @@ variable "tags" { } variable "runner_matcher_config" { - description = "SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `allowed_keys = []`, `blocked_keys = []` (cannot be used together with `allowed_keys`), and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`." + description = "SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2` and identifies the compute provider that owns the queue. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type` or `image-version` for `ghr-microvm-image-version`." type = map(object({ arn = string id = string @@ -46,15 +46,18 @@ variable "runner_matcher_config" { }) })) validation { - condition = try(var.runner_matcher_config.matcherConfig.priority, 999) >= 0 && try(var.runner_matcher_config.matcherConfig.priority, 999) < 1000 + condition = alltrue([ + for config in values(var.runner_matcher_config) : + config.matcherConfig.priority >= 0 && config.matcherConfig.priority < 1000 + ]) error_message = "The priority of the matcher must be between 0 and 999." } validation { condition = alltrue([ for config in values(var.runner_matcher_config) : - lower(trimspace(config.computeProvider)) == "ec2" + contains(["ec2", "microvm"], lower(trimspace(config.computeProvider))) ]) - error_message = "computeProvider must be ec2." + error_message = "computeProvider must be one of: ec2, microvm." } validation { condition = alltrue([ diff --git a/tests/ministack/README.md b/tests/ministack/README.md index 0f1a1965d7..bcd3c17529 100644 --- a/tests/ministack/README.md +++ b/tests/ministack/README.md @@ -1,7 +1,7 @@ # MiniStack example tests The MiniStack workflow runs the `base`, `prebuilt`, `default`, `ephemeral`, -`multi-runner`, `multi-runner-v2`, and `termination-watcher` examples directly +`multi-runner`, and `termination-watcher` examples directly with Terraform 1.5.6 and the latest Terraform release, and with OpenTofu 1.11 and the latest OpenTofu release. The examples with input variables get their inputs from their own tfvars files @@ -11,8 +11,8 @@ into the MicroVM Network Connector. The `termination-watcher` example has no inp and uses the configuration checked into the example itself. No override files, setup module, or Terraform fixture configuration is checked in. The helper creates and removes a temporary AMI override for `default` and -`ephemeral`, temporary SSM parameters for `multi-runner`, and temporary AMI -fixtures for `multi-runner-v2`. The migration test uses its dedicated +`ephemeral`, and temporary SSM parameters for `multi-runner`. The migration +test uses its dedicated `run-migration-test.sh` lifecycle script. Start MiniStack, set the AWS endpoint and test credentials, then run: @@ -28,8 +28,6 @@ tests/ministack/run-example.sh apply ephemeral # or tests/ministack/run-example.sh apply multi-runner # or -tests/ministack/run-example.sh apply multi-runner-v2 -# or tests/ministack/run-example.sh apply termination-watcher ``` @@ -37,7 +35,7 @@ The script also supports `init`, `plan`, and `destroy`. It creates inert Lambda ZIP fixtures in the paths expected by the modules when they are absent, and removes only the files it created. For `prebuilt`, it seeds AMI metadata through MiniStack's AWS-compatible EC2 API, then removes only the resources it created -during cleanup. MiniStack v1.5.11 provides the EC2 image behavior needed by the +during cleanup. MiniStack v1.5.15 provides the EC2 image behavior needed by the `default`, `ephemeral`, and `multi-runner` examples, so they are included in the same lifecycle matrix. @@ -46,45 +44,99 @@ the same lifecycle matrix. The smoke test covers two independent lifecycle chains. The webhook chain sends signed `workflow_job` webhooks through the API Gateway endpoint and verifies the asynchronous path through EventBridge, the dispatcher Lambda, SQS, -and the scale-up Lambda. It runs scale-up once without a dynamic label and once -with `ghr-ec2-instance-type:m5.large`, checking that the first launch uses a -configured default instance type and the second launch uses exactly `m5.large`. -The scale-up Lambda calls a pinned `mockserver/mockserver` container initialized -from `github-api-expectations.json`; the test uses MockServer's verification API -to confirm the expected GitHub API calls for both jobs. It also checks the +and the scale-up Lambda. Each provider runs scale-up once without a dynamic +label and once with a provider-specific dynamic label, checking the provider's +resolved resource configuration. +The smoke runner connects to an already-running MockServer initialized from +`github-api-expectations.json`; it uses MockServer's verification API to confirm +the expected GitHub API calls for both jobs. It also checks the webhook, dispatcher, and scale-up Lambda log groups for each smoke job ID, then -confirms that both MiniStack EC2 runner instances are removed and terminated. +confirms that each provider resource is removed and terminated. The second, pool chain then invokes the pool Lambda with a pool size of one and verifies every expected GitHub API route for pool reconciliation, including the installation, token, runner-list, and registration-token calls, before confirming that it -creates a second EC2 runner. Installation lookup is mocked for configurations +creates a second provider runner. Installation lookup is mocked for configurations that do not provide a stored installation ID, but is conditional and is not a required assertion. The test also verifies the `ghr:Application`, `ghr:created_by`, `ghr:Type`, and `ghr:Owner` tags used to discover managed -instances. MiniStack v1.5.10 propagates the Terraform launch-template tags to +instances. MiniStack v1.5.15 propagates the Terraform launch-template tags to instances, allowing the scale-down Lambda to discover and remove each runner. The smoke test invokes scale-down for the webhook and pool-created runners and verifies the GitHub API calls and EC2 termination. The pool schedule is configured for a far-future date because the test invokes the Lambda directly. -Build the two real Lambda distributions, start MiniStack, and run: +The smoke deployment uses the `multi-runner-webhook` example, which creates +both EC2 and MicroVM lanes behind one webhook endpoint. For each provider, the +shared lifecycle runs scale-up without a dynamic label, scale-up with a dynamic +label, one pool scale-up, and scale-down for all three resources. The provider +implementation supplies the event labels, resource discovery, provider-specific +route checks, and compute-resource assertions. The shared example accepts the +built runner-control and webhook Lambda ZIP files as `runners_lambda_zip` and +`webhook_lambda_zip`. + +To exercise the MicroVM image's lifecycle hook after each MicroVM scale-up, start +the image locally and provide its hook URL. The hook container must use the same +MiniStack endpoint as the smoke test. The scale-up Lambda creates the JIT config +in SSM; the smoke test sends the same `runHookPayload` to the hook, which consumes +that SSM value. + +```sh +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py microvm +``` + +The test sends the outer JSON request with `runHookPayload` encoded as a JSON +string, then waits for +`/github-action-runners/multi-runner-webhook/microvm/runners/tokens/` +to disappear. This proves that the lifecycle hook consumed the one-time SSM +value before the MicroVM is scaled down. + +Build the two real Lambda distributions, start MockServer and MiniStack, and run: ```sh (cd lambdas && yarn install --frozen-lockfile) (cd lambdas && yarn workspace @aws-github-runner/webhook dist) (cd lambdas && yarn workspace @aws-github-runner/control-plane dist) -sh tests/ministack/run-smoke.sh +# Run both provider lanes in one deployment. MockServer must already be running +# and MINISTACK_GITHUB_MOCK_URL must point to it: +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py +# Preserve the deployment and temporary tfvars file for debugging: +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py --keep-deployment +# Run one provider explicitly when debugging: +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py ec2 +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py microvm ``` +The runner writes `ministack-smoke-checklist.txt` in the current directory and +updates it throughout the run. Override the destination with +`MINISTACK_SMOKE_CHECKLIST_FILE` when the file should be retained as a CI +artifact. + +The Python `smoke/lifecycle.py` module owns the provider-neutral scenarios, and +`smoke/provider.py` defines the provider interface. Shared webhook delivery, log +polling, MockServer route verification, GitHub runner-state fixtures, and Lambda +invocation helpers live in `smoke/common.py`. To add a provider, implement the +interface under `smoke/`, register the provider in +`run-webhook-smoke.py`, and add its provider-specific assertions. + The smoke script generates a temporary RSA key and Terraform variables file, -starts the MockServer container on a temporary port, and removes all temporary -state during cleanup. In CI, the pinned MockServer setup action starts the -server and waits for readiness; the expectations are loaded after checkout. +expects an already-running MockServer at `MINISTACK_GITHUB_MOCK_URL`, loads the +expectations into it, and destroys the Terraform deployment during cleanup. +Pass `--keep-deployment` (or set `MINISTACK_SMOKE_KEEP_DEPLOYMENT=1`) to retain +the deployment for debugging; it prints the generated tfvars path so the +deployment can be destroyed separately with `tests/ministack/run-example.sh destroy`. +The MockServer lifecycle is an external test dependency; the Python smoke runner +does not start or stop Docker containers. In CI, the MockServer setup action +starts the server and waits for readiness before the Python smoke runner executes. MiniStack must be able to reach `host.docker.internal`; override the hostname with `MINISTACK_GITHUB_MOCK_HOST` when using a different container runtime. When MiniStack is exposed on a non-default local port, use a host address reachable from its container for `AWS_ENDPOINT_URL`, for example -`AWS_ENDPOINT_URL=http://:14568`, instead of `127.0.0.1`. +`AWS_ENDPOINT_URL=http://:14568`, instead of `localhost`. diff --git a/tests/ministack/microvm.tfvars b/tests/ministack/microvm.tfvars deleted file mode 100644 index c777b7228a..0000000000 --- a/tests/ministack/microvm.tfvars +++ /dev/null @@ -1,12 +0,0 @@ - -aws_region = "eu-west-1" -environment = "microvm-ministack" - -github_app = { - id = "your-github-app-id" - key_base64 = "your-github-app-key-base64" -} - -lambda_artifact_bucket = "github-actions-runner-microvm-ministack" -microvm_image_arn = "arn:aws:lambda:eu-west-1:000000000000:microvm-image:ministack" -egress_network_connector_arn = "arn:aws:lambda:eu-west-1:000000000000:network-connector:ministack" diff --git a/tests/ministack/multi-runner-v2.tfvars b/tests/ministack/multi-runner-v2.tfvars deleted file mode 100644 index 0f9c6073fc..0000000000 --- a/tests/ministack/multi-runner-v2.tfvars +++ /dev/null @@ -1,31 +0,0 @@ -environment = "ministack-v2" -aws_region = "eu-west-1" - -github_app = { - id = "0" - key_base64 = "ministack-invalid-key" -} - -ami = { - "linux-arm64" = { - filter = { - name = ["ministack-v2-linux-arm64"] - state = ["available"] - } - owners = ["self"] - } - "linux-x64" = { - filter = { - name = ["ministack-v2-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - "windows-x64" = { - filter = { - name = ["ministack-v2-windows-x64"] - state = ["available"] - } - owners = ["self"] - } -} diff --git a/tests/ministack/multi-runner-webhook.tfvars b/tests/ministack/multi-runner-webhook.tfvars new file mode 100644 index 0000000000..aaab6d1da1 --- /dev/null +++ b/tests/ministack/multi-runner-webhook.tfvars @@ -0,0 +1,32 @@ +aws_region = "eu-west-1" +environment = "multi-runner-webhook" + +runners_lambda_zip = "../../lambda_output/runners.zip" +webhook_lambda_zip = "../../lambda_output/webhook.zip" + +github_app = { + id = "123" + key_base64 = "ministack-invalid-key" + webhook_secret = "ministack-webhook-secret" +} + +compute_provider = { + aws = { + ec2 = { + instance_types = ["m7a.large", "m5.large"] + ami = { + filter = { + name = ["ministack-webhook-linux-x64"] + state = ["available"] + } + owners = ["self"] + } + } + microvm = { + image_arn = "arn:aws:lambda:eu-west-1:000000000000:microvm-image:ministack" + image_version = "3.0" + egress_network_connectors = ["arn:aws:lambda:eu-west-1:000000000000:network-connector:ministack"] + ingress_network_connectors = [] + } + } +} diff --git a/tests/ministack/run-example.sh b/tests/ministack/run-example.sh index bb52c7689a..01364940ec 100755 --- a/tests/ministack/run-example.sh +++ b/tests/ministack/run-example.sh @@ -22,7 +22,7 @@ case "$iac_binary" in ;; esac case "$example" in - base | prebuilt | default | ephemeral | multi-runner | multi-runner-v2 | microvm-foundation | microvm) + base | prebuilt | default | ephemeral | multi-runner | multi-runner-webhook | microvm-foundation) use_tfvars=true ;; migration-test) @@ -32,15 +32,15 @@ case "$example" in use_tfvars=false ;; *) - echo "Supported examples for the runner are: base, prebuilt, default, ephemeral, multi-runner, multi-runner-v2, microvm-foundation, microvm, migration-test, termination-watcher" >&2 + echo "Supported examples for the runner are: base, prebuilt, default, ephemeral, multi-runner, multi-runner-webhook, microvm-foundation, migration-test, termination-watcher" >&2 exit 64 ;; esac case "$action" in - init | plan | apply | destroy) ;; + init | plan | apply | destroy | output) ;; *) - echo "Usage: $0 {init|plan|apply|destroy} {base|prebuilt|default|ephemeral|multi-runner|multi-runner-v2|microvm-foundation|microvm|migration-test|termination-watcher} [TFVARS_FILE]" >&2 + echo "Usage: $0 {init|plan|apply|destroy|output} {base|prebuilt|default|ephemeral|multi-runner|multi-runner-webhook|microvm-foundation|migration-test|termination-watcher} [TFVARS_FILE]" >&2 exit 64 ;; esac @@ -196,11 +196,11 @@ create_ami_fixture() { architecture="$2" ami_id=$(ministack_aws ec2 describe-images \ --owners self \ - --filters "Name=name,Values=$ami_name" "Name=state,Values=available" \ + --filters "Name=name,Values=$ami_name" \ --query 'Images[0].ImageId' \ --output text) - if [ "$ami_id" = "None" ]; then + if [ "$ami_id" = "None" ] || [ -z "$ami_id" ]; then ami_id=$(ministack_aws ec2 register-image \ --name "$ami_name" \ --description "MiniStack test-only AMI" \ @@ -214,6 +214,25 @@ create_ami_fixture() { $ami_id" fi + attempts=60 + while [ "$attempts" -gt 0 ]; do + ami_state=$(ministack_aws ec2 describe-images \ + --owners self \ + --image-ids "$ami_id" \ + --query 'Images[0].State' \ + --output text) + if [ "$ami_state" = "available" ]; then + return + fi + + attempts=$((attempts - 1)) + if [ "$attempts" -eq 0 ]; then + echo "AMI $ami_id ($ami_name) did not become available; last state: $ami_state" >&2 + exit 70 + fi + sleep 1 + done + } create_ssm_fixture() { @@ -347,29 +366,8 @@ $lambda_zip" "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64" \ "ami-0abcdef1234567890" ;; - multi-runner-v2) - create_ami_fixture "ministack-v2-linux-arm64" arm64 >/dev/null - create_ami_fixture "ministack-v2-linux-x64" x86_64 >/dev/null - create_ami_fixture "ministack-v2-windows-x64" x86_64 >/dev/null - ;; - microvm) - create_ssm_fixture \ - "/ministack/microvm/github-app-key" \ - "test-only" - create_ssm_fixture \ - "/ministack/microvm/github-app-id" \ - "123456" - create_ssm_fixture \ - "/ministack/microvm/webhook-secret" \ - "test-only" - create_s3_fixture \ - "github-actions-runner-microvm-ministack" \ - "runners.zip" \ - "$lambda_fixture_dir/ministack-lambda.zip" - create_s3_fixture \ - "github-actions-runner-microvm-ministack" \ - "webhook.zip" \ - "$lambda_fixture_dir/ministack-lambda.zip" + multi-runner-webhook) + create_ami_fixture "ministack-webhook-linux-x64" x86_64 >/dev/null ;; esac } @@ -408,4 +406,8 @@ case "$action" in iac_init iac_example destroy -auto-approve -input=false -parallelism=1 ;; + output) + iac_init + iac_example output + ;; esac diff --git a/tests/ministack/run-smoke.sh b/tests/ministack/run-smoke.sh deleted file mode 100644 index d4d9c7ab3e..0000000000 --- a/tests/ministack/run-smoke.sh +++ /dev/null @@ -1,721 +0,0 @@ -#!/bin/sh - -set -eu - -export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" -export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" -export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" -export AWS_REGION="${AWS_REGION:-eu-west-1}" -export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" -export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) -example_root="$source_root/examples/default" -mock_expectations="$script_dir/github-api-expectations.json" -fixture="$script_dir/workflow_job_event.json" -dynamic_fixture=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-dynamic-workflow-job.XXXXXX") -mock_host="${MINISTACK_GITHUB_MOCK_HOST:-host.docker.internal}" -mock_port="${MINISTACK_GITHUB_MOCK_PORT:-}" -mock_service_url="${MINISTACK_GITHUB_MOCK_URL:-}" -mock_image="${MINISTACK_GITHUB_MOCK_IMAGE:-mockserver/mockserver:7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290}" -mock_container="" -tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-smoke.XXXXXX") -app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-github-app.XXXXXX") -response_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-smoke-response.XXXXXX") -lambda_response_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-lambda-response.XXXXXX") -override_file="$example_root/zz_ministack_smoke_override.tf" -terraform_initialized=false -discovered_instance_ids="" - -cleanup() { - set +e - for instance_id in $discovered_instance_ids; do - aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 terminate-instances \ - --instance-ids "$instance_id" >/dev/null 2>&1 - done - if [ "$terraform_initialized" = true ]; then - "$source_root/tests/ministack/run-example.sh" destroy default "$tfvars_file" >/dev/null 2>&1 - fi - if [ -n "$mock_container" ]; then - docker rm -f "$mock_container" >/dev/null 2>&1 - fi - rm -f "$override_file" "$tfvars_file" "$app_key_file" "$response_file" "$lambda_response_file" "$dynamic_fixture" -} -trap cleanup EXIT INT TERM - -require_command() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "$1 is required to run the MiniStack smoke test." >&2 - exit 69 - fi -} - -for command in aws curl openssl python3 terraform; do - require_command "$command" -done -if [ -z "$mock_service_url" ]; then - require_command docker -fi - -for lambda_zip in \ - "$source_root/lambdas/functions/webhook/webhook.zip" \ - "$source_root/lambdas/functions/control-plane/runners.zip"; do - if [ ! -f "$lambda_zip" ]; then - echo "Missing $lambda_zip. Build the webhook and control-plane distributions first." >&2 - exit 66 - fi -done - -if [ -z "$mock_port" ]; then - if [ -n "$mock_service_url" ]; then - mock_port=1080 - else - mock_port=$(python3 -c 'import socket; s = socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') - fi -fi - -if [ -z "$mock_service_url" ]; then - mock_container="terraform-aws-github-runner-github-api-mock-$$" - mock_service_url="http://127.0.0.1:${mock_port}" -fi - -openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null -app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') -APP_KEY_BASE64="$app_key_base64" python3 - "$script_dir/default.tfvars" "$tfvars_file" <<'PY' -import os -import sys - -source, destination = sys.argv[1:] -replacement = os.environ["APP_KEY_BASE64"] -with open(source, encoding="utf-8") as source_file: - lines = source_file.readlines() -with open(destination, "w", encoding="utf-8") as destination_file: - for line in lines: - if line.lstrip().startswith("key_base64 ="): - destination_file.write(f' key_base64 = "{replacement}"\n') - elif line.lstrip().startswith('id') and '=' in line: - destination_file.write(' id = "123"\n') - else: - destination_file.write(line) -PY -unset app_key_base64 APP_KEY_BASE64 - -printf '%s\n' \ - 'module "runners" {' \ - " ghes_url = \"http://${mock_host}:${mock_port}\"" \ - ' ghes_ssl_verify = false' \ - ' eventbridge = {' \ - ' enable = true' \ - ' accept_events = ["workflow_job"]' \ - ' }' \ - ' delay_webhook_event = 0' \ - ' runners_maximum_count = 1' \ - ' instance_types = ["m7a.large"]' \ - ' enable_dynamic_labels = true' \ - ' minimum_running_time_in_minutes = 0' \ - ' pool_runner_owner = "test-owner"' \ - ' pool_config = [{ schedule_expression = "cron(0 0 1 1 ? 2099)", size = 1 }]' \ - ' scale_down_schedule_expression = "cron(0 0 1 1 ? 2099)"' \ - ' enable_job_queued_check = true' \ - ' enable_jit_config = false' \ - ' enable_runner_binaries_syncer = false' \ - ' log_level = "debug"' \ - '}' \ - '' \ - 'module "webhook_github_app" {' \ - ' count = 0' \ - '}' > "$override_file" - -if [ -n "$mock_container" ]; then - docker run --detach --name "$mock_container" --publish "${mock_port}:1080" \ - --volume "$mock_expectations:/config/github-api-expectations.json:ro" \ - --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/github-api-expectations.json \ - "$mock_image" >/dev/null -fi - -attempts=30 -while ! curl -fsS --max-time 2 -X PUT "${mock_service_url}/mockserver/status" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "MockServer did not become ready." >&2 - if [ -n "$mock_container" ]; then - docker logs "$mock_container" >&2 - fi - exit 70 - fi - sleep 1 -done - -if [ -z "$mock_container" ]; then - MOCKSERVER_URL="$mock_service_url" python3 - "$mock_expectations" <<'PY' -import json -import os -import sys -import urllib.request - -with open(sys.argv[1], encoding="utf-8") as expectations_file: - expectations = json.load(expectations_file) - -for expectation in expectations: - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}/mockserver/expectation', - data=json.dumps(expectation).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="PUT", - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201): - raise RuntimeError(f"MockServer expectation rejected with HTTP {response.status}") -PY -fi - -python3 - "$fixture" "$dynamic_fixture" <<'PY' -import json -import sys - -source, destination = sys.argv[1:] -with open(source, encoding="utf-8") as source_file: - event = json.load(source_file) - -job = event["workflow_job"] -job["id"] = 123457 -job["run_id"] = 654322 -job["run_url"] = job["run_url"].replace("654321", "654322") -job["url"] = job["url"].replace("123456", "123457") -job["html_url"] = job["html_url"].replace("123456", "123457") -job["name"] = "ministack-smoke-dynamic" -job["labels"].append("ghr-ec2-instance-type:m5.large") - -with open(destination, "w", encoding="utf-8") as destination_file: - json.dump(event, destination_file) -PY - -terraform_initialized=true -"$source_root/tests/ministack/run-example.sh" apply default "$tfvars_file" - -printf '%s\n' \ - 'MiniStack smoke chain evidence checklist:' \ - ' [ ] API Gateway accepted the signed workflow_job webhook (HTTP 201)' \ - ' [ ] Webhook Lambda log contains workflow job 123456' \ - ' [ ] EventBridge invoked the dispatcher Lambda (dispatcher log contains 123456)' \ - ' [ ] Dispatcher delivered the job through SQS (scale-up log contains 123456)' \ - ' [ ] Scale-up without a dynamic label called each expected GitHub API route in MockServer' \ - ' [ ] MiniStack EC2 API reports a standard scale-up instance with default EC2 configuration' \ - ' [ ] Scale-up with ghr-ec2-instance-type:m5.large called each expected GitHub API route in MockServer' \ - ' [ ] Dynamic label selected EC2 instance type m5.large' \ - ' [ ] Scale-up EC2 instance has the expected runner discovery tags' \ - ' [ ] Pool called every expected GitHub API route in MockServer' \ - ' [ ] Pool Lambda created a runner instance' \ - ' [ ] Pool EC2 instance has the expected runner discovery tags' \ - ' [ ] Scale-down removed each runner from GitHub and terminated its EC2 instance' - -webhook_endpoint=$(terraform -chdir="$example_root" output -raw webhook_endpoint) -endpoint_host_port=${AWS_ENDPOINT_URL#*://} -endpoint_port=${endpoint_host_port##*:} -api_host_port=${webhook_endpoint#*://} -api_host_port=${api_host_port%%/*} -api_host=${api_host_port%:*} -webhook_secret=$(terraform -chdir="$example_root" output -raw webhook_secret) - -send_webhook() { - fixture_file="$1" - delivery_id="$2" - signature=$(openssl dgst -sha256 -hmac "$webhook_secret" "$fixture_file" | awk '{print $NF}') - status_code=$(curl -sS --max-time 15 -o "$response_file" -w '%{http_code}' \ - --connect-to "${api_host}:4566:127.0.0.1:${endpoint_port}" \ - -X POST "$webhook_endpoint" \ - -H 'Content-Type: application/json' \ - -H 'X-GitHub-Event: workflow_job' \ - -H "X-GitHub-Delivery: ${delivery_id}" \ - -H 'X-GitHub-Hook-Installation-Target-ID: 123' \ - -H "X-Hub-Signature-256: sha256=${signature}" \ - --data-binary "@${fixture_file}") - - if [ "$status_code" != 201 ]; then - echo "Webhook smoke request failed with HTTP $status_code." >&2 - sed -n '1,80p' "$response_file" >&2 - exit 1 - fi - echo " [PASS] API Gateway accepted the signed workflow_job webhook ${delivery_id} (HTTP 201)" -} - -send_webhook "$fixture" "ministack-smoke-123456" - -wait_for_log_event() { - log_group="$1" - marker="$2" - description="$3" - attempts=60 - while ! aws --endpoint-url "$AWS_ENDPOINT_URL" logs filter-log-events \ - --log-group-name "$log_group" --filter-pattern "$marker" --limit 1 --output text 2>/dev/null | grep -Fq "$marker"; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for MiniStack log marker '$marker' in $log_group." >&2 - exit 1 - fi - sleep 2 - done - printf ' [PASS] %s (log group %s contains %s)\n' "$description" "$log_group" "$marker" -} - -wait_for_optional_log_event() { - log_group="$1" - marker="$2" - description="$3" - attempts=60 - while ! aws --endpoint-url "$AWS_ENDPOINT_URL" logs filter-log-events \ - --log-group-name "$log_group" --filter-pattern "$marker" --limit 1 --output text 2>/dev/null | grep -Fq "$marker"; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - printf ' [WARN] %s (log marker %s was not observed in %s)\n' \ - "$description" "$marker" "$log_group" - return 0 - fi - sleep 2 - done - printf ' [PASS] %s (log group %s contains %s)\n' "$description" "$log_group" "$marker" -} - -wait_for_log_event "/aws/lambda/ministack-default-webhook" "123456" \ - "Webhook Lambda received workflow job 123456" -wait_for_log_event "/aws/lambda/ministack-default-dispatch-to-runner" "123456" \ - "EventBridge invoked the dispatcher Lambda" -wait_for_log_event "/aws/lambda/ministack-default-scale-up" "123456" \ - "Dispatcher delivered workflow job 123456 through SQS to scale-up" - -wait_for_mock_route() { - method="$1" - route="$2" - description="$3" - verification_body=$(printf '{"httpRequest":{"method":"%s","path":"%s"},"times":{"atLeast":1}}' "$method" "$route") - attempts=60 - while ! curl -fsS --max-time 5 -X PUT "${mock_service_url}/mockserver/verify" \ - -H 'Content-Type: application/json' \ - --data-binary "$verification_body" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for MockServer route: $method $route" >&2 - curl -sS --max-time 5 -X PUT \ - "${mock_service_url}/mockserver/retrieve?type=REQUEST_RESPONSES&format=JSON" >&2 || true - exit 1 - fi - sleep 2 - done - printf ' [PASS] %s (MockServer verified %s %s)\n' "$description" "$method" "$route" -} - -clear_mock_request_log() { - if ! curl -fsS --max-time 5 -X PUT \ - "${mock_service_url}/mockserver/clear?type=log" >/dev/null 2>&1; then - echo "Failed to clear MockServer request history before the next lifecycle phase." >&2 - exit 1 - fi -} - -assert_scale_down_github_routes() { - wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ - "Scale-down requested a GitHub App installation token" - wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners" \ - "Scale-down listed organization runners" - wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners/${1}" \ - "Scale-down checked the runner busy state" - wait_for_mock_route DELETE "/api/v3/orgs/test-owner/actions/runners/${1}" \ - "Scale-down deleted the runner from GitHub" -} - -assert_pool_github_routes() { - wait_for_mock_route GET "/api/v3/orgs/test-owner/installation" \ - "Pool looked up the GitHub App installation" - wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ - "Pool requested a GitHub App installation token" - wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners" \ - "Pool listed organization runners" - wait_for_mock_route POST "/api/v3/orgs/test-owner/actions/runners/registration-token" \ - "Pool requested a GitHub runner registration token" -} - -assert_scale_up_github_routes() { - job_id="$1" - wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ - "Scale-up requested a GitHub App installation token for job ${job_id}" - wait_for_mock_route GET "/api/v3/repos/test-owner/test-repo/actions/jobs/${job_id}" \ - "Scale-up checked the queued GitHub job ${job_id}" - wait_for_mock_route POST "/api/v3/orgs/test-owner/actions/runners/registration-token" \ - "Scale-up requested a GitHub runner registration token for job ${job_id}" -} - -assert_scale_up_github_routes 123456 - -wait_for_ec2_instance() { - source="$1" - description="$2" - attempts=60 - while :; do - found_instance_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --filters \ - "Name=instance-state-name,Values=running,pending" \ - "Name=tag:ghr:Application,Values=github-action-runner" \ - "Name=tag:ghr:created_by,Values=$source" \ - --query 'Reservations[].Instances[].InstanceId | [0]' \ - --output text 2>/dev/null || true) - if [ -n "$found_instance_id" ] && [ "$found_instance_id" != "None" ]; then - case " $discovered_instance_ids " in - *" $found_instance_id "*) ;; - *) discovered_instance_ids="$discovered_instance_ids $found_instance_id" ;; - esac - printf ' [PASS] MiniStack EC2 API reports %s: %s\n' "$description" "$found_instance_id" - return - fi - - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $description in the MiniStack EC2 API." >&2 - aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --filters \ - "Name=instance-state-name,Values=running,pending" \ - "Name=tag:ghr:Application,Values=github-action-runner" \ - "Name=tag:ghr:created_by,Values=$source" \ - --output json >&2 || true - exit 1 - fi - sleep 2 - done -} - -wait_for_ec2_instance "scale-up-lambda" "a scale-up instance" -scale_up_instance_id="$found_instance_id" - -assert_ec2_tag() { - instance_id="$1" - key="$2" - expected_value="$3" - description="$4" - actual_value=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query "Reservations[].Instances[].Tags[?Key=='${key}'].Value | [0]" \ - --output text 2>/dev/null || true) - if [ "$actual_value" != "$expected_value" ]; then - echo "Expected $description tag $key=$expected_value on $instance_id, got $actual_value." >&2 - exit 1 - fi -} - -assert_ec2_runner_tags() { - instance_id="$1" - source="$2" - description="$3" - assert_ec2_tag "$instance_id" "ghr:Application" "github-action-runner" "$description" - assert_ec2_tag "$instance_id" "ghr:created_by" "$source" "$description" - assert_ec2_tag "$instance_id" "ghr:Type" "Org" "$description" - assert_ec2_tag "$instance_id" "ghr:Owner" "test-owner" "$description" - printf ' [PASS] MiniStack EC2 API reports correct runner tags on %s\n' "$instance_id" -} - -assert_ec2_runner_tags "$scale_up_instance_id" "scale-up-lambda" "the scale-up runner" - -assert_ec2_default_instance_type() { - instance_id="$1" - actual_type=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].InstanceType' \ - --output text 2>/dev/null || true) - if [ "$actual_type" != "m7a.large" ]; then - echo "Expected standard scale-up to use the configured default m7a.large, got $actual_type." >&2 - exit 1 - fi - printf ' [PASS] Standard scale-up used the configured default EC2 instance type: %s\n' "$actual_type" -} - -assert_ec2_instance_type() { - instance_id="$1" - expected_type="$2" - actual_type=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].InstanceType' \ - --output text 2>/dev/null || true) - if [ "$actual_type" != "$expected_type" ]; then - echo "Expected $instance_id to use EC2 instance type $expected_type, got $actual_type." >&2 - exit 1 - fi - printf ' [PASS] EC2 dynamic label selected instance type %s on %s\n' "$expected_type" "$instance_id" -} - -assert_ec2_default_instance_type "$scale_up_instance_id" - -configure_mock_runner_state() { - instance_id="$1" - runner_id="$2" - MOCKSERVER_URL="$mock_service_url" python3 - "$instance_id" "$runner_id" <<'PY' -import json -import os -import sys -import urllib.request - -instance_id, runner_id = sys.argv[1:] -runner_id = int(runner_id) -base = "/api/v3/orgs/test-owner/actions/runners" - -def control(path, method, payload): - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}{path}', - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method=method, - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201, 202): - raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') - -def clear(method, path): - control("/mockserver/clear", "PUT", {"httpRequest": {"method": method, "path": path}}) - -def expect(method, path, status, body=None): - response = {"statusCode": status} - if body is not None: - response["headers"] = {"Content-Type": ["application/json"]} - response["body"] = json.dumps(body) - control( - "/mockserver/expectation", - "PUT", - {"httpRequest": {"method": method, "path": path}, "httpResponse": response}, - ) - -state_path = f"{base}/{runner_id}" -clear("GET", base) -clear("GET", state_path) -clear("DELETE", state_path) -expect( - "GET", - base, - 200, - { - "total_count": 1, - "runners": [ - { - "id": runner_id, - "name": f"ministack-smoke-{instance_id}", - "os": "linux", - "status": "offline", - "busy": False, - "labels": [], - } - ], - }, -) -expect( - "GET", - state_path, - 200, - { - "id": runner_id, - "name": f"ministack-smoke-{instance_id}", - "os": "linux", - "status": "offline", - "busy": False, - "labels": [], - }, -) -expect("DELETE", state_path, 204) -PY -} - -configure_mock_runner_removed() { - runner_id="$1" - MOCKSERVER_URL="$mock_service_url" python3 - "$runner_id" <<'PY' -import json -import os -import sys -import urllib.request - -runner_id = sys.argv[1] -path = f"/api/v3/orgs/test-owner/actions/runners/{runner_id}" - -def control(path, method, payload): - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}{path}', - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method=method, - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201, 202): - raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') - -control("/mockserver/clear", "PUT", {"httpRequest": {"method": "GET", "path": path}}) -control( - "/mockserver/expectation", - "PUT", - { - "httpRequest": {"method": "GET", "path": path}, - "httpResponse": { - "statusCode": 404, - "headers": {"Content-Type": ["application/json"]}, - "body": '{"message":"Not Found"}', - }, - }, -) -PY -} - -configure_empty_mock_runner_list() { - MOCKSERVER_URL="$mock_service_url" python3 - <<'PY' -import json -import os -import urllib.request - -path = "/api/v3/orgs/test-owner/actions/runners" - -def control(path, method, payload): - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}{path}', - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method=method, - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201, 202): - raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') - -control("/mockserver/clear", "PUT", {"httpRequest": {"method": "GET", "path": path}}) -control( - "/mockserver/expectation", - "PUT", - { - "httpRequest": {"method": "GET", "path": path}, - "httpResponse": { - "statusCode": 200, - "headers": {"Content-Type": ["application/json"]}, - "body": '{"total_count":0,"runners":[]}', - }, - }, -) -PY -} - -assert_mock_runner_removed() { - runner_id="$1" - status_code=$(curl -sS --max-time 5 -o "$response_file" -w '%{http_code}' \ - "${mock_service_url}/api/v3/orgs/test-owner/actions/runners/${runner_id}") - if [ "$status_code" != 404 ]; then - echo "Expected GitHub API mock to return 404 for removed runner $runner_id, got HTTP $status_code." >&2 - sed -n '1,80p' "$response_file" >&2 - exit 1 - fi - printf ' [PASS] GitHub API mock reports runner %s removed (HTTP 404)\n' "$runner_id" -} - -wait_for_ec2_termination() { - instance_id="$1" - description="$2" - attempts=60 - while :; do - if state=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query 'Reservations[].Instances[].State.Name | [0]' \ - --output text 2>/dev/null); then - if [ -z "$state" ] || [ "$state" = "None" ] || [ "$state" = "terminated" ]; then - printf ' [PASS] MiniStack EC2 API reports %s terminated\n' "$description" - return - fi - else - state="describe-instances failed" - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $description to terminate; current state: $state." >&2 - exit 1 - fi - sleep 2 - done -} - -invoke_lambda() { - function_name="$1" - payload="$2" - description="$3" - invocation_result=$(aws --endpoint-url "$AWS_ENDPOINT_URL" lambda invoke \ - --cli-binary-format raw-in-base64-out \ - --invocation-type RequestResponse \ - --function-name "$function_name" \ - --payload "$payload" \ - "$lambda_response_file" --output json) - if printf '%s' "$invocation_result" | grep -Fq '"FunctionError"'; then - echo "Lambda invocation returned FunctionError for $function_name." >&2 - exit 1 - fi - printf ' [PASS] %s (Lambda API accepted the request)\n' "$description" -} - -scale_up_runner_id=987654321 -configure_mock_runner_state "$scale_up_instance_id" "$scale_up_runner_id" -clear_mock_request_log -invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-scale-up-scale-down"}' \ - "Scale-down Lambda invoked for the scale-up runner" -wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-scale-up-scale-down" \ - "Scale-down Lambda started processing the scale-up runner" -assert_scale_down_github_routes "$scale_up_runner_id" -configure_mock_runner_removed "$scale_up_runner_id" -assert_mock_runner_removed "$scale_up_runner_id" -wait_for_ec2_termination "$scale_up_instance_id" "the scale-up instance" -wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$scale_up_instance_id" \ - "Scale-down log recorded termination of the scale-up EC2 runner" - -clear_mock_request_log -send_webhook "$dynamic_fixture" "ministack-smoke-123457" -wait_for_log_event "/aws/lambda/ministack-default-webhook" "123457" \ - "Webhook Lambda received dynamic-label workflow job 123457" -wait_for_log_event "/aws/lambda/ministack-default-dispatch-to-runner" "123457" \ - "EventBridge invoked the dispatcher for dynamic-label workflow job 123457" -wait_for_log_event "/aws/lambda/ministack-default-scale-up" "123457" \ - "Dispatcher delivered dynamic-label workflow job 123457 through SQS to scale-up" -assert_scale_up_github_routes 123457 -wait_for_ec2_instance "scale-up-lambda" "a dynamic-label scale-up instance" -dynamic_scale_up_instance_id="$found_instance_id" -assert_ec2_runner_tags "$dynamic_scale_up_instance_id" "scale-up-lambda" \ - "the dynamic-label scale-up runner" -assert_ec2_instance_type "$dynamic_scale_up_instance_id" "m5.large" - -dynamic_scale_up_runner_id=987654323 -configure_mock_runner_state "$dynamic_scale_up_instance_id" "$dynamic_scale_up_runner_id" -clear_mock_request_log -invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-dynamic-scale-up-scale-down"}' \ - "Scale-down Lambda invoked for the dynamic-label scale-up runner" -wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-dynamic-scale-up-scale-down" \ - "Scale-down Lambda started processing the dynamic-label scale-up runner" -assert_scale_down_github_routes "$dynamic_scale_up_runner_id" -configure_mock_runner_removed "$dynamic_scale_up_runner_id" -assert_mock_runner_removed "$dynamic_scale_up_runner_id" -wait_for_ec2_termination "$dynamic_scale_up_instance_id" "the dynamic-label scale-up instance" -wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$dynamic_scale_up_instance_id" \ - "Scale-down log recorded termination of the dynamic-label scale-up EC2 runner" - -echo "MiniStack smoke chain 1 passed: API Gateway -> webhook -> EventBridge -> dispatcher -> SQS -> scale-up without and with EC2 dynamic label -> GitHub API mock." - -configure_empty_mock_runner_list -clear_mock_request_log -invoke_lambda "ministack-default-pool" '{"poolSize":1,"type":"ec2"}' \ - "Pool Lambda invoked to maintain one runner" -assert_pool_github_routes -wait_for_log_event "/aws/lambda/ministack-default-pool" "topped up with 1 runners" \ - "Pool Lambda requested one runner" -wait_for_ec2_instance "pool-lambda" "a pool instance" -pool_instance_id="$found_instance_id" -assert_ec2_runner_tags "$pool_instance_id" "pool-lambda" "the pool runner" - -pool_runner_id=987654322 -configure_mock_runner_state "$pool_instance_id" "$pool_runner_id" -clear_mock_request_log -invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-pool-scale-down"}' \ - "Scale-down Lambda invoked for the pool runner" -wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-pool-scale-down" \ - "Scale-down Lambda started processing the pool runner" -assert_scale_down_github_routes "$pool_runner_id" -configure_mock_runner_removed "$pool_runner_id" -assert_mock_runner_removed "$pool_runner_id" -wait_for_ec2_termination "$pool_instance_id" "the pool instance" -wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$pool_instance_id" \ - "Scale-down log recorded termination of the pool EC2 runner" - -echo "MiniStack smoke chain 2 passed: pool -> GitHub API mock -> EC2 runner creation -> scale-down -> GitHub API mock -> EC2 termination." -echo "MiniStack smoke tests passed: both lifecycle chains completed." diff --git a/tests/ministack/run-webhook-smoke.py b/tests/ministack/run-webhook-smoke.py new file mode 100644 index 0000000000..10a2afab00 --- /dev/null +++ b/tests/ministack/run-webhook-smoke.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Run the shared multi-runner webhook smoke test once for both providers.""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from smoke.common import SmokeContext # noqa: E402 +from smoke.lifecycle import run # noqa: E402 +from smoke import ec2, microvm # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run the multi-runner webhook smoke test for one or all compute providers." + ) + parser.add_argument("provider", nargs="?", choices=("all", "ec2", "microvm"), default="all") + parser.add_argument( + "--keep-deployment", + action="store_true", + default=False, + help="Keep the Terraform deployment and temporary tfvars file for debugging", + ) + args = parser.parse_args() + + providers = {item.slug: item for item in (ec2.provider, microvm.provider)} + + context = SmokeContext(Path(__file__).parent, keep_deployment=args.keep_deployment) + selected = tuple(providers.values()) if args.provider == "all" else (providers[args.provider],) + context.initialize_checklist([item.slug for item in selected]) + succeeded = False + try: + with context.step("Webhook Test"): + with context.step("Prepare deployment"): + context.prepare() + for selected_provider in selected: + run(context, selected_provider) + tested = "EC2 and MicroVM" if args.provider == "all" else providers[args.provider].display_name + print(f"MiniStack multi-runner-webhook smoke tests passed for {tested}.", flush=True) + succeeded = True + return 0 + except BaseException as error: + context.record_checklist_failure(error) + raise + finally: + try: + context.cleanup() + finally: + context.finish_checklist(succeeded) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ministack/smoke/README.md b/tests/ministack/smoke/README.md new file mode 100644 index 0000000000..fe13be8253 --- /dev/null +++ b/tests/ministack/smoke/README.md @@ -0,0 +1,266 @@ +# MiniStack webhook smoke test + +This directory contains the provider-neutral smoke-test harness for the +`multi-runner-webhook` example. It exercises the complete webhook lifecycle +against MiniStack and a GitHub API MockServer without calling GitHub. + +The test can run the EC2 provider, the MicroVM provider, or both providers in a +single Terraform deployment. + +## Entry point + +Run the harness from the repository root: + +```sh +python3 tests/ministack/run-webhook-smoke.py [all|ec2|microvm] +``` + +The default is `all`. Use `--keep-deployment` to retain the temporary Terraform +variables and deployed resources after a failure: + +```sh +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py --keep-deployment +``` + +The ordinary smoke test requires MiniStack on `AWS_ENDPOINT_URL` and an +already-running MockServer exposed through `MINISTACK_GITHUB_MOCK_URL`. + +Step progress is written to the test step. Detailed subprocess output is +tee'd to `MINISTACK_SMOKE_LOG_FILE` (default: `ministack-smoke.log`) and is +printed to the test step only when a command fails. The checklist and command +log are separate files: use `MINISTACK_SMOKE_CHECKLIST_FILE` for assertion +status and the log file for Terraform, Packer, Docker, and AWS CLI output. + +## Complete execution flow + +### 1. Select providers and create the checklist + +`run-webhook-smoke.py` creates one `SmokeContext`, selects the requested +providers, and initializes `ministack-smoke-checklist.txt`. Each checklist item +is marked as the corresponding assertion passes. + +The checklist records: + +- webhook acceptance; +- the webhook, EventBridge, dispatcher, SQS, and scale-up chain; +- standard and dynamic scale-up routes and resources; +- pool routes and resources; +- standard, dynamic, and pool scale-down; +- MicroVM hook consumption when the hook URL is enabled. + +### 2. Configure MockServer + +`SmokeContext.prepare()` first checks the required local commands and waits for +MockServer. It loads the static expectations from: + +```text +tests/ministack/smoke/fixtures/github-api-expectations.json +``` + +Those expectations cover the GitHub job lookup, installation-token exchange, +runner-group lookup, JIT configuration generation, and registration-token +fallback routes. + +Provider-specific expectations are added later when each provider is +configured. + +### 3. Create temporary Terraform input + +The harness generates a temporary RSA key, replaces the invalid fixture +GitHub App key, and adds the local GitHub Enterprise Server configuration. The +temporary variables also point Terraform at the real Lambda ZIPs: + +- `lambdas/functions/control-plane/runners.zip`; +- `lambdas/functions/webhook/webhook.zip`. + +The key and temporary variables are removed during cleanup unless +`--keep-deployment` is used. + +### 4. Apply the `multi-runner-webhook` example + +The harness invokes: + +```sh +tests/ministack/run-example.sh apply multi-runner-webhook +``` + +This deploys the webhook, dispatcher, scale-up, scale-down, pool, EC2, and +MicroVM configuration. After apply, the harness reads the Terraform outputs +for the webhook endpoint and webhook secret. + +The webhook endpoint is normally an API Gateway-style hostname such as +`b3855cb6.execute-api.localhost:4566`. Requests are sent to +`localhost:4566` while preserving that hostname in the HTTP `Host` header so +MiniStack routes the request correctly. + +### 5. Configure provider expectations + +Each provider adds the runner-group and JIT configuration expectations required +by its scale-up Lambda. The MicroVM provider keeps this logic in +`microvm.py`; EC2 keeps its equivalent provider setup in `ec2.py`. + +The JIT value is an internal MockServer fixture value. It is returned by the +mock GitHub API, written by the scale-up Lambda to MiniStack SSM, and consumed +from SSM by the lifecycle hook. No external JIT configuration variable is +required. + +### 6. Run the standard scale-up scenario + +For job `123456`, the harness: + +1. Deletes the provider's cached runner-group parameter. +2. Clears MockServer request history. +3. Creates and signs a `workflow_job` webhook using the configured secret. +4. Sends the webhook to the deployed endpoint. +5. Waits for the webhook, dispatcher, and provider scale-up Lambda logs. +6. Verifies the GitHub token and queued-job API routes. +7. Discovers the created compute resource. +8. Verifies provider-specific resource state and ownership metadata. + +For EC2 this resource is an instance. For MicroVM it is a MicroVM plus SSM +metadata under the configured MicroVM paths. + +### 7. Exercise the MicroVM lifecycle hook + +When the `microvm` provider is selected, the harness automatically builds and +starts the lifecycle-hook container on `127.0.0.1:8080`: + +```sh +MINISTACK_GITHUB_MOCK_URL=http://localhost:1080 \ + python3 tests/ministack/run-webhook-smoke.py microvm +``` + +When enabled, `MicrovmProvider.configure()` builds and starts the local image +once before the lifecycle scenarios: + +1. Reads the `microvm` Terraform output. +2. Logs in to MiniStack ECR. +3. Pulls, tags, and pushes the ARM64 Ubuntu base image. +4. Exports the MicroVM foundation outputs to the Packer environment. +5. Runs `packer build .` from `images/microvm-ubuntu`. +6. Copies the lifecycle-hook ZIP already produced by CI into the Docker build + context. +7. Builds the `microvm-lifecycle-hook` ARM64 Docker image. +8. Starts the `microvm-lifecycle-hook` container with Docker's default bridge + network, publishes `8080:8080`, and waits for its readiness endpoint. The + `host.docker.internal` host-gateway mapping lets it reach MiniStack on the + host-published port. + +After the MicroVM scale-up creates the resource, the test waits for the SSM +JIT parameter created by that scale-up. It then sends the same version-1 +`runHookPayload` shape used by the MicroVM control-plane code when it calls +`RunMicrovm`: + +```json +{ + "version": 1, + "imageArn": "", + "imageVersion": "", + "runnerConfigSsmPath": "/github-action-runners/multi-runner-webhook/microvm/runners/config", + "runnerTokenSsmPath": "/github-action-runners/multi-runner-webhook/microvm/runners/tokens" +} +``` + +The outer request contains the MicroVM identifier and the payload encoded as a +JSON string. The hook uses the MicroVM identifier and token path to consume the +JIT value written by scale-up. The test waits until the token parameter is +gone, proving that the one-time SSM value was consumed. + +The image build is guarded by the provider instance and happens only once per +smoke-test run, not once per scale-up. + +The lifecycle ZIP must already exist at: + +```text +lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip +``` + +The CI pipeline produces this artifact before running the MicroVM image build. + +### 8. Run the standard scale-down scenario + +The provider-specific scale-down implementation: + +1. Adds MockServer runner-list, runner-detail, and delete expectations. +2. Keeps all active smoke resources visible and marks only the selected runner + as removable. +3. Invokes the provider scale-down Lambda directly. +4. Waits for the scale-down log marker. +5. Verifies the installation-token, runner-list, runner-detail, and runner + deletion routes. +6. Changes the selected runner lookup to HTTP 404. +7. Verifies that the compute resource is terminated. + +For MicroVM, the hook termination endpoint is also called after the resource +termination check. + +### 9. Run the dynamic-label scenario + +The same scale-up and scale-down sequence is repeated for job `123457`, but +the event includes the provider-specific dynamic label: + +- EC2: `ghr-ec2-instance-type:m5.large`; +- MicroVM: `ghr-microvm-image-version:3.0`. + +The test verifies that the resource uses the requested dynamic configuration. + +### 10. Run the pool scenario + +The pool path is invoked directly rather than through a webhook: + +```json +{"poolSize": 1, "type": "ec2|microvm"} +``` + +The harness verifies the pool GitHub routes, discovers the created resource, +checks its provider-specific state, and then performs the same scale-down +assertions. + +### 11. Cleanup + +On success or failure, the harness: + +- marks the checklist as `cleanup`, `passed`, or `failed`; +- removes the local MicroVM Docker container when used; +- terminates discovered EC2 instances; +- terminates discovered MicroVMs; +- destroys the `multi-runner-webhook` Terraform deployment; +- removes temporary variables and response files. + +Use `--keep-deployment` or `MINISTACK_SMOKE_KEEP_DEPLOYMENT=1` when the +Terraform deployment and temporary variables are needed for investigation. + +## Provider responsibilities + +| File | Responsibility | +| --- | --- | +| `lifecycle.py` | Shared standard, dynamic, pool, and scale-down scenarios | +| `provider.py` | Provider interface and resource abstraction | +| `ec2.py` | EC2 discovery, tag assertions, and termination | +| `microvm.py` | MicroVM discovery, metadata assertions, image build, hook handoff, and termination | +| `common.py` | Terraform, AWS CLI, HTTP, MockServer, checklist, and cleanup plumbing | +| `fixtures/` | Static GitHub API and workflow-job test data | + +## Troubleshooting + +The checklist is the first artifact to inspect: + +```sh +cat ministack-smoke-checklist.txt +``` + +For a retained deployment, inspect Terraform state and outputs from: + +```sh +terraform -chdir=examples/multi-runner-webhook output +``` + +If a route assertion times out, check the relevant Lambda log group and the +MockServer request history. If the MicroVM hook is enabled, confirm that the +container is ready at: + +```sh +curl --fail --request POST \ + http://127.0.0.1:8080/aws/lambda-microvms/runtime/v1/ready +``` diff --git a/tests/ministack/smoke/__init__.py b/tests/ministack/smoke/__init__.py new file mode 100644 index 0000000000..e3cfc2fae6 --- /dev/null +++ b/tests/ministack/smoke/__init__.py @@ -0,0 +1 @@ +"""Provider-specific MiniStack smoke checks.""" diff --git a/tests/ministack/smoke/common.py b/tests/ministack/smoke/common.py new file mode 100644 index 0000000000..0bb53f5d40 --- /dev/null +++ b/tests/ministack/smoke/common.py @@ -0,0 +1,607 @@ +"""Shared MiniStack smoke-test plumbing.""" + +from __future__ import annotations + +import base64 +from contextlib import contextmanager +import hashlib +import hmac +import json +import os +import shlex +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Iterator +from urllib.parse import urlsplit, urlunsplit + +class SmokeContext: + def __init__(self, script_dir: Path, *, keep_deployment: bool = False) -> None: + self.script_dir = script_dir + self.fixture_dir = Path(__file__).parent / "fixtures" + self.source_root = script_dir.parent.parent + self.example_root = self.source_root / "examples" / "multi-runner-webhook" + self.aws_endpoint = os.environ.get("AWS_ENDPOINT_URL", "http://localhost:4566") + self.region = os.environ.get("AWS_DEFAULT_REGION", "eu-west-1") + self.environment = os.environ.copy() + self.environment.setdefault("AWS_ACCESS_KEY_ID", "000000000000") + self.environment.setdefault("AWS_SECRET_ACCESS_KEY", "test-only") + self.environment.setdefault("AWS_DEFAULT_REGION", self.region) + self.environment.setdefault("AWS_REGION", self.region) + self.environment.setdefault("AWS_ENDPOINT_URL", self.aws_endpoint) + self.environment.setdefault("AWS_EC2_METADATA_DISABLED", "true") + self.mock_host = os.environ.get("MINISTACK_GITHUB_MOCK_HOST", "host.docker.internal") + self.mock_url = os.environ.get("MINISTACK_GITHUB_MOCK_URL") + self.mock_port = int(os.environ.get("MINISTACK_GITHUB_MOCK_PORT", "0") or 0) + self.tfvars_path: Path | None = None + self.webhook_endpoint = "" + self.webhook_secret = "" + self.discovered_instance_ids: list[str] = [] + self.discovered_microvm_ids: list[str] = [] + self.before_microvm_ids: set[str] = set() + self.response_path = Path(tempfile.mkstemp(prefix="ministack-smoke-response.")[1]) + self.checklist_path = Path(os.environ.get("MINISTACK_SMOKE_CHECKLIST_FILE", "ministack-smoke-checklist.txt")) + self.log_path = Path(os.environ.get("MINISTACK_SMOKE_LOG_FILE", "ministack-smoke.log")) + self.checklist: dict[str, list[dict[str, str | bool]]] = {} + self.checklist_failure = "" + self.keep_deployment = keep_deployment or os.environ.get("MINISTACK_SMOKE_KEEP_DEPLOYMENT") == "1" + self.step_depth = 0 + + def _redact_log(self, value: str) -> str: + for secret in ( + self.webhook_secret, + self.environment.get("AWS_ACCESS_KEY_ID", ""), + self.environment.get("AWS_SECRET_ACCESS_KEY", ""), + ): + if secret: + value = value.replace(secret, "[REDACTED]") + return value + + def _append_log(self, value: str) -> None: + if not value: + return + self.log_path.parent.mkdir(parents=True, exist_ok=True) + with self.log_path.open("a", encoding="utf-8") as log_file: + log_file.write(self._redact_log(value)) + + def progress(self, message: str) -> None: + line = f"{' ' * self.step_depth}{message}" + print(line, flush=True) + self._append_log(f"{line}\n") + + @contextmanager + def step(self, name: str) -> Iterator[None]: + github_actions = self.environment.get("GITHUB_ACTIONS") == "true" + if github_actions: + group_start = f"::group::{name}\n" + print(group_start, end="", flush=True) + self._append_log(group_start) + else: + self.progress(name) + self.step_depth += 1 + try: + yield + finally: + self.step_depth -= 1 + if github_actions: + group_end = "::endgroup::\n" + print(group_end, end="", flush=True) + self._append_log(group_end) + + def _log_command_output( + self, + command: list[str], + stdout: str | None, + stderr: str | None, + *, + include_command: bool = True, + ) -> None: + output = "" + if include_command: + output += f"\n$ {shlex.join(command)}\n" + if stdout: + output += stdout + if stderr: + output += stderr + self._append_log(output) + + def command(self, name: str) -> None: + if not shutil_which(name): + raise RuntimeError(f"{name} is required to run the MiniStack smoke test") + + def run( + self, + command: list[str], + *, + check: bool = True, + stream: bool = False, + cwd: Path | None = None, + log_output: bool = True, + ) -> subprocess.CompletedProcess[str]: + if not stream: + try: + result = subprocess.run( + command, + check=check, + text=True, + capture_output=True, + env=self.environment, + cwd=cwd, + ) + except subprocess.CalledProcessError as error: + if log_output: + self._log_command_output(command, error.stdout, error.stderr) + raise + if log_output: + self._log_command_output(command, result.stdout, result.stderr) + return result + + log_file = self.log_path.open("a", encoding="utf-8") if log_output else None + try: + process = subprocess.Popen( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + env=self.environment, + cwd=cwd, + ) + except BaseException: + if log_file is not None: + log_file.close() + raise + + output: list[str] = [] + try: + if log_file is not None: + log_file.write(self._redact_log(f"\n$ {shlex.join(command)}\n")) + assert process.stdout is not None + for line in process.stdout: + output.append(line) + if log_file is not None: + log_file.write(self._redact_log(line)) + log_file.flush() + return_code = process.wait() + finally: + if log_file is not None: + log_file.close() + + result = subprocess.CompletedProcess(command, return_code, "".join(output), None) + if check and return_code != 0: + self.progress(f"Command failed: {shlex.join(command)}") + sys.stdout.write(self._redact_log(result.stdout)) + sys.stdout.flush() + raise subprocess.CalledProcessError(return_code, command, output=result.stdout) + return result + + def aws(self, *args: str, check: bool = True) -> Any: + result = self.run( + ["aws", "--endpoint-url", self.aws_endpoint, "--region", self.region, *args, "--output", "json"], + check=check, + ) + if not result.stdout.strip(): + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return result.stdout.strip() + + def terraform(self, *args: str, check: bool = True, log_output: bool = True) -> str: + result = self.run( + ["terraform", f"-chdir={self.example_root}", *args], + check=check, + log_output=log_output, + ) + return result.stdout.strip() + + def http(self, method: str, url: str, body: Any = None) -> tuple[int, str]: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request( + url, + data=data, + headers={"Content-Type": "application/json"} if data else {}, + method=method, + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, response.read().decode() + except urllib.error.HTTPError as error: + return error.code, error.read().decode() + except (TimeoutError, urllib.error.URLError) as error: + return 0, str(error) + + def wait_for(self, predicate, description: str, attempts: int = 60) -> Any: + for attempt in range(1, attempts + 1): + result = predicate() + if result: + return result + if attempt == 1 or attempt % 10 == 0: + self.progress(f"Still waiting for {description} ({attempt}/{attempts})") + time.sleep(2) + raise RuntimeError(f"Timed out waiting for {description}") + + def initialize_checklist(self, providers: list[str]) -> None: + checks = ( + ("webhook", "API Gateway accepted the signed workflow_job webhook (HTTP 201)"), + ("chain", "Webhook, EventBridge, dispatcher, SQS, and scale-up logs contain the workflow job"), + ("scale_up_standard_routes", "Standard scale-up called every expected GitHub API route"), + ("scale_up_standard_resource", "Standard scale-up created the expected compute resource"), + ("scale_up_dynamic_routes", "Dynamic-label scale-up called every expected GitHub API route"), + ("scale_up_dynamic_resource", "Dynamic-label scale-up created the expected compute resource"), + ("pool_routes", "Pool called every expected GitHub API route"), + ("pool_resource", "Pool created the expected compute resource"), + ("scale_down_standard", "Scale-down removed the standard runner and compute resource"), + ("scale_down_dynamic", "Scale-down removed the dynamic-label runner and compute resource"), + ("scale_down_pool", "Scale-down removed the pool runner and compute resource"), + ) + if "microvm" in providers: + checks += (("microvm_hook", "MicroVM lifecycle hook consumed SSM and handed off the JIT runner"),) + self.checklist = { + provider: [{"key": key, "label": label, "passed": False} for key, label in checks] + for provider in providers + } + self._write_checklist("running") + self.progress(f"Writing smoke checklist to {self.checklist_path}") + self.progress(f"Writing smoke command output to {self.log_path}") + + def mark_check(self, provider: str, key: str) -> None: + for check in self.checklist.get(provider, []): + if check["key"] == key: + check["passed"] = True + break + self._write_checklist("running") + + def record_checklist_failure(self, error: BaseException) -> None: + self.checklist_failure = f"{type(error).__name__}: {error}" + + def finish_checklist(self, passed: bool) -> None: + self._write_checklist("passed" if passed else "failed") + + def update_checklist_status(self, status: str) -> None: + self._write_checklist(status) + + def _write_checklist(self, status: str) -> None: + lines = [f"MiniStack multi-runner-webhook smoke checklist", f"Status: {status}"] + if self.checklist_failure: + lines.append(f"Failure: {self.checklist_failure}") + for provider, checks in self.checklist.items(): + lines.append("") + lines.append(f"[{provider}]") + lines.extend(f" [{'x' if check['passed'] else ' '}] {check['label']}" for check in checks) + self.checklist_path.parent.mkdir(parents=True, exist_ok=True) + self.checklist_path.write_text("\n".join(lines) + "\n") + + def configure_mockserver(self) -> None: + if not self.mock_url: + raise RuntimeError("MINISTACK_GITHUB_MOCK_URL must point to an already-running MockServer") + self.progress(f"Using external MockServer at {self.mock_url}") + if not self.mock_port: + self.mock_port = urlsplit(self.mock_url).port or 1080 + self.wait_for(lambda: self.http("PUT", f"{self.mock_url}/mockserver/status")[0] < 300, "MockServer") + expectations = json.loads((self.fixture_dir / "github-api-expectations.json").read_text()) + for expectation in expectations: + status, body = self.http("PUT", f"{self.mock_url}/mockserver/expectation", expectation) + if status >= 300: + raise RuntimeError(f"MockServer rejected an expectation: {status} {body}") + + def add_expectation(self, method: str, path: str, status: int, body: Any = None) -> None: + response: dict[str, Any] = {"statusCode": status} + if body is not None: + response.update(headers={"Content-Type": ["application/json"]}, body=json.dumps(body)) + code, text = self.http( + "PUT", + f"{self.mock_url}/mockserver/expectation", + {"httpRequest": {"method": method, "path": path}, "httpResponse": response}, + ) + if code >= 300: + raise RuntimeError(f"MockServer expectation failed: {code} {text}") + + def clear_expectation(self, method: str, path: str) -> None: + code, text = self.http( + "PUT", + f"{self.mock_url}/mockserver/clear", + {"httpRequest": {"method": method, "path": path}}, + ) + if code >= 300: + raise RuntimeError(f"MockServer expectation clear failed: {code} {text}") + + def clear_runner_group_cache(self, provider: str) -> None: + parameter_name = ( + f"/github-action-runners/multi-runner-webhook/{provider}/runners/config/runner-group/Default" + ) + self.progress(f"Clearing {provider} runner-group cache") + self.aws("ssm", "delete-parameter", "--name", parameter_name, check=False) + + def clear_requests(self) -> None: + self.progress("Clearing MockServer request history") + code, _ = self.http("PUT", f"{self.mock_url}/mockserver/clear?type=log") + if code >= 300: + raise RuntimeError("Failed to clear MockServer request history") + self.progress("MockServer request history cleared") + + def verify_route(self, method: str, path: str, description: str) -> None: + def verify() -> bool: + code, _ = self.http( + "PUT", + f"{self.mock_url}/mockserver/verify", + {"httpRequest": {"method": method, "path": path}, "times": {"atLeast": 1}}, + ) + return code < 300 + + try: + self.wait_for(verify, description) + except RuntimeError as error: + raise RuntimeError( + f"{error}. Recent messages in the provider Lambda logs may be available in {self.log_path}" + ) from error + + def send_webhook(self, event: dict[str, Any], delivery_id: str) -> None: + payload = json.dumps(event).encode() + signature = hmac.new(self.webhook_secret.encode(), payload, hashlib.sha256).hexdigest() + parsed_endpoint = urlsplit(self.webhook_endpoint) + endpoint = urlunsplit((parsed_endpoint.scheme, f"localhost:{parsed_endpoint.port or 4566}", parsed_endpoint.path, parsed_endpoint.query, parsed_endpoint.fragment)) + self.progress(f"Sending webhook {delivery_id} to {endpoint} (Host: {parsed_endpoint.netloc})") + request = urllib.request.Request( + endpoint, + data=payload, + headers={ + "Content-Type": "application/json", + "Host": parsed_endpoint.netloc, + "X-GitHub-Event": "workflow_job", + "X-GitHub-Delivery": delivery_id, + "X-GitHub-Hook-Installation-Target-ID": "123", + "X-Hub-Signature-256": f"sha256={signature}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + status = response.status + except urllib.error.HTTPError as error: + status = error.code + body = error.read().decode() + raise RuntimeError( + f"Webhook smoke request failed with HTTP {status} at {endpoint} " + f"(Host: {parsed_endpoint.netloc}): {body}" + ) from error + if status != 201: + raise RuntimeError(f"Webhook smoke request failed with HTTP {status}") + self.progress(f"Webhook {delivery_id} accepted with HTTP 201") + + def wait_for_log(self, group: str, marker: str, description: str) -> None: + self.progress(f"Waiting for {description} ({group})") + def found() -> bool: + events = self.aws("logs", "filter-log-events", "--log-group-name", group, "--filter-pattern", marker, "--limit", "1", check=False) + return bool(events and events.get("events")) + + try: + self.wait_for(found, description) + except RuntimeError as error: + groups = self.aws( + "logs", "describe-log-groups", + "--log-group-name-prefix", "/aws/lambda/multi-runner-webhook", + check=False, + ) or {} + available = [item.get("logGroupName") for item in groups.get("logGroups", [])] + recent = self.aws( + "logs", "filter-log-events", "--log-group-name", group, "--limit", "10", check=False, + ) or {} + messages = [item.get("message", "") for item in recent.get("events", [])] + raise RuntimeError( + f"{error}. Available smoke log groups: {available}. " + f"Recent messages in {group}: {messages}" + ) from error + self.progress(f"Found {description}") + + def recent_log_messages(self, group: str, limit: int = 50) -> list[str]: + events = self.aws( + "logs", + "filter-log-events", + "--log-group-name", + group, + "--limit", + str(limit), + check=False, + ) or {} + return [item.get("message", "") for item in events.get("events", [])] + + def invoke(self, function_name: str, payload: dict[str, Any], description: str) -> None: + payload_path = Path(tempfile.mkstemp(prefix="ministack-smoke-payload.")[1]) + output_path = Path(tempfile.mkstemp(prefix="ministack-smoke-lambda.")[1]) + try: + payload_path.write_text(json.dumps(payload)) + result = self.run([ + "aws", "--endpoint-url", self.aws_endpoint, "--region", self.region, + "lambda", "invoke", "--function-name", function_name, + "--payload", f"fileb://{payload_path}", str(output_path), "--output", "json", + ], check=False) + if result.returncode != 0: + raise RuntimeError( + f"{description} failed with exit code {result.returncode}: " + f"stdout={result.stdout.strip()} stderr={result.stderr.strip()}" + ) + metadata = json.loads(result.stdout) + if metadata.get("FunctionError"): + raise RuntimeError(output_path.read_text()) + finally: + payload_path.unlink(missing_ok=True) + output_path.unlink(missing_ok=True) + + def configure_runner_fixtures( + self, + provider: str, + runners: list[tuple[int, str]], + target_runner_id: int, + ) -> None: + """Expose all active GitHub runners while selecting one for removal. + + The scale-down Lambda evaluates every active provider resource. If the + GitHub list only contains the target runner, the remaining provider + resources are incorrectly marked as orphans and later invocations skip + the normal list-runners path. Keep the other runners visible and busy + so the fixture models a real multi-runner environment and only the + selected runner is eligible for termination. + """ + base = "/api/v3/orgs/test-owner/actions/runners" + if not any(runner_id == target_runner_id for runner_id, _ in runners): + raise RuntimeError(f"Target GitHub runner {target_runner_id} is not in the active runner fixtures") + + github_runners = [ + { + "id": runner_id, + "name": f"{provider}-{resource_id}", + "os": "linux", + "status": "offline", + "busy": runner_id != target_runner_id, + "labels": [], + } + for runner_id, resource_id in runners + ] + paths = [("GET", base)] + paths.extend(("GET", f"{base}/{runner_id}") for runner_id, _ in runners) + paths.append(("DELETE", f"{base}/{target_runner_id}")) + for method, path in paths: + self.http("PUT", f"{self.mock_url}/mockserver/clear", {"httpRequest": {"method": method, "path": path}}) + self.add_expectation("GET", base, 200, {"total_count": len(github_runners), "runners": github_runners}) + for runner in github_runners: + self.add_expectation("GET", f"{base}/{runner['id']}", 200, runner) + self.add_expectation("DELETE", f"{base}/{target_runner_id}", 204) + + def configure_runner_removed(self, runner_id: int) -> None: + path = f"/api/v3/orgs/test-owner/actions/runners/{runner_id}" + self.http("PUT", f"{self.mock_url}/mockserver/clear", {"httpRequest": {"method": "GET", "path": path}}) + self.add_expectation("GET", path, 404, {"message": "Not Found"}) + + def configure_empty_runner_list(self) -> None: + path = "/api/v3/orgs/test-owner/actions/runners" + self.http("PUT", f"{self.mock_url}/mockserver/clear", {"httpRequest": {"method": "GET", "path": path}}) + self.add_expectation("GET", path, 200, {"total_count": 0, "runners": []}) + + def assert_runner_removed(self, runner_id: int) -> None: + code, _ = self.http("GET", f"{self.mock_url}/api/v3/orgs/test-owner/actions/runners/{runner_id}") + if code != 404: + raise RuntimeError(f"Expected runner {runner_id} to be removed, got HTTP {code}") + + def scale_down_routes(self, runner_id: int, log_group: str | None = None) -> None: + try: + self.verify_route("POST", "/api/v3/app/installations/123/access_tokens", "Scale-down requested a GitHub App token") + self.verify_route("GET", "/api/v3/orgs/test-owner/actions/runners", "Scale-down listed organization runners") + self.verify_route("GET", f"/api/v3/orgs/test-owner/actions/runners/{runner_id}", "Scale-down checked runner state") + self.verify_route("DELETE", f"/api/v3/orgs/test-owner/actions/runners/{runner_id}", "Scale-down deleted the GitHub runner") + except RuntimeError as error: + if log_group is None: + raise + messages = self.recent_log_messages(log_group) + raise RuntimeError(f"{error}. Recent messages in {log_group}: {messages}") from error + + def scale_up_routes(self, job_id: int, provider: str) -> None: + self.verify_route("POST", "/api/v3/app/installations/123/access_tokens", f"{provider} scale-up requested a GitHub token for {job_id}") + self.verify_route("GET", f"/api/v3/repos/test-owner/test-repo/actions/jobs/{job_id}", f"{provider} scale-up checked queued job {job_id}") + + def pool_routes(self, provider: str) -> None: + self.verify_route("GET", "/api/v3/orgs/test-owner/installation", f"{provider} pool looked up the GitHub App installation") + self.verify_route("POST", "/api/v3/app/installations/123/access_tokens", f"{provider} pool requested a GitHub token") + self.verify_route("GET", "/api/v3/orgs/test-owner/actions/runners", f"{provider} pool listed organization runners") + + def prepare(self) -> None: + self.progress("Preparing multi-runner-webhook smoke deployment") + commands = ("aws", "openssl", "terraform") + for command in commands: + self.command(command) + self.configure_mockserver() + key_path = Path(tempfile.mkstemp(prefix="ministack-smoke-key.")[1]) + try: + self.run(["openssl", "genpkey", "-algorithm", "RSA", "-pkeyopt", "rsa_keygen_bits:2048", "-out", str(key_path)], check=True) + key = base64.b64encode(key_path.read_bytes()).decode() + finally: + key_path.unlink(missing_ok=True) + source = self.script_dir / "multi-runner-webhook.tfvars" + text = source.read_text().replace('key_base64 = "ministack-invalid-key"', f'key_base64 = "{key}"') + self.tfvars_path = Path(tempfile.mkstemp(prefix="terraform-aws-github-runner-smoke.")[1]) + additions = [ + "github_enterprise_server = {", + f' url = "http://{self.mock_host}:{self.mock_port}"', + " ssl_verify = false", + "}", + ] + if "runners_lambda_zip" not in text: + additions.append(f'runners_lambda_zip = "{self.source_root / "lambdas/functions/control-plane/runners.zip"}"') + if "webhook_lambda_zip" not in text: + additions.append(f'webhook_lambda_zip = "{self.source_root / "lambdas/functions/webhook/webhook.zip"}"') + self.tfvars_path.write_text(text + "\n" + "\n".join(additions) + "\n") + self.progress("Applying multi-runner-webhook Terraform example") + self.run( + [str(self.source_root / "tests/ministack/run-example.sh"), "apply", "multi-runner-webhook", str(self.tfvars_path)], + stream=True, + ) + self.webhook_endpoint = self.terraform("output", "-raw", "webhook_endpoint") + self.webhook_secret = self.terraform("output", "-raw", "webhook_secret", log_output=False) + self.wait_for_webhook_route() + self.progress("Deployment ready; starting provider lifecycle checks") + + def wait_for_webhook_route(self) -> None: + hostname = urlsplit(self.webhook_endpoint).hostname + if not hostname: + raise RuntimeError(f"Invalid webhook endpoint: {self.webhook_endpoint}") + api_id = hostname.split(".", 1)[0] + + def route_ready() -> bool: + routes = self.aws( + "apigatewayv2", + "get-routes", + "--api-id", + api_id, + check=False, + ) or {} + return any(route.get("RouteKey") == "POST /webhook" for route in routes.get("Items", [])) + + self.wait_for(route_ready, "API Gateway POST /webhook route", attempts=30) + + def cleanup(self) -> None: + self.update_checklist_status("cleanup") + if "microvm" in self.checklist: + self.progress("MicroVM lifecycle hook container logs (last 200 lines):") + self.run( + ["docker", "logs", "--timestamps", "--tail", "200", "microvm-lifecycle-hook"], + check=False, + stream=True, + ) + self.run(["docker", "rm", "--force", "microvm-lifecycle-hook"], check=False) + for instance_id in self.discovered_instance_ids: + self.aws("ec2", "terminate-instances", "--instance-ids", instance_id, check=False) + for microvm_id in self.discovered_microvm_ids: + self.aws( + "lambda-microvms", + "terminate-microvm", + "--microvm-identifier", + microvm_id, + check=False, + ) + if self.tfvars_path and self.keep_deployment: + self.progress(f"Terraform deployment retained; tfvars file: {self.tfvars_path}") + elif self.tfvars_path: + self.progress("Destroying multi-runner-webhook Terraform deployment") + self.run( + [ + str(self.source_root / "tests/ministack/run-example.sh"), + "destroy", + "multi-runner-webhook", + str(self.tfvars_path), + ], + stream=True, + ) + self.tfvars_path.unlink(missing_ok=True) + self.response_path.unlink(missing_ok=True) + + +def shutil_which(name: str) -> str | None: + for directory in os.environ.get("PATH", "").split(os.pathsep): + candidate = Path(directory) / name + if candidate.is_file() and os.access(candidate, os.X_OK): + return str(candidate) + return None diff --git a/tests/ministack/smoke/ec2.py b/tests/ministack/smoke/ec2.py new file mode 100644 index 0000000000..085309be35 --- /dev/null +++ b/tests/ministack/smoke/ec2.py @@ -0,0 +1,160 @@ +"""EC2 implementation of the provider smoke-test interface.""" + +import base64 +import json +from typing import Any + +from .common import SmokeContext +from .provider import RunnerResource + + +class Ec2Provider: + slug = "ec2" + display_name = "EC2" + + def configure(self, context: SmokeContext) -> None: + self._configure_jit_expectations(context) + + def _configure_jit_expectations(self, context: SmokeContext) -> None: + context.add_expectation( + "GET", + "/api/v3/orgs/test-owner/actions/runner-groups", + 200, + [{"id": 1, "name": "Default"}], + ) + context.add_expectation( + "POST", + "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", + 200, + { + "runner": {"id": 987654321, "labels": [{"name": "self-hosted"}, {"name": "linux"}]}, + # EC2 does not launch the runner in this smoke; keep the fixture + # Base64-shaped so it cannot mask a JIT handoff failure. + "encoded_jit_config": base64.b64encode(b"{}").decode(), + }, + ) + + def event(self, context: SmokeContext, job_id: int, dynamic: bool) -> dict[str, Any]: + value = json.loads((context.fixture_dir / "workflow_job_event.json").read_text()) + job = value["workflow_job"] + job["id"] = job_id + job["name"] = f"multi-runner-webhook-ec2-{job_id}" + job["labels"] = ["self-hosted", "linux", "x64", "ec2"] + if dynamic: + job["labels"].append("ghr-ec2-instance-type:m5.large") + return value + + def verify_scale_up_routes(self, context: SmokeContext, job_id: int) -> None: + context.verify_route( + "GET", + "/api/v3/orgs/test-owner/actions/runner-groups", + f"EC2 scale-up resolved the runner group for {job_id}", + ) + context.verify_route( + "POST", + "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", + f"EC2 scale-up generated JIT configuration for {job_id}", + ) + + def verify_pool_routes(self, context: SmokeContext) -> None: + context.verify_route( + "GET", + "/api/v3/orgs/test-owner/actions/runner-groups", + "EC2 pool resolved the runner group", + ) + context.verify_route( + "POST", + "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", + "EC2 pool generated JIT configuration", + ) + + def _wait_for_instance(self, context: SmokeContext, source: str, description: str) -> RunnerResource: + def find() -> str | None: + result = context.aws( + "ec2", "describe-instances", "--filters", + "Name=instance-state-name,Values=running,pending", + "Name=tag:ghr:Application,Values=github-action-runner", + f"Name=tag:ghr:created_by,Values={source}", check=False, + ) or {} + for reservation in result.get("Reservations", []): + for instance in reservation.get("Instances", []): + instance_id = instance.get("InstanceId") + if instance_id and instance_id not in context.discovered_instance_ids: + context.discovered_instance_ids.append(instance_id) + return instance_id + return None + + return RunnerResource(context.wait_for(find, description)) + + def _instance(self, context: SmokeContext, resource: RunnerResource) -> dict[str, Any]: + result = context.aws("ec2", "describe-instances", "--instance-ids", resource.identifier) + return result["Reservations"][0]["Instances"][0] + + def _assert_tags(self, context: SmokeContext, resource: RunnerResource, source: str) -> None: + tags = {tag["Key"]: tag["Value"] for tag in self._instance(context, resource).get("Tags", [])} + expected = { + "ghr:Application": "github-action-runner", + "ghr:created_by": source, + "ghr:Type": "Org", + "ghr:Owner": "test-owner", + } + for key, value in expected.items(): + if tags.get(key) != value: + raise RuntimeError(f"Unexpected EC2 runner tag {key}: expected {value}, got {tags.get(key)}") + + def wait_for_scale_up(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_instance(context, source, "an EC2 scale-up instance") + + def assert_scale_up(self, context: SmokeContext, resource: RunnerResource, dynamic: bool) -> None: + expected_type = "m5.large" if dynamic else "m7a.large" + actual_type = self._instance(context, resource).get("InstanceType") + if actual_type != expected_type: + raise RuntimeError(f"EC2 scale-up used {actual_type}, expected {expected_type}") + self._assert_tags(context, resource, "scale-up-lambda") + + def start_scale_up_runner(self, context: SmokeContext, resource: RunnerResource) -> bool: + return False + + def wait_for_pool(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_instance(context, source, "an EC2 pool instance") + + def assert_pool(self, context: SmokeContext, resource: RunnerResource) -> None: + self._assert_tags(context, resource, "pool-lambda") + + def _wait_for_termination(self, context: SmokeContext, resource: RunnerResource) -> None: + def terminated() -> bool: + result = context.aws("ec2", "describe-instances", "--instance-ids", resource.identifier, check=False) + if not result: + return True + state = result.get("Reservations", [{}])[0].get("Instances", [{}])[0].get("State", {}).get("Name") + return state in (None, "terminated") + + context.wait_for(terminated, f"EC2 instance {resource.identifier} termination") + + def scale_down( + self, + context: SmokeContext, + resource: RunnerResource, + runner_id: int, + marker: str, + active_runners: list[tuple[int, RunnerResource]], + ) -> None: + context.configure_runner_fixtures( + self.slug, + [(active_runner_id, active_resource.identifier) for active_runner_id, active_resource in active_runners], + runner_id, + ) + context.clear_requests() + context.invoke( + "multi-runner-webhook-ec2-scale-down", + {"smokeMarker": marker, "type": "ec2"}, + "EC2 scale-down Lambda invoked", + ) + context.wait_for_log("/aws/lambda/multi-runner-webhook-ec2-scale-down", marker, "EC2 scale-down Lambda started") + context.scale_down_routes(runner_id, "/aws/lambda/multi-runner-webhook-ec2-scale-down") + context.configure_runner_removed(runner_id) + context.assert_runner_removed(runner_id) + self._wait_for_termination(context, resource) + + +provider = Ec2Provider() diff --git a/tests/ministack/github-api-expectations.json b/tests/ministack/smoke/fixtures/github-api-expectations.json similarity index 100% rename from tests/ministack/github-api-expectations.json rename to tests/ministack/smoke/fixtures/github-api-expectations.json diff --git a/tests/ministack/workflow_job_event.json b/tests/ministack/smoke/fixtures/workflow_job_event.json similarity index 100% rename from tests/ministack/workflow_job_event.json rename to tests/ministack/smoke/fixtures/workflow_job_event.json diff --git a/tests/ministack/smoke/lifecycle.py b/tests/ministack/smoke/lifecycle.py new file mode 100644 index 0000000000..cef2f725a8 --- /dev/null +++ b/tests/ministack/smoke/lifecycle.py @@ -0,0 +1,137 @@ +"""Provider-neutral webhook and runner lifecycle scenarios.""" + +from __future__ import annotations + +from uuid import uuid4 + +from .common import SmokeContext +from .provider import RunnerResource, SmokeProvider + +MOCK_JIT_RUNNER_ID = 987654321 + + +def _log_group(provider: SmokeProvider, stage: str) -> str: + # The webhook and EventBridge dispatcher are shared by all compute + # providers. Only the scale-up Lambda is provider-specific. + if stage in ("webhook", "dispatch-to-runner"): + return f"/aws/lambda/multi-runner-webhook-{stage}" + return f"/aws/lambda/multi-runner-webhook-{provider.slug}-{stage}" + + +def _wait_for_webhook_chain(context: SmokeContext, provider: SmokeProvider, job_id: int) -> None: + context.wait_for_log(_log_group(provider, "webhook"), str(job_id), f"{provider.display_name} webhook received job {job_id}") + context.wait_for_log(_log_group(provider, "dispatch-to-runner"), str(job_id), f"{provider.display_name} dispatcher received job {job_id}") + context.wait_for_log(_log_group(provider, "scale-up"), str(job_id), f"{provider.display_name} scale-up received job {job_id}") + + +def _scale_up( + context: SmokeContext, + provider: SmokeProvider, + job_id: int, + dynamic: bool, + source: str, +) -> RunnerResource: + label_mode = "with dynamic label" if dynamic else "without dynamic label" + with context.step(f"{provider.display_name}: scale-up {label_mode} (job {job_id})"): + with context.step("Prepare scale-up fixtures"): + context.clear_runner_group_cache(provider.slug) + context.clear_requests() + with context.step("Send webhook"): + context.send_webhook( + provider.event(context, job_id, dynamic), + f"multi-runner-webhook-{provider.slug}-{job_id}", + ) + context.mark_check(provider.slug, "webhook") + with context.step("Wait for webhook chain"): + _wait_for_webhook_chain(context, provider, job_id) + context.mark_check(provider.slug, "chain") + with context.step("Verify shared scale-up routes"): + context.scale_up_routes(job_id, provider.display_name) + with context.step("Verify provider scale-up routes"): + provider.verify_scale_up_routes(context, job_id) + context.mark_check(provider.slug, "scale_up_dynamic_routes" if dynamic else "scale_up_standard_routes") + with context.step("Wait for compute resource"): + resource = provider.wait_for_scale_up(context, source) + with context.step("Validate compute resource"): + provider.assert_scale_up(context, resource, dynamic) + context.mark_check(provider.slug, "scale_up_dynamic_resource" if dynamic else "scale_up_standard_resource") + with context.step("Start runner"): + if provider.start_scale_up_runner(context, resource): + context.mark_check(provider.slug, "microvm_hook") + return resource + + +def _pool(context: SmokeContext, provider: SmokeProvider, pool_size: int) -> RunnerResource: + with context.step(f"{provider.display_name}: pool scale-up (target size {pool_size})"): + with context.step("Prepare pool fixtures"): + context.configure_empty_runner_list() + context.clear_runner_group_cache(provider.slug) + context.clear_requests() + with context.step("Invoke pool Lambda"): + context.invoke( + f"multi-runner-webhook-{provider.slug}-pool", + {"poolSize": pool_size, "type": provider.slug}, + f"{provider.display_name} pool Lambda invoked", + ) + with context.step("Verify shared pool routes"): + context.pool_routes(provider.display_name) + with context.step("Verify provider pool routes"): + provider.verify_pool_routes(context) + context.mark_check(provider.slug, "pool_routes") + with context.step("Wait for compute resource"): + resource = provider.wait_for_pool(context, "pool-lambda") + with context.step("Validate compute resource"): + context.progress(f"Pool created resource {resource.identifier}") + provider.assert_pool(context, resource) + context.mark_check(provider.slug, "pool_resource") + return resource + + +def _scale_down( + context: SmokeContext, + provider: SmokeProvider, + resource: RunnerResource, + runner_id: int, + marker: str, + check: str, +) -> None: + with context.step(f"{provider.display_name}: scale-down resource {resource.identifier}"): + with context.step("Run provider scale-down checks"): + provider.scale_down(context, resource, runner_id, marker, [(runner_id, resource)]) + context.mark_check(provider.slug, check) + + +def run(context: SmokeContext, provider: SmokeProvider) -> None: + with context.step(f"{provider.display_name} lifecycle"): + with context.step("Configure"): + provider.configure(context) + + scale_up = _scale_up(context, provider, 123456, False, "scale-up-lambda") + _scale_down( + context, + provider, + scale_up, + MOCK_JIT_RUNNER_ID, + f"multi-runner-webhook-{provider.slug}-scale-up-scale-down-{uuid4().hex}", + "scale_down_standard", + ) + + dynamic_scale_up = _scale_up(context, provider, 123457, True, "scale-up-lambda") + _scale_down( + context, + provider, + dynamic_scale_up, + MOCK_JIT_RUNNER_ID, + f"multi-runner-webhook-{provider.slug}-dynamic-scale-down-{uuid4().hex}", + "scale_down_dynamic", + ) + + pool = _pool(context, provider, pool_size=1) + _scale_down( + context, + provider, + pool, + MOCK_JIT_RUNNER_ID, + f"multi-runner-webhook-{provider.slug}-pool-scale-down-{uuid4().hex}", + "scale_down_pool", + ) diff --git a/tests/ministack/smoke/microvm.py b/tests/ministack/smoke/microvm.py new file mode 100644 index 0000000000..66e5d01941 --- /dev/null +++ b/tests/ministack/smoke/microvm.py @@ -0,0 +1,486 @@ +"""MicroVM implementation of the provider smoke-test interface.""" + +import base64 +import json +import shutil +from typing import Any + +from .common import SmokeContext +from .provider import RunnerResource + +MICROVM_HOOK_CONTAINER = "microvm-lifecycle-hook" +MICROVM_HOOK_PORT = 8080 +MICROVM_HOOK_URL = f"http://127.0.0.1:{MICROVM_HOOK_PORT}" + + +def _base64_json(value: dict[str, Any]) -> str: + return base64.b64encode(json.dumps(value, separators=(",", ":")).encode()).decode() + + +def _smoke_jit_config(context: SmokeContext) -> str: + """Return a synthetic but runner-compatible JIT configuration for MockServer.""" + # The smoke validates JIT handoff, not the runner service protocol. Keep the + # launched runner away from MockServer, whose REST expectations are for the + # control plane and GitHub API only. + runner_server_url = "http://127.0.0.1:65535" + files = { + ".runner": _base64_json( + { + "AgentId": 987654321, + "AgentName": "ministack-microvm", + "DisableUpdate": True, + "Ephemeral": True, + "PoolId": 1, + "PoolName": "Default", + "ServerUrl": runner_server_url, + "WorkFolder": "_work", + } + ), + ".credentials": _base64_json( + { + "scheme": "OAuth", + "data": { + "clientId": "00000000-0000-0000-0000-000000000000", + "authorizationUrl": f"{runner_server_url}/_apis/oauth2/token", + }, + } + ), + # This is a throwaway RSA key used only to let the runner pass its local + # JIT bootstrap. It does not authenticate against a real GitHub service. + ".credentials_rsaparams": _base64_json( + { + "d": "BnkRwk8qg/fMob7o5QboXqqTJsPX2mO7uw7QQAZdFw8FY0P7GmpaiGRPsyu6hhRHH5n6vkMw3gRWvIcP+0rBQ3S32U+tKf4+CaARP9iongbice0xUDdKZKXrTlqSZ9AUND5ZIGIuFNDFn5qXS2J6SyrvF/LopAUu13lWxDrduyEpWRtJkR4RPNKEHi7Lk8NsaZ6N8AIz3+/y0dkWI3pPmelzi+rAssDnz7soK4o6CG9RjIXzeBQzJ4BXefD6zEeXM++mDZylnVJNOoHJWNLvPN+aL5vCfBmgYkk6KZgYzMCFFsxkwtvw+6el3PumxHayginTZ9kd8QJBTypCLtWHWQ==", + "dp": "lkubjli9JdkRuXqnHXHlSDy38RaNGKy+qEa1v4yQTg6h8ni3ZBDMfDhsNhUtlgBHF1AhYod2qwkCZKRNRWAVg00G1Pxswmt57b+4jfW4J0LQ1AytrxhTSrthlyQR5ikUK3d/kEMQs+yP0f2SapkYqXzdaHiU1RA6IhT35bFhHfU=", + "dq": "r7ZtgewAZCZt21o3fBkhAB08Ct+QV0KkzCJlcDr0QbmbjLg/0Nuy6zeiA2QC609LZPgGv6BnPhHMG11bT401WsSkgu/h56L77fK8GwVKCpeZ8SSn3fwyCSpjRHQx3duTerLgi3paVuJTVgL3FdFKSko7wkdSDI1eu9BSHGYnu18=", + "exponent": "AQAB", + "inverseQ": "FSukzwvdLNKkglKWjmYcs1ZCqcgAxecU3bczzVi1TCIYmLN7bLavs15ezr2Xe7MnUJCVz9Lk61sCDVxAA1XK/Bx88iuZvC9GFuM9wZEflvibycx6KI4dvmfSgM0Gff8BnoLs5WinopSuz/fvCzpB26aNfsuv4eCnBgAn8F8bDGU=", + "modulus": "mEkM5pFWZbsCIhVBw2PHC3OfcgP6UtrabLxkAHw6NfxNxdyfRErU6BeI2e6Sh9bRNlo3GbHtq4CizVhcmwJo6CKc1/r1Zrgbb1xQ/FiiHJDA8J6b7cxY894N7rY2r0PqOAxBruGfyAUgG3eFSC5ZSxJfiJe/sd6gwtetrh1ncoCXfeI3IGzZa/dQtIZkefFoqgv5h45gy5KwcAODZ5G0M0aYksFQyUHPLEqSGESsz8LWUOMm1Fgauj2poy8ZHC4xGvfKISPoONRAbHOQDQ7IP09v/w1iAKt02qy9Xdr4vHzZK34a6/Ug/YJpYuIE4fxyTgi096FVIrOl5v+QZg8h4w==", + "p": "zEqTwQJRTY7JmI+zamHZJ/GZ+Hv6n2RSBwFX7BdBS3rxUzAxrijax7fgmhyd4WUgTnMliWZWW3B61Ez/pzXT39ZtSrelOMa6TCMZBbAfq964X5nlWwAdEfHN0SAffKLjXlVBcF6Ov0nhjB8Ci081kjebO1hEgH8ri5awtJLw2S0=", + "q": "vtSnRA6EtRsr+o2gq0E0RA44hBpMe7NMyMIIAxcM4vyMRsKhGc2+vaHIyXLejiaomPlTYWLjCBGoNx8wFN/K0ZoshEJAPAYBVgX9hX9eywigdoGWufkaJqHG1a5YmIqTRqO9dQs8rItpGGPeJmToPLwPaPz1rZH1/BoBO7pksU8=", + } + ), + } + return _base64_json(files) + + +class MicrovmProvider: + slug = "microvm" + display_name = "MicroVM" + image_arn = "arn:aws:lambda:eu-west-1:000000000000:microvm-image:ministack" + image_version = "3.0" + hook_url = MICROVM_HOOK_URL + runner_config_path = "/github-action-runners/multi-runner-webhook/microvm/runners/config" + runner_token_path = "/github-action-runners/multi-runner-webhook/microvm/runners/tokens" + metadata_path = "/github-action-runners/multi-runner-webhook/microvm/runners/config/microvm-metadata" + + def __init__(self) -> None: + self._runner_image_built = False + self._hook_needs_restart = False + + def configure(self, context: SmokeContext) -> None: + self.build_runner_image(context) + self._configure_jit_expectations(context) + context.before_microvm_ids = set(self._metadata_by_path(context)) + + def _configure_jit_expectations(self, context: SmokeContext) -> None: + runner_group_path = "/api/v3/orgs/test-owner/actions/runner-groups" + jit_config_path = "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig" + context.clear_expectation("GET", runner_group_path) + context.clear_expectation("POST", jit_config_path) + context.add_expectation( + "GET", + runner_group_path, + 200, + [{"id": 1, "name": "Default"}], + ) + context.add_expectation( + "POST", + jit_config_path, + 200, + { + "runner": {"id": 987654321, "labels": [{"name": "self-hosted"}, {"name": "linux"}]}, + "encoded_jit_config": _smoke_jit_config(context), + }, + ) + + def build_runner_image(self, context: SmokeContext) -> None: + """Build the local ARM64 runner image once before lifecycle checks.""" + if self._runner_image_built: + return + + with context.step("Test Packer"): + output = json.loads(context.terraform("output", "-json", "microvm")) + foundation = output["microvm_foundation"] + ecr_repository_uri = output["ecr_repo"] + repository_name = ecr_repository_uri.rsplit("/", 1)[-1] + docker_registry = "localhost:4566" + image_tag = "latest" + docker_base_image = f"{docker_registry}/{repository_name}:{image_tag}" + ubuntu_image = ( + ecr_repository_uri + if ":" in ecr_repository_uri.rsplit("/", 1)[-1] + else f"{ecr_repository_uri}:{image_tag}" + ) + image_root = context.source_root / "images" / "microvm-ubuntu" + image_context = image_root / "packer" / "scripts" / "microvm" / "image" + lifecycle_hook_zip = ( + context.source_root + / "lambda_output/" + / "microvm-lifecycle-hooks.zip" + ) + + with context.step("Build base image"): + context.run( + [ + "bash", + "-o", + "pipefail", + "-c", + "aws ecr get-login-password | " + f"docker login --username AWS --password-stdin {docker_registry}", + ], + stream=True, + ) + context.run(["docker", "pull", "--platform", "linux/arm64", "ubuntu:24.04"], stream=True) + context.run(["docker", "tag", "ubuntu:24.04", docker_base_image], stream=True) + context.run(["docker", "push", docker_base_image], stream=True) + + context.environment.update( + { + "MICROVM_ARTIFACT_BUCKET": foundation["artifact_bucket_name"], + "MICROVM_BUILD_ROLE_ARN": foundation["build_role_arn"], + "MICROVM_EGRESS_NETWORK_CONNECTOR_ARN": foundation["connector_arns"]["ministack"], + "MICROVM_IMAGE_NAME": "micro-ubuntu24", + "MICROVM_MEMORY_MIB": "8192", + "MICROVM_IDEMPOTENCY_NONCE": context.environment.get( + "MICROVM_IDEMPOTENCY_NONCE", "ministack-smoke" + ), + "MICROVM_LOG_GROUP": output.get("log_group", "/aws/lambda/microvms/ubuntu24"), + "MICROVM_UBUNTU_IMAGE": ubuntu_image, + "MICROVM_LIFECYCLE_HOOK_ZIP": str(lifecycle_hook_zip), + } + ) + + with context.step("Build MicroVM image"): + context.run(["packer", "build", "."], cwd=image_root, stream=True) + + with context.step("Build lifecycle-hook image"): + shutil.copy2(lifecycle_hook_zip, image_context / "microvm-lifecycle-hooks.zip") + context.run( + [ + "docker", + "build", + "--platform", + "linux/arm64", + "-f", + str(image_context / "ubuntu24.arm64.Dockerfile"), + "--build-arg", + f"UBUNTU_IMAGE={docker_base_image}", + "--tag", + MICROVM_HOOK_CONTAINER, + str(image_context), + ], + stream=True, + ) + + self._start_microvm_hook(context) + self._runner_image_built = True + + def _start_microvm_hook(self, context: SmokeContext) -> None: + with context.step("Start lifecycle-hook container"): + context.run(["docker", "rm", "--force", MICROVM_HOOK_CONTAINER], check=False) + context.run( + [ + "docker", + "run", + "--detach", + "--rm", + "--platform", + "linux/arm64", + "--name", + MICROVM_HOOK_CONTAINER, + "--add-host=host.docker.internal:host-gateway", + "--publish", + f"{MICROVM_HOOK_PORT}:8080", + "--env", + "AWS_ENDPOINT_URL=http://host.docker.internal:4566", + "--env", + "AWS_REGION=eu-west-1", + "--env", + "AWS_DEFAULT_REGION=eu-west-1", + "--env", + "AWS_ACCESS_KEY_ID=000000000000", + "--env", + "AWS_SECRET_ACCESS_KEY=test", + "--env", + "MICROVM_ID=ministack-microvm", + "--env", + f"RUNNER_CONFIG_SSM_PATH={self.runner_config_path}", + MICROVM_HOOK_CONTAINER, + ], + stream=True, + ) + with context.step("Wait for lifecycle-hook readiness"): + context.wait_for( + lambda: context.run( + [ + "curl", + "--fail", + "--silent", + "--show-error", + "--request", + "POST", + f"http://127.0.0.1:{MICROVM_HOOK_PORT}/aws/lambda-microvms/runtime/v1/ready", + ], + check=False, + ).returncode + == 0, + "MicroVM lifecycle hook container readiness", + attempts=30, + ) + + def event(self, context: SmokeContext, job_id: int, dynamic: bool) -> dict[str, Any]: + value = json.loads((context.fixture_dir / "workflow_job_event.json").read_text()) + job = value["workflow_job"] + job["id"] = job_id + job["name"] = f"multi-runner-webhook-microvm-{job_id}" + job["labels"] = ["self-hosted", "linux", "arm64", "microvm"] + if dynamic: + job["labels"].append(f"ghr-microvm-image-version:{self.image_version}") + return value + + def verify_scale_up_routes(self, context: SmokeContext, job_id: int) -> None: + context.verify_route("GET", "/api/v3/orgs/test-owner/actions/runner-groups", "MicroVM scale-up resolved the runner group") + context.verify_route("POST", "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", "MicroVM scale-up generated JIT configuration") + + def verify_pool_routes(self, context: SmokeContext) -> None: + context.verify_route("GET", "/api/v3/orgs/test-owner/actions/runner-groups", "MicroVM pool resolved the runner group") + context.verify_route("POST", "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", "MicroVM pool generated JIT configuration") + + def _metadata(self, context: SmokeContext, microvm_id: str) -> dict[str, Any]: + value = context.aws( + "ssm", "get-parameter", + "--name", f"{self.metadata_path}/{microvm_id}", + check=False, + ) + if not value or value.get("Parameter", {}).get("Value") in (None, "None"): + raise RuntimeError(f"Missing MicroVM ownership metadata for {microvm_id}") + return json.loads(value["Parameter"]["Value"]) + + def _metadata_by_path(self, context: SmokeContext) -> dict[str, dict[str, Any]]: + result = context.aws( + "ssm", + "get-parameters-by-path", + "--path", + self.metadata_path, + check=False, + ) or {} + prefix = f"{self.metadata_path}/" + metadata: dict[str, dict[str, Any]] = {} + for parameter in result.get("Parameters", []): + name = parameter.get("Name", "") + if not name.startswith(prefix): + continue + microvm_id = name[len(prefix):] + if "." in microvm_id: + continue + value = parameter.get("Value") + if value in (None, "None"): + continue + metadata[microvm_id] = json.loads(value) + return metadata + + def _wait_for_microvm(self, context: SmokeContext, source: str, description: str) -> RunnerResource: + def find() -> str | None: + for microvm_id, metadata in self._metadata_by_path(context).items(): + if microvm_id in context.before_microvm_ids or microvm_id in context.discovered_microvm_ids: + continue + if metadata.get("source") != source: + continue + details = self._details(context, RunnerResource(microvm_id)) + if details.get("state") not in ("PENDING", "RUNNING", "SUSPENDING", "SUSPENDED"): + continue + context.discovered_microvm_ids.append(microvm_id) + return microvm_id + return None + + return RunnerResource(context.wait_for(find, description)) + + def _details(self, context: SmokeContext, resource: RunnerResource) -> dict[str, Any]: + return context.aws( + "lambda-microvms", + "get-microvm", + "--microvm-identifier", + resource.identifier, + check=False, + ) or {} + + def _assert_resource(self, context: SmokeContext, resource: RunnerResource, source: str, dynamic: bool) -> None: + details = self._details(context, resource) + if details.get("state") not in ("PENDING", "RUNNING", "SUSPENDING", "SUSPENDED"): + raise RuntimeError(f"MicroVM {resource.identifier} is not active: {details}") + if details.get("imageArn") != self.image_arn: + raise RuntimeError(f"MicroVM {resource.identifier} used {details.get('imageArn')}, expected {self.image_arn}") + if dynamic and details.get("imageVersion") != self.image_version: + raise RuntimeError(f"MicroVM {resource.identifier} used image version {details.get('imageVersion')}, expected {self.image_version}") + metadata = self._metadata(context, resource.identifier) + expected = { + "environment": "multi-runner-webhook-microvm", + "source": source, + "runnerOwner": "test-owner", + "runnerType": "Org", + } + if any(metadata.get(key) != value for key, value in expected.items()): + raise RuntimeError(f"Unexpected MicroVM metadata: {metadata}") + + def wait_for_scale_up(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_microvm(context, source, "a MicroVM scale-up resource") + + def assert_scale_up(self, context: SmokeContext, resource: RunnerResource, dynamic: bool) -> None: + self._assert_resource(context, resource, "scale-up-lambda", dynamic) + + def start_scale_up_runner(self, context: SmokeContext, resource: RunnerResource) -> bool: + if self._hook_needs_restart: + with context.step("Restart lifecycle-hook container"): + self._start_microvm_hook(context) + self._hook_needs_restart = False + + details = self._details(context, resource) + image_arn = details.get("imageArn") + image_version = details.get("imageVersion") + if not isinstance(image_arn, str) or not isinstance(image_version, str): + raise RuntimeError(f"MicroVM {resource.identifier} has incomplete image details: {details}") + + parameter_name = f"{self.runner_token_path.rstrip('/')}/{resource.identifier}" + context.wait_for( + lambda: context.aws("ssm", "get-parameter", "--name", parameter_name, check=False), + f"MicroVM scale-up to create {parameter_name}", + ) + + run_hook_payload = json.dumps( + { + "version": 1, + "imageArn": image_arn, + "imageVersion": image_version, + "runnerConfigSsmPath": self.runner_config_path, + "runnerTokenSsmPath": self.runner_token_path, + }, + separators=(",", ":"), + ) + request_body = json.dumps( + {"microvmId": resource.identifier, "runHookPayload": run_hook_payload}, + separators=(",", ":"), + ) + context.progress(f"Curling MicroVM runner hook for {resource.identifier}") + result = context.run( + [ + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--max-time", + "10", + "--request", + "POST", + f"{self.hook_url}/aws/lambda-microvms/runtime/v1/run", + "--header", + "Content-Type: application/json", + "--data-raw", + request_body, + ], + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + "MicroVM runner hook curl failed with " + f"exit code {result.returncode}: {result.stderr.strip() or result.stdout.strip()}" + ) + + context.wait_for( + lambda: not context.aws("ssm", "get-parameter", "--name", parameter_name, check=False), + f"MicroVM runner hook to consume {parameter_name}", + ) + github_runner_id_parameter = f"{self.metadata_path}/{resource.identifier}.github-runner-id" + context.wait_for( + lambda: bool( + context.aws( + "ssm", + "get-parameter", + "--name", + github_runner_id_parameter, + check=False, + ) + ), + f"MicroVM scale-up to persist {github_runner_id_parameter}", + ) + return True + + def wait_for_pool(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_microvm(context, source, "a MicroVM pool resource") + + def assert_pool(self, context: SmokeContext, resource: RunnerResource) -> None: + self._assert_resource(context, resource, "pool-lambda", False) + + def _wait_for_termination(self, context: SmokeContext, resource: RunnerResource) -> None: + def terminated() -> bool: + details = context.aws( + "lambda-microvms", + "get-microvm", + "--microvm-identifier", + resource.identifier, + check=False, + ) + return not details or details.get("state") == "TERMINATED" + + context.wait_for(terminated, f"MicroVM {resource.identifier} termination") + + def _wait_for_listed_microvm(self, context: SmokeContext, resource: RunnerResource) -> None: + def listed() -> bool: + result = context.aws("lambda-microvms", "list-microvms", check=False) or {} + return any( + item.get("microvmId") == resource.identifier + and item.get("state") in ("PENDING", "RUNNING", "SUSPENDING", "SUSPENDED") + for item in result.get("items", []) + ) + + context.wait_for(listed, f"MicroVM {resource.identifier} to appear in ListMicrovms") + + def scale_down( + self, + context: SmokeContext, + resource: RunnerResource, + runner_id: int, + marker: str, + active_runners: list[tuple[int, RunnerResource]], + ) -> None: + self._wait_for_listed_microvm(context, resource) + context.configure_runner_fixtures( + self.slug, + [(active_runner_id, active_resource.identifier) for active_runner_id, active_resource in active_runners], + runner_id, + ) + context.clear_requests() + context.invoke( + "multi-runner-webhook-microvm-scale-down", + {"smokeMarker": marker, "type": "microvm"}, + "MicroVM scale-down Lambda invoked", + ) + context.wait_for_log("/aws/lambda/multi-runner-webhook-microvm-scale-down", marker, "MicroVM scale-down Lambda started") + context.scale_down_routes(runner_id, "/aws/lambda/multi-runner-webhook-microvm-scale-down") + context.configure_runner_removed(runner_id) + context.assert_runner_removed(runner_id) + self._wait_for_termination(context, resource) + self.stop_microvm_hook(context) + + def stop_microvm_hook(self, context: SmokeContext) -> None: + status, body = context.http( + "POST", + f"{self.hook_url}/aws/lambda-microvms/runtime/v1/terminate", + {}, + ) + if status not in (0, 200, 404): + raise RuntimeError(f"MicroVM lifecycle hook termination failed with HTTP {status}: {body}") + self._hook_needs_restart = True + + +provider = MicrovmProvider() diff --git a/tests/ministack/smoke/provider.py b/tests/ministack/smoke/provider.py new file mode 100644 index 0000000000..c154e03883 --- /dev/null +++ b/tests/ministack/smoke/provider.py @@ -0,0 +1,57 @@ +"""Interface implemented by each compute provider smoke test.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + +from .common import SmokeContext + + +@dataclass(frozen=True) +class RunnerResource: + """Provider resource created by one smoke scenario.""" + + identifier: str + + +class SmokeProvider(Protocol): + slug: str + display_name: str + + def configure(self, context: SmokeContext) -> None: + """Add provider-specific MockServer expectations.""" + + def event(self, context: SmokeContext, job_id: int, dynamic: bool) -> dict[str, Any]: + """Build a workflow_job event that selects this provider.""" + + def verify_scale_up_routes(self, context: SmokeContext, job_id: int) -> None: + """Verify provider-specific GitHub API calls made during scale-up.""" + + def verify_pool_routes(self, context: SmokeContext) -> None: + """Verify provider-specific GitHub API calls made during pool scale-up.""" + + def wait_for_scale_up(self, context: SmokeContext, source: str) -> RunnerResource: + """Find the resource created by a scale-up Lambda.""" + + def assert_scale_up(self, context: SmokeContext, resource: RunnerResource, dynamic: bool) -> None: + """Check provider-specific scale-up state and ownership.""" + + def start_scale_up_runner(self, context: SmokeContext, resource: RunnerResource) -> bool: + """Start the provider-specific runner lifecycle after scale-up.""" + + def wait_for_pool(self, context: SmokeContext, source: str) -> RunnerResource: + """Find the resource created by a pool Lambda.""" + + def assert_pool(self, context: SmokeContext, resource: RunnerResource) -> None: + """Check provider-specific pool state and ownership.""" + + def scale_down( + self, + context: SmokeContext, + resource: RunnerResource, + runner_id: int, + marker: str, + active_runners: list[tuple[int, RunnerResource]], + ) -> None: + """Run provider-specific scale-down checks for one resource."""