From c40c095d26631200819e3c397d0e7b8aa53824c3 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 15:19:45 -0400 Subject: [PATCH 1/9] docx: move file around for consolidated examples section NOTE: need to update redirects --- base.yml | 77 +++++++++++------- docs/docs/how-to/index.md | 8 -- .../container-images}/build-push.md | 0 .../container-images}/build-use.md | 0 docs/examples/container-images/index.md | 0 docs/examples/credentials/idtoken-aws.md | 0 docs/examples/credentials/idtoken-azure.md | 0 docs/examples/credentials/idtoken-vault.md | 0 docs/examples/credentials/index.md | 0 .../{ => fundamentals}/hello-world.md | 0 docs/examples/fundamentals/index.md | 0 .../{ => fundamentals}/inputs-outputs.md | 0 .../{ => fundamentals}/set-pipeline.md | 0 .../assets/ascii-banner-grouped-overview.png | Bin .../assets/ascii-banner-grouped.png | Bin .../assets/ascii-banner-separate.png | Bin .../git-workflows}/basic.md | 0 .../{ => git-workflows}/git-triggered.md | 0 docs/examples/git-workflows/index.md | 0 .../git-workflows}/monorepo.md | 0 .../git-workflows}/multi-branch.md | 0 .../{ => language-pipelines}/golang-lib.md | 0 docs/examples/language-pipelines/index.md | 0 .../{ => language-pipelines}/java-app.md | 0 .../{ => language-pipelines}/nodejs-app.md | 0 .../{ => language-pipelines}/php-app.md | 0 .../{ => language-pipelines}/rails-app.md | 0 .../assets/gated-pipelines-01.png | Bin .../assets/gated-pipelines-02.png | Bin .../assets/gated-pipelines-03.png | Bin .../assets/task-inputs-outputs-01.gif | Bin .../assets/task-inputs-outputs-02a.gif | Bin .../assets/task-inputs-outputs-02b.gif | Bin .../assets/task-inputs-outputs-03.gif | Bin .../assets/task-inputs-outputs-04.gif | Bin .../assets/task-inputs-outputs-05.gif | Bin .../assets/task-inputs-outputs-06.gif | Bin .../assets/time-triggered-pipelines-01.png | Bin .../assets/time-triggered-pipelines-02.png | Bin .../pipeline-patterns}/common-pipeline.md | 0 .../pipeline-patterns}/gated-pipelines.md | 0 docs/examples/pipeline-patterns/index.md | 0 .../managing-pipeline-configs.md | 0 .../pipeline-patterns}/manual-approval.md | 0 .../manually-triggered.md | 0 .../{ => pipeline-patterns}/pipeline-vars.md | 0 .../{ => pipeline-patterns}/serial-job.md | 0 .../pipeline-patterns}/task-inputs-outputs.md | 0 .../time-triggered-pipelines.md | 0 .../{ => pipeline-patterns}/time-triggered.md | 0 docs/examples/task-composition/index.md | 0 .../job-and-task-hooks.md | 0 docs/examples/templating/index.md | 0 .../templating/ytt-multi-file-pipeline.md | 0 .../templating/ytt-single-pipeline.md | 0 docs/libs/examples | 2 +- 56 files changed, 48 insertions(+), 39 deletions(-) delete mode 100644 docs/docs/how-to/index.md rename docs/{docs/how-to/container-image-guides => examples/container-images}/build-push.md (100%) rename docs/{docs/how-to/container-image-guides => examples/container-images}/build-use.md (100%) create mode 100644 docs/examples/container-images/index.md create mode 100644 docs/examples/credentials/idtoken-aws.md create mode 100644 docs/examples/credentials/idtoken-azure.md create mode 100644 docs/examples/credentials/idtoken-vault.md create mode 100644 docs/examples/credentials/index.md rename docs/examples/{ => fundamentals}/hello-world.md (100%) create mode 100644 docs/examples/fundamentals/index.md rename docs/examples/{ => fundamentals}/inputs-outputs.md (100%) rename docs/examples/{ => fundamentals}/set-pipeline.md (100%) rename docs/{docs/how-to/git-guides => examples/git-workflows}/assets/ascii-banner-grouped-overview.png (100%) rename docs/{docs/how-to/git-guides => examples/git-workflows}/assets/ascii-banner-grouped.png (100%) rename docs/{docs/how-to/git-guides => examples/git-workflows}/assets/ascii-banner-separate.png (100%) rename docs/{docs/how-to/git-guides => examples/git-workflows}/basic.md (100%) rename docs/examples/{ => git-workflows}/git-triggered.md (100%) create mode 100644 docs/examples/git-workflows/index.md rename docs/{docs/how-to/git-guides => examples/git-workflows}/monorepo.md (100%) rename docs/{docs/how-to/git-guides => examples/git-workflows}/multi-branch.md (100%) rename docs/examples/{ => language-pipelines}/golang-lib.md (100%) create mode 100644 docs/examples/language-pipelines/index.md rename docs/examples/{ => language-pipelines}/java-app.md (100%) rename docs/examples/{ => language-pipelines}/nodejs-app.md (100%) rename docs/examples/{ => language-pipelines}/php-app.md (100%) rename docs/examples/{ => language-pipelines}/rails-app.md (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/gated-pipelines-01.png (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/gated-pipelines-02.png (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/gated-pipelines-03.png (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-01.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-02a.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-02b.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-03.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-04.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-05.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/task-inputs-outputs-06.gif (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/time-triggered-pipelines-01.png (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/assets/time-triggered-pipelines-02.png (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/common-pipeline.md (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/gated-pipelines.md (100%) create mode 100644 docs/examples/pipeline-patterns/index.md rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/managing-pipeline-configs.md (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/manual-approval.md (100%) rename docs/examples/{ => pipeline-patterns}/manually-triggered.md (100%) rename docs/examples/{ => pipeline-patterns}/pipeline-vars.md (100%) rename docs/examples/{ => pipeline-patterns}/serial-job.md (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/task-inputs-outputs.md (100%) rename docs/{docs/how-to/pipeline-guides => examples/pipeline-patterns}/time-triggered-pipelines.md (100%) rename docs/examples/{ => pipeline-patterns}/time-triggered.md (100%) create mode 100644 docs/examples/task-composition/index.md rename docs/examples/{ => task-composition}/job-and-task-hooks.md (100%) create mode 100644 docs/examples/templating/index.md create mode 100644 docs/examples/templating/ytt-multi-file-pipeline.md create mode 100644 docs/examples/templating/ytt-single-pipeline.md diff --git a/base.yml b/base.yml index 99d6fea03..2c7a8b1e4 100644 --- a/base.yml +++ b/base.yml @@ -201,22 +201,6 @@ nav: - docs/steps/modifier-and-hooks/ensure.md - docs/tasks.md - docs/builds.md - - How-To Guides: - - docs/how-to/index.md - - Pipelines: - - docs/how-to/pipeline-guides/managing-pipeline-configs.md - - docs/how-to/pipeline-guides/common-pipeline.md - - docs/how-to/pipeline-guides/task-inputs-outputs.md - - docs/how-to/pipeline-guides/gated-pipelines.md - - docs/how-to/pipeline-guides/time-triggered-pipelines.md - - docs/how-to/pipeline-guides/manual-approval.md - - Git: - - docs/how-to/git-guides/basic.md - - docs/how-to/git-guides/multi-branch.md - - docs/how-to/git-guides/monorepo.md - - Container Images: - - docs/how-to/container-image-guides/build-push.md - - docs/how-to/container-image-guides/build-use.md - Operation: - docs/operation/index.md - docs/operation/metrics.md @@ -250,20 +234,53 @@ nav: - docs/internals/garbage-collector.md - Examples: - examples/index.md - - examples/hello-world.md - - examples/serial-job.md - - examples/pipeline-vars.md - - examples/set-pipeline.md - - examples/inputs-outputs.md - - examples/time-triggered.md - - examples/git-triggered.md - - examples/manually-triggered.md - - examples/job-and-task-hooks.md - - examples/golang-lib.md - - examples/rails-app.md - - examples/java-app.md - - examples/nodejs-app.md - - examples/php-app.md + - Fundamentals: + - examples/fundamentals/index.md + - examples/fundamentals/hello-world.md + - examples/fundamentals/inputs-outputs.md + - examples/fundamentals/set-pipeline.md + - Pipeline Patterns: + - examples/pipeline-patterns/index.md + - examples/pipeline-patterns/time-triggered.md + - examples/pipeline-patterns/serial-job.md + - examples/pipeline-patterns/pipeline-vars.md + - examples/pipeline-patterns/manually-triggered.md + - examples/pipeline-patterns/managing-pipeline-configs.md + - examples/pipeline-patterns/common-pipeline.md + - examples/pipeline-patterns/task-inputs-outputs.md + - examples/pipeline-patterns/gated-pipelines.md + - examples/pipeline-patterns/time-triggered-pipelines.md + - examples/pipeline-patterns/manual-approval.md + - Git Workflows: + - examples/git-workflows/index.md + - examples/git-workflows/git-triggered.md + - examples/git-workflows/basic.md + - examples/git-workflows/multi-branch.md + - examples/git-workflows/monorepo.md + - Container Images: + - examples/container-images/index.md + - examples/container-images/build-push.md + - examples/container-images/build-use.md + - Task Composition: + - examples/task-composition/index.md + - examples/task-composition/job-and-task-hooks.md + - Language Pipelines: + - examples/language-pipelines/index.md + - examples/language-pipelines/golang-lib.md + - examples/language-pipelines/rails-app.md + - examples/language-pipelines/java-app.md + - examples/language-pipelines/nodejs-app.md + - examples/language-pipelines/php-app.md + - Credentials: + - examples/credentials/index.md + - examples/credentials/idtoken-vault.md + - examples/credentials/idtoken-aws.md + - examples/credentials/idtoken-azure.md + - Templating: + - examples/templating/index.md + - examples/templating/ytt-single-pipeline.md + - examples/templating/ytt-multi-file-pipeline.md + - project/index.md - ecosystem/index.md - support/index.md diff --git a/docs/docs/how-to/index.md b/docs/docs/how-to/index.md deleted file mode 100644 index e4e84a653..000000000 --- a/docs/docs/how-to/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: How-To Guides ---- - -The following pages are guides that show how to accomplish certain workflows within Concourse. Most of the guides will -use specific images, but you are in no way limited to or forced to use these images to accomplish the same task. There -are many ways to accomplish the same thing in Concourse, so don't let these guides limit you in what you think is -possible with Concourse. \ No newline at end of file diff --git a/docs/docs/how-to/container-image-guides/build-push.md b/docs/examples/container-images/build-push.md similarity index 100% rename from docs/docs/how-to/container-image-guides/build-push.md rename to docs/examples/container-images/build-push.md diff --git a/docs/docs/how-to/container-image-guides/build-use.md b/docs/examples/container-images/build-use.md similarity index 100% rename from docs/docs/how-to/container-image-guides/build-use.md rename to docs/examples/container-images/build-use.md diff --git a/docs/examples/container-images/index.md b/docs/examples/container-images/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/credentials/idtoken-aws.md b/docs/examples/credentials/idtoken-aws.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/credentials/idtoken-azure.md b/docs/examples/credentials/idtoken-azure.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/credentials/idtoken-vault.md b/docs/examples/credentials/idtoken-vault.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/credentials/index.md b/docs/examples/credentials/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/hello-world.md b/docs/examples/fundamentals/hello-world.md similarity index 100% rename from docs/examples/hello-world.md rename to docs/examples/fundamentals/hello-world.md diff --git a/docs/examples/fundamentals/index.md b/docs/examples/fundamentals/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/inputs-outputs.md b/docs/examples/fundamentals/inputs-outputs.md similarity index 100% rename from docs/examples/inputs-outputs.md rename to docs/examples/fundamentals/inputs-outputs.md diff --git a/docs/examples/set-pipeline.md b/docs/examples/fundamentals/set-pipeline.md similarity index 100% rename from docs/examples/set-pipeline.md rename to docs/examples/fundamentals/set-pipeline.md diff --git a/docs/docs/how-to/git-guides/assets/ascii-banner-grouped-overview.png b/docs/examples/git-workflows/assets/ascii-banner-grouped-overview.png similarity index 100% rename from docs/docs/how-to/git-guides/assets/ascii-banner-grouped-overview.png rename to docs/examples/git-workflows/assets/ascii-banner-grouped-overview.png diff --git a/docs/docs/how-to/git-guides/assets/ascii-banner-grouped.png b/docs/examples/git-workflows/assets/ascii-banner-grouped.png similarity index 100% rename from docs/docs/how-to/git-guides/assets/ascii-banner-grouped.png rename to docs/examples/git-workflows/assets/ascii-banner-grouped.png diff --git a/docs/docs/how-to/git-guides/assets/ascii-banner-separate.png b/docs/examples/git-workflows/assets/ascii-banner-separate.png similarity index 100% rename from docs/docs/how-to/git-guides/assets/ascii-banner-separate.png rename to docs/examples/git-workflows/assets/ascii-banner-separate.png diff --git a/docs/docs/how-to/git-guides/basic.md b/docs/examples/git-workflows/basic.md similarity index 100% rename from docs/docs/how-to/git-guides/basic.md rename to docs/examples/git-workflows/basic.md diff --git a/docs/examples/git-triggered.md b/docs/examples/git-workflows/git-triggered.md similarity index 100% rename from docs/examples/git-triggered.md rename to docs/examples/git-workflows/git-triggered.md diff --git a/docs/examples/git-workflows/index.md b/docs/examples/git-workflows/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/docs/how-to/git-guides/monorepo.md b/docs/examples/git-workflows/monorepo.md similarity index 100% rename from docs/docs/how-to/git-guides/monorepo.md rename to docs/examples/git-workflows/monorepo.md diff --git a/docs/docs/how-to/git-guides/multi-branch.md b/docs/examples/git-workflows/multi-branch.md similarity index 100% rename from docs/docs/how-to/git-guides/multi-branch.md rename to docs/examples/git-workflows/multi-branch.md diff --git a/docs/examples/golang-lib.md b/docs/examples/language-pipelines/golang-lib.md similarity index 100% rename from docs/examples/golang-lib.md rename to docs/examples/language-pipelines/golang-lib.md diff --git a/docs/examples/language-pipelines/index.md b/docs/examples/language-pipelines/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/java-app.md b/docs/examples/language-pipelines/java-app.md similarity index 100% rename from docs/examples/java-app.md rename to docs/examples/language-pipelines/java-app.md diff --git a/docs/examples/nodejs-app.md b/docs/examples/language-pipelines/nodejs-app.md similarity index 100% rename from docs/examples/nodejs-app.md rename to docs/examples/language-pipelines/nodejs-app.md diff --git a/docs/examples/php-app.md b/docs/examples/language-pipelines/php-app.md similarity index 100% rename from docs/examples/php-app.md rename to docs/examples/language-pipelines/php-app.md diff --git a/docs/examples/rails-app.md b/docs/examples/language-pipelines/rails-app.md similarity index 100% rename from docs/examples/rails-app.md rename to docs/examples/language-pipelines/rails-app.md diff --git a/docs/docs/how-to/pipeline-guides/assets/gated-pipelines-01.png b/docs/examples/pipeline-patterns/assets/gated-pipelines-01.png similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/gated-pipelines-01.png rename to docs/examples/pipeline-patterns/assets/gated-pipelines-01.png diff --git a/docs/docs/how-to/pipeline-guides/assets/gated-pipelines-02.png b/docs/examples/pipeline-patterns/assets/gated-pipelines-02.png similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/gated-pipelines-02.png rename to docs/examples/pipeline-patterns/assets/gated-pipelines-02.png diff --git a/docs/docs/how-to/pipeline-guides/assets/gated-pipelines-03.png b/docs/examples/pipeline-patterns/assets/gated-pipelines-03.png similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/gated-pipelines-03.png rename to docs/examples/pipeline-patterns/assets/gated-pipelines-03.png diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-01.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-01.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-01.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-01.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-02a.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-02a.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-02a.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-02a.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-02b.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-02b.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-02b.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-02b.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-03.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-03.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-03.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-03.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-04.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-04.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-04.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-04.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-05.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-05.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-05.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-05.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-06.gif b/docs/examples/pipeline-patterns/assets/task-inputs-outputs-06.gif similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/task-inputs-outputs-06.gif rename to docs/examples/pipeline-patterns/assets/task-inputs-outputs-06.gif diff --git a/docs/docs/how-to/pipeline-guides/assets/time-triggered-pipelines-01.png b/docs/examples/pipeline-patterns/assets/time-triggered-pipelines-01.png similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/time-triggered-pipelines-01.png rename to docs/examples/pipeline-patterns/assets/time-triggered-pipelines-01.png diff --git a/docs/docs/how-to/pipeline-guides/assets/time-triggered-pipelines-02.png b/docs/examples/pipeline-patterns/assets/time-triggered-pipelines-02.png similarity index 100% rename from docs/docs/how-to/pipeline-guides/assets/time-triggered-pipelines-02.png rename to docs/examples/pipeline-patterns/assets/time-triggered-pipelines-02.png diff --git a/docs/docs/how-to/pipeline-guides/common-pipeline.md b/docs/examples/pipeline-patterns/common-pipeline.md similarity index 100% rename from docs/docs/how-to/pipeline-guides/common-pipeline.md rename to docs/examples/pipeline-patterns/common-pipeline.md diff --git a/docs/docs/how-to/pipeline-guides/gated-pipelines.md b/docs/examples/pipeline-patterns/gated-pipelines.md similarity index 100% rename from docs/docs/how-to/pipeline-guides/gated-pipelines.md rename to docs/examples/pipeline-patterns/gated-pipelines.md diff --git a/docs/examples/pipeline-patterns/index.md b/docs/examples/pipeline-patterns/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/docs/how-to/pipeline-guides/managing-pipeline-configs.md b/docs/examples/pipeline-patterns/managing-pipeline-configs.md similarity index 100% rename from docs/docs/how-to/pipeline-guides/managing-pipeline-configs.md rename to docs/examples/pipeline-patterns/managing-pipeline-configs.md diff --git a/docs/docs/how-to/pipeline-guides/manual-approval.md b/docs/examples/pipeline-patterns/manual-approval.md similarity index 100% rename from docs/docs/how-to/pipeline-guides/manual-approval.md rename to docs/examples/pipeline-patterns/manual-approval.md diff --git a/docs/examples/manually-triggered.md b/docs/examples/pipeline-patterns/manually-triggered.md similarity index 100% rename from docs/examples/manually-triggered.md rename to docs/examples/pipeline-patterns/manually-triggered.md diff --git a/docs/examples/pipeline-vars.md b/docs/examples/pipeline-patterns/pipeline-vars.md similarity index 100% rename from docs/examples/pipeline-vars.md rename to docs/examples/pipeline-patterns/pipeline-vars.md diff --git a/docs/examples/serial-job.md b/docs/examples/pipeline-patterns/serial-job.md similarity index 100% rename from docs/examples/serial-job.md rename to docs/examples/pipeline-patterns/serial-job.md diff --git a/docs/docs/how-to/pipeline-guides/task-inputs-outputs.md b/docs/examples/pipeline-patterns/task-inputs-outputs.md similarity index 100% rename from docs/docs/how-to/pipeline-guides/task-inputs-outputs.md rename to docs/examples/pipeline-patterns/task-inputs-outputs.md diff --git a/docs/docs/how-to/pipeline-guides/time-triggered-pipelines.md b/docs/examples/pipeline-patterns/time-triggered-pipelines.md similarity index 100% rename from docs/docs/how-to/pipeline-guides/time-triggered-pipelines.md rename to docs/examples/pipeline-patterns/time-triggered-pipelines.md diff --git a/docs/examples/time-triggered.md b/docs/examples/pipeline-patterns/time-triggered.md similarity index 100% rename from docs/examples/time-triggered.md rename to docs/examples/pipeline-patterns/time-triggered.md diff --git a/docs/examples/task-composition/index.md b/docs/examples/task-composition/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/job-and-task-hooks.md b/docs/examples/task-composition/job-and-task-hooks.md similarity index 100% rename from docs/examples/job-and-task-hooks.md rename to docs/examples/task-composition/job-and-task-hooks.md diff --git a/docs/examples/templating/index.md b/docs/examples/templating/index.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/templating/ytt-multi-file-pipeline.md b/docs/examples/templating/ytt-multi-file-pipeline.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/examples/templating/ytt-single-pipeline.md b/docs/examples/templating/ytt-single-pipeline.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/libs/examples b/docs/libs/examples index a7698ada0..667033071 160000 --- a/docs/libs/examples +++ b/docs/libs/examples @@ -1 +1 @@ -Subproject commit a7698ada000b8cf641c9e103e5cbc0dedcd0c908 +Subproject commit 667033071b4f78f2a525fe156f6ddaf1d41102ea From 4555219bfa5ea28b3b9d2d5e49aa57c777913e1f Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 16:19:04 -0400 Subject: [PATCH 2/9] fix: update all internal docs references --- ...-to-build-and-publish-a-container-image.md | 2 +- docs/docs/getting-started/resources.md | 8 +- docs/docs/steps/modifier-and-hooks/across.md | 2 +- docs/docs/steps/task.md | 4 +- docs/examples/container-images/build-push.md | 16 +-- docs/examples/container-images/build-use.md | 4 +- docs/examples/container-images/index.md | 30 ++++++ docs/examples/credentials/index.md | 44 +++++++++ docs/examples/fundamentals/hello-world.md | 8 +- docs/examples/fundamentals/index.md | 51 ++++++++++ docs/examples/fundamentals/inputs-outputs.md | 12 +-- docs/examples/fundamentals/set-pipeline.md | 6 +- docs/examples/git-workflows/basic.md | 2 +- docs/examples/git-workflows/git-triggered.md | 8 +- docs/examples/git-workflows/index.md | 46 +++++++++ docs/examples/git-workflows/monorepo.md | 2 +- docs/examples/git-workflows/multi-branch.md | 20 ++-- docs/examples/index.md | 97 ++++++++++++++++--- .../examples/language-pipelines/golang-lib.md | 14 +-- docs/examples/language-pipelines/index.md | 53 ++++++++++ docs/examples/language-pipelines/java-app.md | 6 +- .../examples/language-pipelines/nodejs-app.md | 6 +- docs/examples/language-pipelines/php-app.md | 6 +- docs/examples/language-pipelines/rails-app.md | 6 +- .../pipeline-patterns/common-pipeline.md | 22 ++--- .../pipeline-patterns/gated-pipelines.md | 6 +- docs/examples/pipeline-patterns/index.md | 91 +++++++++++++++++ .../managing-pipeline-configs.md | 16 +-- .../pipeline-patterns/manual-approval.md | 4 +- .../pipeline-patterns/manually-triggered.md | 8 +- .../pipeline-patterns/pipeline-vars.md | 6 +- docs/examples/pipeline-patterns/serial-job.md | 10 +- .../pipeline-patterns/task-inputs-outputs.md | 20 ++-- .../time-triggered-pipelines.md | 2 +- .../pipeline-patterns/time-triggered.md | 8 +- docs/examples/task-composition/index.md | 23 +++++ .../task-composition/job-and-task-hooks.md | 22 ++--- docs/examples/templating/index.md | 49 ++++++++++ 38 files changed, 599 insertions(+), 141 deletions(-) diff --git a/docs/blog/posts/2020/06/2020-06-19-how-to-build-and-publish-a-container-image.md b/docs/blog/posts/2020/06/2020-06-19-how-to-build-and-publish-a-container-image.md index 07184e644..9d0505ad1 100644 --- a/docs/blog/posts/2020/06/2020-06-19-how-to-build-and-publish-a-container-image.md +++ b/docs/blog/posts/2020/06/2020-06-19-how-to-build-and-publish-a-container-image.md @@ -43,7 +43,7 @@ Now we can start building out our pipeline. Let's declare our [resources](../../ first. We will need one resource to pull in the repo where our Dockerfile is located, and a second resource pointing to where we want to push the built container image to. -_There are some [variables](../../../../examples/pipeline-vars.md#variables) in this file that we will +_There are some [variables](../../../../examples/pipeline-patterns/pipeline-vars.md#variables) in this file that we will fill out later._ ```yaml diff --git a/docs/docs/getting-started/resources.md b/docs/docs/getting-started/resources.md index 4c48455ed..cbc4de7b7 100644 --- a/docs/docs/getting-started/resources.md +++ b/docs/docs/getting-started/resources.md @@ -562,10 +562,10 @@ at [this page](../../resource-types-list.md). This brings us to the end of the tutorial. You should have a basic understanding about how to read Concourse pipelines and start creating your own. Here are some other parts of the site to help you take off with Concourse: -* [How-To Guides](../how-to/index.md) - Contains practical guides - for [working with pipelines](../how-to/pipeline-guides/common-pipeline.md) and examples of common pipeline workflows, - such as [git](../how-to/git-guides/basic.md) - and [container](../how-to/container-image-guides/build-push.md) workflows. +* [How-To Guides](../../examples/index.md) - Contains practical guides + for [working with pipelines](../../examples/pipeline-patterns/common-pipeline.md) and examples of common pipeline workflows, + such as [git](../../examples/git-workflows/basic.md) + and [container](../../examples/container-images/build-push.md) workflows. * Check out all the reference documentation: * [Jobs](../jobs.md) * [Tasks](../tasks.md) diff --git a/docs/docs/steps/modifier-and-hooks/across.md b/docs/docs/steps/modifier-and-hooks/across.md index da8b29117..fde140045 100644 --- a/docs/docs/steps/modifier-and-hooks/across.md +++ b/docs/docs/steps/modifier-and-hooks/across.md @@ -303,7 +303,7 @@ See [Across Step & Dynamic Vars](../../vars.md#across-step-dynamic-vars) for det When a new branch is added, a new pipeline will be created. When a branch is deleted, the pipeline will be automatically archived as described in the [`set_pipeline` step](../set-pipeline.md). - For a more complete example, refer to [Multi-Branch Workflows](../../how-to/git-guides/multi-branch.md). + For a more complete example, refer to [Multi-Branch Workflows](../../../examples/git-workflows/multi-branch.md). ## Limitations diff --git a/docs/docs/steps/task.md b/docs/docs/steps/task.md index 926c9aa79..732faf3e5 100644 --- a/docs/docs/steps/task.md +++ b/docs/docs/steps/task.md @@ -143,7 +143,7 @@ the result of a task. This can be used to explicitly keep track of dependent images. You could also modify it to build and push the image in one job and use it in later jobs. See [Building and Pushing an - Image](../how-to/container-image-guides/build-push.md). + Image](../../examples/container-images/build-push.md). ```yaml resources: @@ -169,7 +169,7 @@ the result of a task. ??? example "Building and using an image" - [Building an Image and Using it in a Task](../how-to/container-image-guides/build-use.md) + [Building an Image and Using it in a Task](../../examples/container-images/build-use.md) ??? info "`privileged`: [`boolean`](../config-basics.md#boolean-schema)" diff --git a/docs/examples/container-images/build-push.md b/docs/examples/container-images/build-push.md index 48d344f11..7ce5c5d48 100644 --- a/docs/examples/container-images/build-push.md +++ b/docs/examples/container-images/build-push.md @@ -32,11 +32,11 @@ The `stanger` text file: ## Defining Pipeline Resources -Now we can start building out our pipeline. Let's declare our [Resources](../../resources/index.md) first. We will need +Now we can start building out our pipeline. Let's declare our [Resources](../../docs/resources/index.md) first. We will need one resource to pull in the repo where our Dockerfile is located, and a second resource pointing to where we want to push the built container image to. -There are some [Variables](../../../examples/pipeline-vars.md#variables) in this file that we will fill out when setting +There are some [Variables](../pipeline-patterns/pipeline-vars.md#variables) in this file that we will fill out when setting the pipeline. ```yaml linenums="1" title="build-push.yml" @@ -45,7 +45,7 @@ the pipeline. ## Create the Job -Next we will create a [job](../../jobs.md) that will build and push our container image. +Next we will create a [job](../../docs/jobs.md) that will build and push our container image. To build the job we will need to pull in the repo where the `Dockerfile` is. @@ -62,7 +62,7 @@ resources: ... # omitting resource section from above The second step in our job will build the container image. To build the container image we are going to use the [oci-build-task](https://github.com/concourse/oci-build-task). The -oci-build-task is a container image that is meant to be used in a Concourse [task](../../tasks.md) to build other +oci-build-task is a container image that is meant to be used in a Concourse [task](../../docs/tasks.md) to build other container images. Check out the [`README.md`](https://github.com/concourse/oci-build-task/blob/master/README.md) in the repo for more details on how to configure and use the oci-build-task in more complex build scenarios. @@ -75,8 +75,8 @@ resources: ... # omitting resource section from above [//]: # (@formatter:on) Next we will add [concourse-examples](https://github.com/concourse/examples) as an [ -`input`](../../tasks.md#task-config-schema) to the build task to ensure the artifact from the [ -`get` step](../../steps/get.md) (where our `Dockerfile` is fetched) is mounted in our `build-image` step. +`input`](../../docs/tasks.md#task-config-schema) to the build task to ensure the artifact from the [ +`get` step](../../docs/steps/get.md) (where our `Dockerfile` is fetched) is mounted in our `build-image` step. [//]: # (@formatter:off) ```yaml linenums="1" title="build-push.yml" @@ -115,11 +115,11 @@ resources: ... # omitting resource section from above ## Publish the Container Image -To push the container image add a [`put` step](../../steps/put.md) to our job plan and tell the registry-image resource +To push the container image add a [`put` step](../../docs/steps/put.md) to our job plan and tell the registry-image resource where the tarball of the container image is. The `put` step will push the container image using the information defined previously in the -resource's [source](../../resources/index.md#resource-schema). +resource's [source](../../docs/resources/index.md#resource-schema). [//]: # (@formatter:off) ```yaml linenums="1" title="build-push.yml" diff --git a/docs/examples/container-images/build-use.md b/docs/examples/container-images/build-use.md index bd3583c3f..604682e6d 100644 --- a/docs/examples/container-images/build-use.md +++ b/docs/examples/container-images/build-use.md @@ -2,7 +2,7 @@ title: Building an Image and Using it in a Task --- -This guide will show you how to build and use an image within one [job](../../jobs.md) without pushing the image to an +This guide will show you how to build and use an image within one [job](../../docs/jobs.md) without pushing the image to an external image registry like Docker Hub. ## Build The Image @@ -28,7 +28,7 @@ The above pipeline will build a container image and also output it in Concourse' ## Use the Image Next we want to add a second task to this job which will use the image generated from the first task as its container -image. To use the image from the previous step add the top-level `image` key to the [`task` step](../../steps/task.md). +image. To use the image from the previous step add the top-level `image` key to the [`task` step](../../docs/steps/task.md). [//]: # (@formatter:off) ```yaml linenums="1" title="build-push.yml" diff --git a/docs/examples/container-images/index.md b/docs/examples/container-images/index.md index e69de29bb..3dc7c3a61 100644 --- a/docs/examples/container-images/index.md +++ b/docs/examples/container-images/index.md @@ -0,0 +1,30 @@ +--- +title: Container Images +hide: + - toc +--- + +Guides for building container images as part of a pipeline and using +them in later steps, instead of relying only on pre-built images. + +
+ +- __Building and Pushing an Image__ + + --- + + Build an image from a Dockerfile in your repo and push it to a + registry. + + [:octicons-arrow-right-24: View guide](build-push.md) + +- __Building an Image and Using it in a Task__ + + --- + + Build an image in one job and use it as the `image_resource` for a + task in the same pipeline. + + [:octicons-arrow-right-24: View guide](build-use.md) + +
\ No newline at end of file diff --git a/docs/examples/credentials/index.md b/docs/examples/credentials/index.md index e69de29bb..bb2e52806 100644 --- a/docs/examples/credentials/index.md +++ b/docs/examples/credentials/index.md @@ -0,0 +1,44 @@ +--- +title: Credentials & Identity Federation +hide: + - toc +--- + +Guides for authenticating pipelines to external services using +Concourse-issued JWTs (the [`idtoken` var source](../../docs/operation/creds/id-token.md)) +instead of storing long-lived secrets. For general credential management — +Vault, AWS Secrets Manager, and other cluster-wide credential managers — see +[Credential Management](../../docs/operation/creds/index.md) in the Docs. + +
+ +- :material-shield-key-outline:{ .lg .middle } __Authenticating to Vault via IDToken__ + + --- + + Let a pipeline log in to Vault directly, beyond what Concourse's + native Vault integration offers. + + [:octicons-arrow-right-24: View guide](idtoken-vault.md) + +- :material-aws:{ .lg .middle } __Authenticating to AWS via IDToken__ + + --- + + Assume an IAM role from a pipeline without managing IAM users or + long-lived access keys. + + [:octicons-arrow-right-24: View guide](idtoken-aws.md) + +- :material-microsoft-azure:{ .lg .middle } __Authenticating to Azure via IDToken__ + + --- + + Log in to Azure from a pipeline using a federated credential. + + [:octicons-arrow-right-24: View guide](idtoken-azure.md) + +
+ +For details on JWT claims, key rotation, and the `subject_scope` setting, see the +[IDToken credential manager](../../docs/operation/creds/id-token.md) reference page. \ No newline at end of file diff --git a/docs/examples/fundamentals/hello-world.md b/docs/examples/fundamentals/hello-world.md index 6c62511d6..2b3046e12 100644 --- a/docs/examples/fundamentals/hello-world.md +++ b/docs/examples/fundamentals/hello-world.md @@ -17,7 +17,7 @@ A single job is the simplest form of pipeline. While this is less of an example pipeline, this is a simple introduction to a critical primitive to form pipelines. -Also, due to the fact that there are minimal external factors ([Resources](../docs/resources/index.md)) for +Also, due to the fact that there are minimal external factors ([Resources](../../docs/resources/index.md)) for the system to check and resolve, this is often used to test overall system health. ## Pipeline Configuration @@ -28,6 +28,6 @@ the system to check and resolve, this is often used to test overall system healt ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/fundamentals/index.md b/docs/examples/fundamentals/index.md index e69de29bb..a5af299ce 100644 --- a/docs/examples/fundamentals/index.md +++ b/docs/examples/fundamentals/index.md @@ -0,0 +1,51 @@ +--- +title: Fundamentals +hide: + - toc +--- + +Bare pipeline files for the basics — copy one, deploy it with +[`fly set-pipeline`](../../docs/pipelines/setting-pipelines.md#fly-set-pipeline), and +poke at it. No walkthrough attached. + +New to Concourse? + +The [Getting Started tutorial](../../docs/getting-started/index.md) in the Docs covers +the same ground with full explanations and a guided setup — start there instead if +you haven't run a Concourse pipeline before. + +
+ +- __Hello World Pipeline__ + + --- + + The smallest possible pipeline: one job, one task, one `echo`. + See also: the tutorial version in + [Getting Started](../../docs/getting-started/hello-world.md). + + [:octicons-arrow-right-24: View example](hello-world.md) + +- __Inputs and Outputs__ + + --- + + Pass files between tasks in the same job using inputs and outputs. + See also: the tutorial version in + [Getting Started](../../docs/getting-started/inputs-outputs.md). + + [:octicons-arrow-right-24: View example](inputs-outputs.md) + +- __Set Pipeline Example__ + + --- + + Use a `set_pipeline` step so a pipeline can configure other pipelines + (or itself). + + [:octicons-arrow-right-24: View example](set-pipeline.md) + +
+ +Once these feel familiar, move on to [Pipeline Patterns](../pipeline-patterns/index.md) +for more advanced shapes. \ No newline at end of file diff --git a/docs/examples/fundamentals/inputs-outputs.md b/docs/examples/fundamentals/inputs-outputs.md index fb31c2226..6b7b60d27 100644 --- a/docs/examples/fundamentals/inputs-outputs.md +++ b/docs/examples/fundamentals/inputs-outputs.md @@ -22,8 +22,8 @@ While you are free to create as many jobs as you'd like for your pipeline, you h inbetween them. These constructs give you the ability to design a pipeline that can process artifacts in many different ways -via [Tasks](../docs/tasks.md), and then store those processed artifacts externally -via [Resources](../docs/resources/index.md). +via [Tasks](../../docs/tasks.md), and then store those processed artifacts externally +via [Resources](../../docs/resources/index.md). ## Pipeline Configuration @@ -33,7 +33,7 @@ via [Resources](../docs/resources/index.md). ## References -* [`task-config.outputs`](../docs/tasks.md#output-schema) -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [`task-config.outputs`](../../docs/tasks.md#task-config-schema) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/fundamentals/set-pipeline.md b/docs/examples/fundamentals/set-pipeline.md index 6fd1f7e85..c2fcc5606 100644 --- a/docs/examples/fundamentals/set-pipeline.md +++ b/docs/examples/fundamentals/set-pipeline.md @@ -23,6 +23,6 @@ You can set a static set of pipelines from another pipeline on the same team. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [`set-pipeline` step](../docs/steps/set-pipeline.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [`set-pipeline` step](../../docs/steps/set-pipeline.md) diff --git a/docs/examples/git-workflows/basic.md b/docs/examples/git-workflows/basic.md index 0d116255a..20ce81644 100644 --- a/docs/examples/git-workflows/basic.md +++ b/docs/examples/git-workflows/basic.md @@ -11,7 +11,7 @@ in the tarball on the [GitHub release page](https://github.com/concourse/concourse/releases). It is not the only resource available for working with git-related resources. If you don't see your use-case on this page then there is probably another resource that you can -use. Check out the [Resource Types](../../../resource-types-list.md) page for +use. Check out the [Resource Types](../../resource-types-list.md) page for other Git-related resources. Check out the diff --git a/docs/examples/git-workflows/git-triggered.md b/docs/examples/git-workflows/git-triggered.md index ace60128c..21fe71b77 100644 --- a/docs/examples/git-workflows/git-triggered.md +++ b/docs/examples/git-workflows/git-triggered.md @@ -27,7 +27,7 @@ The [`git` resource](https://github.com/concourse/git-resource) can be used to t ## References -* [Resources](../docs/resources/index.md) -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Resources](../../docs/resources/index.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/git-workflows/index.md b/docs/examples/git-workflows/index.md index e69de29bb..f099a0707 100644 --- a/docs/examples/git-workflows/index.md +++ b/docs/examples/git-workflows/index.md @@ -0,0 +1,46 @@ +--- +title: Git Workflows +hide: + - toc +--- + +Guides for triggering pipelines off Git activity and structuring pipelines +around common repository layouts. + +
+ +- __Basic Git Operations__ + + --- + + Configure the `git` resource, check out a repo, and trigger on new + commits. + + [:octicons-arrow-right-24: View guide](basic.md) + +- __Multi-Branch Workflows__ + + --- + + Run different pipeline behavior depending on which branch changed. + + [:octicons-arrow-right-24: View guide](multi-branch.md) + +- __Monorepo Workflows__ + + --- + + Trigger only the jobs affected by changes to a specific subdirectory + in a repo with multiple projects. + + [:octicons-arrow-right-24: View guide](monorepo.md) + +- __Git-Triggered Job Example__ + + --- + + A minimal end-to-end pipeline that runs a job whenever a repo updates. + + [:octicons-arrow-right-24: View guide](git-triggered.md) + +
\ No newline at end of file diff --git a/docs/examples/git-workflows/monorepo.md b/docs/examples/git-workflows/monorepo.md index 36eabe124..9e353c757 100644 --- a/docs/examples/git-workflows/monorepo.md +++ b/docs/examples/git-workflows/monorepo.md @@ -11,7 +11,7 @@ in the tarball on the [GitHub release page](https://github.com/concourse/concourse/releases). It is not the only resource available for working with git-related resources. If you don't see your use-case on this page then there is probably another resource that you can -use. Check out the [Resource Types](../../../resource-types-list.md) page for +use. Check out the [Resource Types](../../resource-types-list.md) page for other Git-related resources. Check out the diff --git a/docs/examples/git-workflows/multi-branch.md b/docs/examples/git-workflows/multi-branch.md index af77e1be9..e3788f3c9 100644 --- a/docs/examples/git-workflows/multi-branch.md +++ b/docs/examples/git-workflows/multi-branch.md @@ -12,20 +12,20 @@ the feature branch. To model this in Concourse, you'll need to have a pipeline for each active feature branch. Manually setting (and eventually archiving) a pipeline for each feature branch would be quite a burden. For this type of workflow, Concourse has a few important tools to help you out: the -[`set_pipeline` step](../../steps/set-pipeline.md), [ -`across`](../../steps/modifier-and-hooks/across.md), and [instance -pipelines](../../pipelines/grouping-pipelines.md). +[`set_pipeline` step](../../docs/steps/set-pipeline.md), [ +`across`](../../docs/steps/modifier-and-hooks/across.md), and [instance +pipelines](../../docs/pipelines/grouping-pipelines.md). In this guide, we'll cover: -1. Writing a pipeline to [Test and Build](#test-build-deploy) a feature branch. -1. Automatically creating pipelines for each feature branch from a "parent" +1. Writing a pipeline to Test and Build a feature branch. +2. Automatically creating pipelines for each feature branch from a "parent" pipeline. We'll use the [git resource](https://github.com/concourse/git-resource/), [`set_pipeline` - step](../../steps/set-pipeline.md), and [`across` - step](../../steps/modifier-and-hooks/across.md) to create instance + step](../../docs/steps/set-pipeline.md), and [`across` + step](../../docs/steps/modifier-and-hooks/across.md) to create instance pipelines. -1. Cleaning up pipelines once feature branches are merged and deleted. +3. Cleaning up pipelines once feature branches are merged and deleted. We'll use [this example Go app](https://github.com/concourse/examples/tree/main/apps/golang) for testing @@ -40,7 +40,7 @@ single feature branch. We only need two resources in our pipeline: 1. A `git` resource that will track and fetch commits from our feature branch. The `((branch))` is how we'll tell the pipeline which feature branch to track. -1. A `registry-image` resource that will pull the [golang +2. A `registry-image` resource that will pull the [golang image](https://hub.docker.com/_/golang) to test and build our example app. We'll then have two jobs: @@ -49,7 +49,7 @@ We'll then have two jobs: ```yaml linenums="1" title="go-test.yml" --8<-- "libs/examples/tasks/go-test.yml" ``` -1. The second job will compile and run the binary using these two task files. +2. The second job will compile and run the binary using these two task files. The first runs `go build` and tars up the final binary into a `.tgz` file. Both the binary and the tgz are saved in the output directory `binary`. ```yaml linenums="1" title="go-build.yml" diff --git a/docs/examples/index.md b/docs/examples/index.md index 7de2e8fd8..6e29d587d 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -1,26 +1,97 @@ --- -title: Examples -search: - exclude: true +title: Examples & Guides hide: - toc --- -Setting up self-contained Concourse [pipelines](../docs/pipelines/index.md) is an excellent way to -experiment before exploring the more comprehensive documentation. +Practical, copy-pasteable pipelines and task-oriented guides for building with Concourse. +Every page here gives you something runnable — deploy it with +[`fly set-pipeline`](../docs/pipelines/setting-pipelines.md#fly-set-pipeline), then adapt it to your use case. +For full configuration reference, see the [Docs](../docs/index.md). -Each example presents a pipeline YAML snippet which can be copied to a local file and deployed to your instance via [ -`fly set-pipeline`](../docs/pipelines/setting-pipelines.md#fly-set-pipeline). From there you can experiment and -modify parts of the configuration to better understand how everything works. All configuration options are detailed in -the [Docs](../docs/index.md). +
-For a practical real-world example, -examine [Concourse's own pipeline](https://ci.concourse-ci.org/teams/main/pipelines/concourse) (and -its [configuration](https://github.com/concourse/ci/blob/master/pipelines/concourse.yml)): +- :material-flask-outline:{ .lg .middle } __Fundamentals__ + + --- + + Bare, copy-pasteable pipeline files for the basics — no walkthrough + attached. New to Concourse? Try the guided + [Getting Started tutorial](../docs/getting-started/index.md) instead. + + [:octicons-arrow-right-24: View guides](fundamentals/index.md) + +- :material-sitemap-outline:{ .lg .middle } __Pipeline Patterns__ + + --- + + Common shapes: gating on approval, scheduling jobs, passing vars, + and structuring configuration as pipelines grow. + + [:octicons-arrow-right-24: View guides](pipeline-patterns/index.md) + +- :material-source-branch:{ .lg .middle } __Git Workflows__ + + --- + + Triggering off commits, working across branches, and handling + monorepos with multiple independently-versioned projects. + + [:octicons-arrow-right-24: View guides](git-workflows/index.md) + +- :material-docker:{ .lg .middle } __Container Images__ + + --- + + Build images as part of a pipeline and use them in later steps. + + [:octicons-arrow-right-24: View guides](container-images/index.md) + +- :material-tools:{ .lg .middle } __Task Composition & Hooks__ + + --- + + React to success, failure, and errors, and share data between + tasks in the same job. + + [:octicons-arrow-right-24: View guides](task-composition/index.md) + +- :material-language-python:{ .lg .middle } __Language Pipelines__ + + --- + + Ready-to-adapt CI pipelines for Go, Ruby on Rails, Java, Node.js, + and PHP projects. + + [:octicons-arrow-right-24: View guides](language-pipelines/index.md) + +- :material-key-outline:{ .lg .middle } __Credentials & Identity Federation__ + + --- + + Authenticate pipelines to Vault, AWS, and Azure without storing + long-lived secrets. + + [:octicons-arrow-right-24: View guides](credentials/index.md) + +- :material-file-code-outline:{ .lg .middle } __Templating with ytt__ + + --- + + Cut down on repeated YAML by templating pipelines with ytt before + setting them. + + [:octicons-arrow-right-24: View guides](templating/index.md) + +
+ +For a practical, real-world example, see +[Concourse's own pipeline](https://ci.concourse-ci.org/teams/main/pipelines/concourse) +and its [configuration](https://github.com/concourse/ci/blob/master/pipelines/concourse.yml).
-
+ \ No newline at end of file diff --git a/docs/examples/language-pipelines/golang-lib.md b/docs/examples/language-pipelines/golang-lib.md index c8c7ec6b5..cd7c3f0b6 100644 --- a/docs/examples/language-pipelines/golang-lib.md +++ b/docs/examples/language-pipelines/golang-lib.md @@ -16,12 +16,12 @@ You can run the tests for a Golang library across any specified versions. This example shows how to have multiple versions of a language, environment, or dependency fetched and integrated in to -a [Pipeline](../docs/pipelines/index.md). +a [Pipeline](../../docs/pipelines/index.md). -For these Docker images, defining them as [Resources](../docs/resources/index.md) has two advantages for +For these Docker images, defining them as [Resources](../../docs/resources/index.md) has two advantages for this use case. First, this enables the pipeline to be triggered when there are new versions of those images available. -Second, referencing them in the task's [`task` step **`image`**](../docs/steps/task.md) param is helpful as it will -ensure consistency between the image versions fetched by the [Resource](../docs/resources/index.md) and the image +Second, referencing them in the task's [`task` step **`image`**](../../docs/steps/task.md) param is helpful as it will +ensure consistency between the image versions fetched by the [Resource](../../docs/resources/index.md) and the image version running in the job. ## Pipeline Configuration @@ -32,6 +32,6 @@ version running in the job. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/language-pipelines/index.md b/docs/examples/language-pipelines/index.md index e69de29bb..6fcb7f518 100644 --- a/docs/examples/language-pipelines/index.md +++ b/docs/examples/language-pipelines/index.md @@ -0,0 +1,53 @@ +--- +title: Language Pipelines +hide: + - toc +--- + +Ready-to-adapt CI pipelines for testing applications in common languages +and frameworks. Each one checks out a repo, installs dependencies, and +runs the test suite for that ecosystem. + +
+ +- :material-language-go:{ .lg .middle } __Golang Library Testing__ + + --- + + Test a Go library with `go test`. + + [:octicons-arrow-right-24: View example](golang-lib.md) + +- :material-language-ruby:{ .lg .middle } __Rails Application Testing__ + + --- + + Test a Rails app, including database setup. + + [:octicons-arrow-right-24: View example](rails-app.md) + +- :material-language-java:{ .lg .middle } __Java Application Testing__ + + --- + + Test a Java app with Maven or Gradle. + + [:octicons-arrow-right-24: View example](java-app.md) + +- :material-nodejs:{ .lg .middle } __Node.js Application Testing__ + + --- + + Test a Node.js app with npm. + + [:octicons-arrow-right-24: View example](nodejs-app.md) + +- :material-language-php:{ .lg .middle } __PHP Application Testing__ + + --- + + Test a PHP app with PHPUnit. + + [:octicons-arrow-right-24: View example](php-app.md) + +
\ No newline at end of file diff --git a/docs/examples/language-pipelines/java-app.md b/docs/examples/language-pipelines/java-app.md index ec3484cff..88d7406cc 100644 --- a/docs/examples/language-pipelines/java-app.md +++ b/docs/examples/language-pipelines/java-app.md @@ -23,6 +23,6 @@ You can run the tests for a Java application. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/language-pipelines/nodejs-app.md b/docs/examples/language-pipelines/nodejs-app.md index a2e33d36d..d658deaf9 100644 --- a/docs/examples/language-pipelines/nodejs-app.md +++ b/docs/examples/language-pipelines/nodejs-app.md @@ -23,6 +23,6 @@ You can run the tests for a Nodejs application. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/language-pipelines/php-app.md b/docs/examples/language-pipelines/php-app.md index ed60f340b..4de448976 100644 --- a/docs/examples/language-pipelines/php-app.md +++ b/docs/examples/language-pipelines/php-app.md @@ -23,6 +23,6 @@ You can run the tests for a PHP application. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/language-pipelines/rails-app.md b/docs/examples/language-pipelines/rails-app.md index d68ef4e44..5a3ab2077 100644 --- a/docs/examples/language-pipelines/rails-app.md +++ b/docs/examples/language-pipelines/rails-app.md @@ -23,6 +23,6 @@ You can run the tests for a Rails that requires a specific version of ruby and r ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/pipeline-patterns/common-pipeline.md b/docs/examples/pipeline-patterns/common-pipeline.md index 7188f5429..1cb9d38fd 100644 --- a/docs/examples/pipeline-patterns/common-pipeline.md +++ b/docs/examples/pipeline-patterns/common-pipeline.md @@ -7,7 +7,7 @@ they are simply common and may or may not work for you and your team. ## Parallelizing Get Steps in Jobs -All jobs usually have [`get` steps](../../steps/get.md) as their first set of steps. +All jobs usually have [`get` steps](../../docs/steps/get.md) as their first set of steps. ```yaml jobs: @@ -20,7 +20,7 @@ jobs: ``` To reduce the waiting time to the length of the longest running get step, put all `get` steps under an [ -`in_parallel` step](../../steps/in-parallel.md). +`in_parallel` step](../../docs/steps/in-parallel.md). ```yaml jobs: @@ -40,13 +40,13 @@ jobs: You can still manually specify `inputs` if `detect` doesn't find all inputs needed by your `put` step. -By default, [`put` step's](../../steps/put.md) have all artifacts from a job mounted in their resource container. This -can result in long initialization times for put steps. It's likely that a [`put` step](../../steps/put.md) only needs a +By default, [`put` step's](../../docs/steps/put.md) have all artifacts from a job mounted in their resource container. This +can result in long initialization times for put steps. It's likely that a [`put` step](../../docs/steps/put.md) only needs a subset of all available artifacts generated throughout the job. -There are two ways to specify which artifacts to send to a [`put` step](../../steps/put.md). You can specify `detect` as -the [`put` step `inputs`](../../steps/put.md), or you can pass in an exact list of all artifacts the [ -`put` step](../../steps/put.md) needs. +There are two ways to specify which artifacts to send to a [`put` step](../../docs/steps/put.md). You can specify `detect` as +the [`put` step `inputs`](../../docs/steps/put.md), or you can pass in an exact list of all artifacts the [ +`put` step](../../docs/steps/put.md) needs. Using `detect`: @@ -67,7 +67,7 @@ jobs: basket: apple-basket # matches the second get step ``` -Specifying the exact inputs needed for the [`put` step](../../steps/put.md): +Specifying the exact inputs needed for the [`put` step](../../docs/steps/put.md): ```yaml jobs: @@ -87,7 +87,7 @@ jobs: ## Putting Task Configs in Files A lot of the pipeline examples that you will find on this site and in resource repos will embed a [`task` step -`config`](../../steps/task.md) directly in the pipeline. This is a nice way of clearly seeing what inputs/outputs the +`config`](../../docs/steps/task.md) directly in the pipeline. This is a nice way of clearly seeing what inputs/outputs the task uses. Tasks are usually designed to be used in multiple places, maybe with slightly different configuration. To support this scenario, most users store task configs in files instead of embedding the config directly in the pipeline. @@ -126,8 +126,8 @@ jobs: ## `Get` Images for Tasks Instead of using Anonymous Image Resources It is easy to let Concourse fetch images for tasks right when they are needed by using the `task-config.image_resource` -field in a [task config](../../steps/task.md). It's the easy out-of-the-box solution. Another way is to pass the image -for a task as an input to the job by setting the [`task` step `image`](../../steps/task.md) field. This also allows you +field in a [task config](../../docs/steps/task.md). It's the easy out-of-the-box solution. Another way is to pass the image +for a task as an input to the job by setting the [`task` step `image`](../../docs/steps/task.md) field. This also allows you to track the version of the image being used by the task and also avoid getting rate-limited by configuring the resource with credentials. diff --git a/docs/examples/pipeline-patterns/gated-pipelines.md b/docs/examples/pipeline-patterns/gated-pipelines.md index b71866af8..9a6ff44db 100644 --- a/docs/examples/pipeline-patterns/gated-pipelines.md +++ b/docs/examples/pipeline-patterns/gated-pipelines.md @@ -11,12 +11,12 @@ trigger of such step. ## 1) - A Simple Gated Pipeline -By default, all [Jobs](../../jobs.md) only run when manually triggered. That means a user has to run [ -`fly trigger-job`](../../jobs.md#fly-trigger-job) or click the plus button in the web interface for a job to run. A job +By default, all [Jobs](../../docs/jobs.md) only run when manually triggered. That means a user has to run [ +`fly trigger-job`](../../docs/jobs.md#fly-trigger-job) or click the plus button in the web interface for a job to run. A job only runs automatically if one of its resources has the `trigger: true` parameter set. Therefore, in order to create a gated job in a pipeline you simply need to create a job that can only be manually -triggered. That means not setting `trigger: true` for any of the jobs' [`get` steps](../../steps/get.md). +triggered. That means not setting `trigger: true` for any of the jobs' [`get` steps](../../docs/steps/get.md). ```yaml jobs: diff --git a/docs/examples/pipeline-patterns/index.md b/docs/examples/pipeline-patterns/index.md index e69de29bb..8368e6e90 100644 --- a/docs/examples/pipeline-patterns/index.md +++ b/docs/examples/pipeline-patterns/index.md @@ -0,0 +1,91 @@ +--- +title: Pipeline Patterns +hide: + - toc +--- + +Guides for structuring pipelines as they grow past a single job: gating +deploys on approval, running jobs on a schedule, passing configuration +between pipelines, and keeping configs maintainable over time. + +
+ +- __Common Pipeline Practices__ + + --- + + Conventions worth adopting early, before a pipeline grows past a + handful of jobs. + + [:octicons-arrow-right-24: View guide](common-pipeline.md) + +- __Managing Pipeline Configurations__ + + --- + + Strategies for keeping pipeline YAML maintainable as it grows — + templating, includes, and file organization. + + [:octicons-arrow-right-24: View guide](managing-pipeline-configs.md) + +- __Task Input and Output Scenarios__ + + --- + + Worked scenarios beyond the basics: optional inputs, multiple + outputs, and passing artifacts across jobs. + + [:octicons-arrow-right-24: View guide](task-inputs-outputs.md) + +- __Gated Pipeline Patterns__ + + --- + + Hold a job until a condition is met — a passed version, a manual + check, or an external signal. + + [:octicons-arrow-right-24: View guide](gated-pipelines.md) + +- __Manual Approval Step__ + + --- + + Require a human to approve a job before it proceeds. + + [:octicons-arrow-right-24: View guide](manual-approval.md) + +- __Time-Triggered Pipelines__ + + --- + + Run a job on a schedule using the `time` resource. + + [:octicons-arrow-right-24: View guide](time-triggered.md) + +- __Serial Job Example__ + + --- + + Prevent overlapping runs of the same job with `serial: true`. + + [:octicons-arrow-right-24: View guide](serial-job.md) + +- __Pipeline `((vars))` Example__ + + --- + + Parameterize a pipeline so the same config can deploy to multiple + environments. + + [:octicons-arrow-right-24: View guide](pipeline-vars.md) + +- __Manually Triggered Job Example__ + + --- + + Disable automatic triggering so a job only runs when someone + explicitly asks for it. + + [:octicons-arrow-right-24: View guide](manually-triggered.md) + +
\ No newline at end of file diff --git a/docs/examples/pipeline-patterns/managing-pipeline-configs.md b/docs/examples/pipeline-patterns/managing-pipeline-configs.md index 4b9be2b42..4a7246291 100644 --- a/docs/examples/pipeline-patterns/managing-pipeline-configs.md +++ b/docs/examples/pipeline-patterns/managing-pipeline-configs.md @@ -29,9 +29,9 @@ Then store your pipeline YAML files in that directory. Again, there's no "specia store your pipeline YAML files. Do whatever makes sense to you! To automatically update your pipeline in Concourse with what's stored in your Git repository, use the [ -`set_pipeline` step](../../steps/set-pipeline.md) in a job. You can view an example of a pipeline updating -itself [in the examples section](../../../examples/set-pipeline.md). There are also examples on the [ -`set_pipeline` step](../../steps/set-pipeline.md) page. +`set_pipeline` step](../../docs/steps/set-pipeline.md) in a job. You can view an example of a pipeline updating +itself [in the examples section](../fundamentals/set-pipeline.md). There are also examples on the [ +`set_pipeline` step](../../docs/steps/set-pipeline.md) page. ## 2) In a Different Git Repository @@ -45,20 +45,20 @@ the [resource type repositories](https://github.com/concourse/?q=resource). !!! tip - The following is also described on the [`set_pipeline` step](../../steps/set-pipeline.md) page. + The following is also described on the [`set_pipeline` step](../../docs/steps/set-pipeline.md) page. If you are setting multiple pipelines, or multiple instances of the same pipeline, it can be helpful to manage them from one place. Concourse allows you to use the `set_pipeline` step to create other pipelines. -The [set_pipeline step](../../steps/set-pipeline.md) is not limited to updating the current pipeline. +The [set_pipeline step](../../docs/steps/set-pipeline.md) is not limited to updating the current pipeline. When you use one pipeline to create other pipelines, this creates a parent-child relationship that Concourse tracks. You -can see an example of this [here in `set-pipelines.yml`](../../../examples/set-pipeline.md). +can see an example of this [here in `set-pipelines.yml`](../fundamentals/set-pipeline.md). As long as the parent pipeline continues to set/update the child pipeline(s), the child pipeline(s) will remain active. If the parent pipeline stops updating the child pipeline(s) (e.g. you updated the parent pipeline to not set/update the child pipeline(s) anymore), Concourse -will [archive the pipeline](../../pipelines/managing-pipelines.md#fly-archive-pipeline). This pauses the child pipeline( +will [archive the pipeline](../../docs/pipelines/managing-pipelines.md#fly-archive-pipeline). This pauses the child pipeline( s) and hides them from the web UI. The child pipeline configuration is deleted, but its build logs are retained. If you want to fully delete a pipeline, use [ -`fly destroy-pipeline`](../../pipelines/managing-pipelines.md#fly-destroy-pipeline). \ No newline at end of file +`fly destroy-pipeline`](../../docs/pipelines/managing-pipelines.md#fly-destroy-pipeline). \ No newline at end of file diff --git a/docs/examples/pipeline-patterns/manual-approval.md b/docs/examples/pipeline-patterns/manual-approval.md index 5a7866e77..d0edbd20c 100644 --- a/docs/examples/pipeline-patterns/manual-approval.md +++ b/docs/examples/pipeline-patterns/manual-approval.md @@ -2,10 +2,10 @@ title: Manual Approval Step --- -This is an example of a [`task` step](../../tasks.md) you can add to your [Jobs](../../jobs.md) that requires a human to +This is an example of a [`task` step](../../docs/tasks.md) you can add to your [Jobs](../../docs/jobs.md) that requires a human to approve or reject the job from running. This is probably the most minimal version of a manual approval step you can have in Concourse that doesn't require pulling in a bunch of other tech into your stack. It's definitely not the best UX -since you need to use the [`fly` CLI](../../fly.md) to approve the step. +since you need to use the [`fly` CLI](../../docs/fly.md) to approve the step. Task configuration, `config.yml`: diff --git a/docs/examples/pipeline-patterns/manually-triggered.md b/docs/examples/pipeline-patterns/manually-triggered.md index c851ca506..e2b25a04d 100644 --- a/docs/examples/pipeline-patterns/manually-triggered.md +++ b/docs/examples/pipeline-patterns/manually-triggered.md @@ -23,7 +23,7 @@ A job can be triggered by a resource. After it's complete, the next job can run ## References -* [Resources](../docs/resources/index.md) -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Resources](../../docs/resources/index.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/pipeline-patterns/pipeline-vars.md b/docs/examples/pipeline-patterns/pipeline-vars.md index 4afd7cc92..a9363cdd8 100644 --- a/docs/examples/pipeline-patterns/pipeline-vars.md +++ b/docs/examples/pipeline-patterns/pipeline-vars.md @@ -31,6 +31,6 @@ You can use params in a pipelines configuration file. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/pipeline-patterns/serial-job.md b/docs/examples/pipeline-patterns/serial-job.md index dc7469bf9..0d6585e97 100644 --- a/docs/examples/pipeline-patterns/serial-job.md +++ b/docs/examples/pipeline-patterns/serial-job.md @@ -6,7 +6,7 @@ hide: - toc --- -Setting the [`job.serial`](../docs/jobs.md#job-schema) flag restricts a job to run one build +Setting the [`job.serial`](../../docs/jobs.md#job-schema) flag restricts a job to run one build at a time.
@@ -19,7 +19,7 @@ at a time. By default, jobs are run in parallel. For some use cases this might be ideal (ex. testing all incoming commits from a repository). For other use cases this might be less ideal (ex. deploying an application). -You can also set the [`job.max_in_flight`](../docs/jobs.md#job-schema) value to 1 to +You can also set the [`job.max_in_flight`](../../docs/jobs.md#job-schema) value to 1 to disable parallel job runs. ## Pipeline Configuration @@ -30,6 +30,6 @@ disable parallel job runs. ## References -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/pipeline-patterns/task-inputs-outputs.md b/docs/examples/pipeline-patterns/task-inputs-outputs.md index c26844eeb..e4c77fb4a 100644 --- a/docs/examples/pipeline-patterns/task-inputs-outputs.md +++ b/docs/examples/pipeline-patterns/task-inputs-outputs.md @@ -7,13 +7,13 @@ you through a few example pipelines to show you how inputs and outputs work with you should understand how inputs and outputs work within the context of a single job. To run the pipelines in the following examples yourself you can get your own Concourse running locally by following -the [Quick Start](../../getting-started/quick-start.md) guide. Then use [ -`fly set-pipeline`](../../pipelines/setting-pipelines.md#fly-set-pipeline) to see the pipelines in action. +the [Quick Start](../../docs/getting-started/quick-start.md) guide. Then use [ +`fly set-pipeline`](../../docs/pipelines/setting-pipelines.md#fly-set-pipeline) to see the pipelines in action. ## 1) - Passing Inputs Between Tasks -This pipeline will show us how to create outputs and pass outputs as inputs to the next [step](../../steps/index.md) in -a [job plan](../../jobs.md). +This pipeline will show us how to create outputs and pass outputs as inputs to the next [step](../../docs/steps/index.md) in +a [job plan](../../docs/jobs.md). This pipeline has two tasks. The first task outputs a file with the date. The second task reads and prints the contents of the file from the first task. @@ -80,8 +80,8 @@ fly -t tutorial trigger-job --job passing-artifacts/the-job --watch This scenario is to satisfy the curiosity cat inside all of us. Never do this in real life because you're definitely going to hurt yourself! -There are two [Jobs](../../jobs.md) in this pipeline. The first job, `writing-in-parallel`, has -two [Steps](../../steps/index.md); both steps will produce an artifact named `the-output` in parallel. If you run the +There are two [Jobs](../../docs/jobs.md) in this pipeline. The first job, `writing-in-parallel`, has +two [Steps](../../docs/steps/index.md); both steps will produce an artifact named `the-output` in parallel. If you run the `writing-to-the-same-output-in-parallel` job multiple times you'll see the file in `the-output` folder changes depending on which of the parallel tasks finished last. Here's a visualization of the first job. @@ -206,7 +206,7 @@ fly -t tutorial trigger-job --job parallel-artifacts/writing-to-the-same-output- ## 3) - Mapping the Names of Inputs and Outputs -Sometimes the names of inputs and outputs don't match between multiple [task configs](../../steps/task.md), or they do +Sometimes the names of inputs and outputs don't match between multiple [task configs](../../docs/steps/task.md), or they do match, and you don't want them overwriting each other, like in the previous example. That's when `input_mapping` and `output_mapping` become helpful. Both of these features rename the inputs/outputs in the task's config to some other name in the job plan. @@ -222,7 +222,7 @@ The third task reads and prints the contents of the file under another name, `ge in the job plan is mapped to `generic-input`. The fourth task tries to use the artifact named `the-output` as its input. This task fails to even start because there -was no artifact with the name `the-output` available in the [job plan](../../jobs.md#job-schema); it was remapped to +was no artifact with the name `the-output` available in the [job plan](../../docs/jobs.md#job-schema); it was remapped to `demo-disk`. Here's a visualization of the job. @@ -485,8 +485,8 @@ fly -t tutorial trigger-job --job multiple-artifacts/multiple-outputs --watch ## 6) - Get Steps Generate Artifacts -The majority of Concourse pipelines have at least one [resource](../../resources/index.md), which means they have at -least one [`get` step](../../steps/get.md). Using a [`get` step](../../steps/get.md) in a job makes an artifact with the +The majority of Concourse pipelines have at least one [resource](../../docs/resources/index.md), which means they have at +least one [`get` step](../../docs/steps/get.md). Using a [`get` step](../../docs/steps/get.md) in a job makes an artifact with the name of the get step available for later steps in the job plan to consume as inputs. Here's a visualization of the job. diff --git a/docs/examples/pipeline-patterns/time-triggered-pipelines.md b/docs/examples/pipeline-patterns/time-triggered-pipelines.md index 759a35aca..94baf1bc7 100644 --- a/docs/examples/pipeline-patterns/time-triggered-pipelines.md +++ b/docs/examples/pipeline-patterns/time-triggered-pipelines.md @@ -3,7 +3,7 @@ title: Time Triggered Pipeline Patterns --- The [time resource](https://github.com/concourse/time-resource/) produces a -new [version](../../getting-started/resources.md#versions) for the time interval that was declared in its definition in +new [version](../../docs/getting-started/resources.md#versions) for the time interval that was declared in its definition in the pipeline configuration file. The two most common usages are having the time resource trigger on an interval: diff --git a/docs/examples/pipeline-patterns/time-triggered.md b/docs/examples/pipeline-patterns/time-triggered.md index 3f6113a74..29be3f707 100644 --- a/docs/examples/pipeline-patterns/time-triggered.md +++ b/docs/examples/pipeline-patterns/time-triggered.md @@ -25,7 +25,7 @@ The [`time` resource](https://github.com/concourse/time-resource) can be used to ## References -* [Resources](../docs/resources/index.md) -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [Resources](../../docs/resources/index.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/task-composition/index.md b/docs/examples/task-composition/index.md index e69de29bb..51f8f2c00 100644 --- a/docs/examples/task-composition/index.md +++ b/docs/examples/task-composition/index.md @@ -0,0 +1,23 @@ +--- +title: Task Composition & Hooks +hide: + - toc +--- + +Guides for reacting to success, failure, and errors within a job, and for +sharing data between steps and tasks. + +
+ +- __Job & Task Hooks Example__ + + --- + + Attach `on_success`, `on_failure`, and `on_error` hooks to a job so + you can notify, clean up, or retry based on outcome. + + [:octicons-arrow-right-24: View guide](job-and-task-hooks.md) + +
+ +For the underlying step reference, see [Modifiers and Hooks](../../docs/steps/modifier-and-hooks/index.md). \ No newline at end of file diff --git a/docs/examples/task-composition/job-and-task-hooks.md b/docs/examples/task-composition/job-and-task-hooks.md index 348561bbc..89c554524 100644 --- a/docs/examples/task-composition/job-and-task-hooks.md +++ b/docs/examples/task-composition/job-and-task-hooks.md @@ -6,8 +6,8 @@ hide: - toc --- -Job hooks like [`job.on_success`](../docs/jobs.md#job-schema) and Step hooks like [ -`on_success`](../docs/steps/modifier-and-hooks/on-success.md) are available to perform actions based on +Job hooks like [`job.on_success`](../../docs/jobs.md#job-schema) and Step hooks like [ +`on_success`](../../docs/steps/modifier-and-hooks/on-success.md) are available to perform actions based on the success, failure, or abortion of a job.
@@ -25,12 +25,12 @@ the success, failure, or abortion of a job. ## References -* [`job.on_success`](../docs/jobs.md#job-schema) -* [`job.on_failure`](../docs/jobs.md#job-schema) -* [`job.on_abort`](../docs/jobs.md#job-schema) -* [`on_success`](../docs/steps/modifier-and-hooks/on-success.md) -* [`on_failure`](../docs/steps/modifier-and-hooks/on-failure.md) -* [`on_abort`](../docs/steps/modifier-and-hooks/on-abort.md) -* [Jobs](../docs/jobs.md) -* [Steps](../docs/steps/index.md) -* [Tasks](../docs/tasks.md) +* [`job.on_success`](../../docs/jobs.md#job-schema) +* [`job.on_failure`](../../docs/jobs.md#job-schema) +* [`job.on_abort`](../../docs/jobs.md#job-schema) +* [`on_success`](../../docs/steps/modifier-and-hooks/on-success.md) +* [`on_failure`](../../docs/steps/modifier-and-hooks/on-failure.md) +* [`on_abort`](../../docs/steps/modifier-and-hooks/on-abort.md) +* [Jobs](../../docs/jobs.md) +* [Steps](../../docs/steps/index.md) +* [Tasks](../../docs/tasks.md) diff --git a/docs/examples/templating/index.md b/docs/examples/templating/index.md index e69de29bb..f83c19dc1 100644 --- a/docs/examples/templating/index.md +++ b/docs/examples/templating/index.md @@ -0,0 +1,49 @@ +--- +title: Templating with ytt +hide: + - toc +--- + +Concourse deliberately has no built-in templating — see +[Config Basics](../../docs/config-basics.md) for why — so if a pipeline's YAML +is getting repetitive, the recommended path is to reach for an external +templating tool and render plain YAML before you set the pipeline. These +guides use [ytt](https://carvel.dev/ytt/), the Carvel project's YAML +templating tool, since it understands YAML structure natively rather than +treating it as text. + +Both guides follow the same shape: template files in, `ytt -f > out.yml` +to render, then `fly set-pipeline` (or a `set_pipeline` step) on the result. + +
+ +- __Templating a Single Pipeline__ + + --- + + Start here. Extract repeated values into a data values file and + render a single pipeline with ytt. + + [:octicons-arrow-right-24: View guide](ytt-single-pipeline.md) + +- __Templating Across Multiple Files__ + + --- + + Split a template across multiple files — useful once a pipeline's + template grows past what's comfortable in one file — and render + them together. + + [:octicons-arrow-right-24: View guide](ytt-multi-file-pipeline.md) + +
+ +Looking to automate the render-and-set step itself, rather than run `ytt` +locally? See how the Concourse project does it in +[`set-pipelines.yml`](https://github.com/concourse/examples/blob/main/pipelines/set-pipelines.yml), +which runs `ytt` inside a task and feeds the rendered output straight into +`set_pipeline` steps. + +For other approaches to keeping pipeline config maintainable — not +specific to ytt — see +[Managing Pipeline Configurations](../pipeline-patterns/managing-pipeline-configs.md). \ No newline at end of file From e51789f3e2efb754e744ccb1602887a9575de82c Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 19:20:09 -0400 Subject: [PATCH 3/9] fix: update all redirect logic to support refactor --- mkdocs.template.yml | 1 - redirects/redirect.data.yml | 159 ++++++++++++++++++++++++--------- redirects/redirect.helper.star | 9 +- redirects/redirect.schema.yml | 6 +- 4 files changed, 126 insertions(+), 49 deletions(-) diff --git a/mkdocs.template.yml b/mkdocs.template.yml index d79bf87e1..5370f1650 100644 --- a/mkdocs.template.yml +++ b/mkdocs.template.yml @@ -23,7 +23,6 @@ plugins: resources: #@ template.replace(generateRoutes(docs.resources)) resource-types: #@ template.replace(generateRoutes(docs.resource_types)) steps: #@ template.replace(generateRoutes(docs.steps)) - how-to: #@ template.replace(generateRoutes(docs.how_to)) operation: #@ template.replace(generateRoutes(docs.operation)) internals: #@ template.replace(generateRoutes(docs.internals)) examples: #@ template.replace(generateRoutes(data.values.examples)) diff --git a/redirects/redirect.data.yml b/redirects/redirect.data.yml index 632da71c7..ecb95a6d6 100644 --- a/redirects/redirect.data.yml +++ b/redirects/redirect.data.yml @@ -155,33 +155,6 @@ docs: to: modifier-and-hooks/on-error - from: ensure-hook to: modifier-and-hooks/ensure - how_to: - routes: - - from: how-to-guides - to: index - - - from: managing-pipeline-configurations - to: pipeline-guides/managing-pipeline-configs - - from: common-pipeline-practices - to: pipeline-guides/common-pipeline - - from: exploring-task-input-and-output-scenarios - to: pipeline-guides/task-inputs-outputs - - from: gated-pipeline-patterns - to: pipeline-guides/gated-pipelines - - from: time-triggered-pipeline-patterns - to: pipeline-guides/time-triggered-pipelines - - from: manual-approval-step - to: pipeline-guides/manual-approval - - - from: basic-git-operations - to: git-guides/basic - - from: multi-branch-workflows - to: git-guides/multi-branch - - - from: building-and-pushing-an-image - to: container-image-guides/build-push - - from: building-an-image-and-using-it-in-a-task - to: container-image-guides/build-use operation: routes: @@ -249,33 +222,137 @@ examples: to: index html_only: true - from: hello-world-example - to: hello-world + to: fundamentals/hello-world - from: serial-job-example - to: serial-job + to: pipeline-patterns/serial-job - from: pipeline-vars-example - to: pipeline-vars + to: pipeline-patterns/pipeline-vars - from: set-pipelines-example - to: set-pipeline + to: fundamentals/set-pipeline - from: task-inputs-outputs-example - to: inputs-outputs + to: fundamentals/inputs-outputs - from: time-trigger-example - to: time-triggered + to: pipeline-patterns/time-triggered - from: git-trigger-example - to: git-triggered + to: git-workflows/git-triggered - from: manual-trigger-example - to: manually-triggered + to: pipeline-patterns/manually-triggered - from: hooks-example - to: job-and-task-hooks + to: task-composition/job-and-task-hooks - from: golang-library-example - to: golang-lib + to: language-pipelines/golang-lib - from: rails-example - to: rails-app + to: language-pipelines/rails-app - from: java-example - to: java-app + to: language-pipelines/java-app - from: nodejs-example - to: nodejs-app + to: language-pipelines/nodejs-app - from: php-example - to: php-app + to: language-pipelines/php-app + + #! --- Legacy Booklit aliases: How-To Guides (retired docs/how-to root -> examples) --- + - from: how-to-guides + to: index + - from: managing-pipeline-configurations + to: pipeline-patterns/managing-pipeline-configs + - from: common-pipeline-practices + to: pipeline-patterns/common-pipeline + - from: exploring-task-input-and-output-scenarios + to: pipeline-patterns/task-inputs-outputs + - from: gated-pipeline-patterns + to: pipeline-patterns/gated-pipelines + - from: time-triggered-pipeline-patterns + to: pipeline-patterns/time-triggered + - from: manual-approval-step + to: pipeline-patterns/manual-approval + - from: basic-git-operations + to: git-workflows/basic + - from: multi-branch-workflows + to: git-workflows/multi-branch + - from: building-and-pushing-an-image + to: container-images/build-push + - from: building-an-image-and-using-it-in-a-task + to: container-images/build-use + + #! --- Current mkdocs-era URLs being retired by the consolidation --- + - from: docs/how-to/index + to: index + md_only: true + - from: docs/how-to/pipeline-guides/managing-pipeline-configs + to: pipeline-patterns/managing-pipeline-configs + md_only: true + - from: docs/how-to/pipeline-guides/common-pipeline + to: pipeline-patterns/common-pipeline + md_only: true + - from: docs/how-to/pipeline-guides/task-inputs-outputs + to: pipeline-patterns/task-inputs-outputs + md_only: true + - from: docs/how-to/pipeline-guides/gated-pipelines + to: pipeline-patterns/gated-pipelines + md_only: true + - from: docs/how-to/pipeline-guides/time-triggered-pipelines + to: pipeline-patterns/time-triggered + md_only: true + - from: docs/how-to/pipeline-guides/manual-approval + to: pipeline-patterns/manual-approval + md_only: true + - from: docs/how-to/git-guides/basic + to: git-workflows/basic + md_only: true + - from: docs/how-to/git-guides/multi-branch + to: git-workflows/multi-branch + md_only: true + - from: docs/how-to/git-guides/monorepo + to: git-workflows/monorepo + md_only: true + - from: docs/how-to/container-image-guides/build-push + to: container-images/build-push + md_only: true + - from: docs/how-to/container-image-guides/build-use + to: container-images/build-use + md_only: true + - from: examples/hello-world + to: fundamentals/hello-world + md_only: true + - from: examples/inputs-outputs + to: fundamentals/inputs-outputs + md_only: true + - from: examples/set-pipeline + to: fundamentals/set-pipeline + md_only: true + - from: examples/serial-job + to: pipeline-patterns/serial-job + md_only: true + - from: examples/pipeline-vars + to: pipeline-patterns/pipeline-vars + md_only: true + - from: examples/time-triggered + to: pipeline-patterns/time-triggered + md_only: true + - from: examples/manually-triggered + to: pipeline-patterns/manually-triggered + md_only: true + - from: examples/git-triggered + to: git-workflows/git-triggered + md_only: true + - from: examples/job-and-task-hooks + to: task-composition/job-and-task-hooks + md_only: true + - from: examples/golang-lib + to: language-pipelines/golang-lib + md_only: true + - from: examples/rails-app + to: language-pipelines/rails-app + md_only: true + - from: examples/java-app + to: language-pipelines/java-app + md_only: true + - from: examples/nodejs-app + to: language-pipelines/nodejs-app + md_only: true + - from: examples/php-app + to: language-pipelines/php-app + md_only: true project: routes: diff --git a/redirects/redirect.helper.star b/redirects/redirect.helper.star index bb22b0405..4815c5630 100644 --- a/redirects/redirect.helper.star +++ b/redirects/redirect.helper.star @@ -4,8 +4,13 @@ def generateRoutes(section): for route in section.routes: toRoute = "{}/{}.md".format(section.root, route["to"]) - if hasattr(route, "html_only") and route["html_only"]: + htmlOnly = hasattr(route, "html_only") and route["html_only"] + mdOnly = hasattr(route, "md_only") and route["md_only"] + + if htmlOnly: created.update({"{}.html.md".format(route["from"]): toRoute}) + elif mdOnly: + created.update({"{}.md".format(route["from"]): toRoute}) else: created.update({"{}.html.md".format(route["from"]): toRoute}) created.update({"{}.md".format(route["from"]): toRoute}) @@ -14,4 +19,4 @@ def generateRoutes(section): end return created -end \ No newline at end of file +end diff --git a/redirects/redirect.schema.yml b/redirects/redirect.schema.yml index 52babc2b9..3394449f1 100644 --- a/redirects/redirect.schema.yml +++ b/redirects/redirect.schema.yml @@ -43,11 +43,6 @@ docs: routes: - from: "" to: "" - how_to: - root: "docs/how-to" - routes: - - from: "" - to: "" operation: root: "docs/operation" routes: @@ -65,6 +60,7 @@ examples: - from: "" to: "" html_only: false + md_only: false project: root: "project" From f630452d56b738344ca011156eabd6c1ea1a34a6 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 21:54:49 -0400 Subject: [PATCH 4/9] docx: update AWS IDToken Guide with TF step by step --- docs/docs/operation/creds/id-token.md | 236 +----------------- .../credentials/assets/idtoken-aws-source.tf | 99 ++++++++ docs/examples/credentials/idtoken-aws.md | 93 +++++++ docs/examples/credentials/idtoken-azure.md | 54 ++++ docs/examples/credentials/idtoken-vault.md | 121 +++++++++ .../templating/ytt-multi-file-pipeline.md | 19 ++ .../templating/ytt-single-pipeline.md | 11 + 7 files changed, 402 insertions(+), 231 deletions(-) create mode 100644 docs/examples/credentials/assets/idtoken-aws-source.tf diff --git a/docs/docs/operation/creds/id-token.md b/docs/docs/operation/creds/id-token.md index 5d3be63ea..8e4d3e773 100644 --- a/docs/docs/operation/creds/id-token.md +++ b/docs/docs/operation/creds/id-token.md @@ -172,236 +172,10 @@ while keeping your Concourse instance private. out of sync with Concourse's JWKS, token verification will fail. The recommended approach is to use a reverse proxy that forwards requests to your private Concourse in real-time, eliminating sync delays during key rotation. -## Examples +## Guides -### Vault +For end-to-end setup with a specific provider, see: -You can use JWTs to authenticate -with [HashiCorp Vault](https://developer.hashicorp.com/vault/docs/auth/jwt#jwt-authentication). This way your pipelines -can directly communicate with Vault and use all of its features, beyond what Concourse's native Vault-integration -offers. - -First enable the JWT auth method in your Vault Server: - -```shell -vault auth enable jwt -``` - -Now configure the JWT auth method to accept JWTs issued by your Concourse (use your `--oidc-issuer-url` if configured, -otherwise your external URL - see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)): - -```shell -vault write auth/jwt/config \ - oidc_discovery_url="https://" \ - default_role="demo" -``` - -Lastly, configure a role for JWT auth. Make sure to use the same value in your pipeline that you used for -_bound_audiences_ (the best would be the URL of your Vault). _bound_subject_ must be the sub-claim value of your JWT, if -you use the _subject_scope_ setting to change the contents of your sub-claim, adapt this accordingly! - -```shell -vault write auth/jwt/role/demo \ - role_type="jwt"\ - user_claim="sub" \ - bound_subject="main/your-pipeline" \ - bound_audiences="my-vault-server.com" \ - policies=webapps \ - ttl=1h -``` - -This role will allow the holder of a JWT with aud: "`my-vault-server.com`" and sub: "`main/your-pipeline`" to get a -Vault token with the Vault-policy `webapps`. If the policy you want to assign has a different name, simply change it in -the above example. Make sure to adapt the value for `bound_subject` according to your team and pipeline name. - -Pipelines can now do the following: - -```yaml -var_sources: - - name: vaulttoken - type: idtoken - config: - audience: [ "my-vault-server.com" ] - -jobs: - - name: vault-login - plan: - - task: login - config: - platform: linux - image_resource: - type: registry-image - source: { repository: hashicorp/vault } - run: - path: sh - args: - - -e - - -c - - | - export VAULT_ADDR=https://my-vault-server.com - vault write auth/jwt/login \ - role=demo \ - jwt=((vaulttoken:token)) \ - --format=json > vault-response.json - echo "Now do something with the token in vault-response.json" -``` - -You don't have to create a role and a policy for every single of your pipelines! You can use claims from the JWT with -Vault's [policy templating](https://developer.hashicorp.com/vault/tutorials/policies/policy-templating) feature. This -way you can define a policy that allows a pipeline read to all the secrets it would usually have access to using -Concourse's native Vault-integration: - -```hcl -path "concourse/metadata/{{ identity.entity.aliases..metadata.team }}" { - capabilities = ["list"] -} - -path "concourse/data/{{ identity.entity.aliases..metadata.team }}/+" { - capabilities = ["read"] -} - -path "concourse/metadata/{{ identity.entity.aliases..metadata.team }}/{{ identity.entity.aliases..metadata.pipeline }}" { - capabilities = ["list"] -} - -path "concourse/metadata/{{ identity.entity.aliases..metadata.team }}/{{ identity.entity.aliases..metadata.pipeline }}/*" { - capabilities = ["read", "list"] -} - -path "concourse/data/{{ identity.entity.aliases..metadata.team }}/{{ identity.entity.aliases..metadata.pipeline }}/*" { - capabilities = ["read", "list"] -} -``` - -!!! note - - Make sure to set `` to the actual mount-accessor value of your JWT Auth method! You can use `vault - auth list --format=json | jq -r '."jwt/".accessor'` to get the accessor for your jwt auth method. - -With a policy like this you don't need to configure `bound_subject` in your JWT auth role. Every single pipeline can -simply use the same role and the policy will take care that they can only access secrets meant for them. However, you -need to explicitly configure claim to metadata mapping: - -```shell -vault write auth/jwt/role/demo \ - role_type="jwt"\ - user_claim="sub" \ - bound_subject= \ - bound_audiences="my-vault-server.com" \ - policies=pipeline-new \ - claim_mappings='team=team' \ - claim_mappings='pipeline=pipeline' \ - ttl=1h -``` - -### AWS - -AWS -supports [federation with external identity providers](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers.html). -Using this, you can allow identities managed by an external identity provider to perform actions in your AWS account. - -In this scenario the external identity provider is Concourse and the identities are teams/pipelines/jobs. This means you -are able to grant a specific pipeline or job permission to perform actions in AWS (like deploying something), all -without managing IAM users or dealing with long-lived credentials. - -First you need -to [create an OpenID Connect identity provider](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html) -in your AWS Account. Set _Provider URL_ to the external URL of your Concourse server (or the `--oidc-issuer-url` if -you're using a separate OIDC issuer - see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)). -For _Audience_, you can choose any string you like, but using a value like `sts.amazonaws.com` is recommended. You have -to use the same string later in the configuration of your [`idtoken` var source](../../vars.md#id-token). - -Next you will need -to [create an IAM-Role that can be assumed using your JWT](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html#idp_oidc_Create). -Set _Identity Provider_ to the value you previously set _Audience_ to. Add a condition on the sub-claim with -type `StringEquals` and value `yourteam/yourpipeline`. This will allow ONLY that specific pipeline (and any instanced -versions of it) to assume that IAM Role using a JWT. If you use the `subject_scope` setting to change the contents of -your sub-claim, adapt this condition accordingly! In the next step you will be able to choose which AWS permissions your -role will get. - -Now you can use -the [AWS AssumeRoleWithWebIdentity API operation](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) -to assume your role via a JWT issued by Concourse. The easiest way is to do this is via -the [assume-role-with-web-identity AWS CLI command](https://docs.aws.amazon.com/cli/latest/reference/sts/assume-role-with-web-identity.html): - -```yaml -var_sources: - - name: awstoken - type: idtoken - config: - audience: [ "sts.amazonaws.com" ] - -jobs: - - name: aws-login - plan: - - task: print - config: - platform: linux - image_resource: - type: registry-image - source: { repository: amazon/aws-cli } - run: - path: bash - args: - - -e - - -c - - | - aws sts assume-role-with-web-identity \ - --role-session-name Concourse \ - --role-arn arn:aws:iam:::role/ \ - --web-identity-token ((awstoken:token)) > creds.json - echo "Now do something with the temporary credentials in creds.json" -``` - -### Azure - -Azure also supports a way to grant the holder of a JWT permissions in the Cloud. This is done via a feature -called [Federated Credentials](https://learn.microsoft.com/en-us/graph/api/resources/federatedidentitycredentials-overview?view=graph-rest-1.0). - -First, [create an EntraID App Registration](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app). -This app registration will be the service principal used by your pipeline. - -Now [create a federated credential](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp#other-identity-providers) -for the app registration you just created. - -For _Scenario_ select "Other". For Issuer set it to the external URL of your Concourse server (or the -`--oidc-issuer-url` if you're using a separate OIDC issuer - -see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)). -For _Type_ select "Explicit subject identifier" and set _Value_ to `/` of the pipeline that -should be able to use the identity. If you use the `subject_scope` setting to change the contents of your sub-claim, -change this setting here accordingly. - -You can now assign IAM permissions to the identity of the app registration, which define what the identity is allowed to -do in your Azure subscription. - -Your pipeline can now use the `az cli` to log in to Azure using a JWT generated by Concourse: - -```yaml -var_sources: - - name: azuretoken - type: idtoken - config: - audience: [ "api://AzureADTokenExchange" ] - -jobs: - - name: azure-deploy - plan: - - task: login - config: - platform: linux - image_resource: - type: registry-image - source: { repository: mcr.microsoft.com/azure-cli } - run: - path: bash - args: - - -e - - -c - - | - echo ((azuretoken:token)) - az login --service-principal \ - -u \ - --tenant \ - --federated-token ((azuretoken:token)) - echo "You are now authenticated with Azure. Do something with it!" -``` \ No newline at end of file +- [Authenticating to Vault via IDToken](../../../examples/credentials/idtoken-vault) +- [Authenticating to AWS via IDToken](../../../examples/credentials/idtoken-aws) +- [Authenticating to Azure via IDToken](../../../examples/credentials/idtoken-azure) diff --git a/docs/examples/credentials/assets/idtoken-aws-source.tf b/docs/examples/credentials/assets/idtoken-aws-source.tf new file mode 100644 index 000000000..5df4cfb07 --- /dev/null +++ b/docs/examples/credentials/assets/idtoken-aws-source.tf @@ -0,0 +1,99 @@ +terraform { + required_providers { + tls = { + source = "hashicorp/tls" + version = "4.2.1" + } + aws = { + source = "hashicorp/aws" + version = "6.36.0" + } + } +} + +variable "concourse_url" { + description = "Concourse URL" + type = string + + default = "https://ci.concourse-ci.org" +} + +data "tls_certificate" "root_certificate" { + url = var.concourse_url +} + +resource "aws_iam_openid_connect_provider" "oidc_provider" { + url = var.concourse_url + + client_id_list = [ + "sts.amazonaws.com" + ] + + thumbprint_list = [ + data.tls_certificate.root_certificate.certificates[0].sha1_fingerprint + ] +} + +data "aws_iam_policy_document" "assume_role_policy" { + statement { + sid = "ConcourseOIDCWebIdentity" + effect = "Allow" + actions = [ + "sts:AssumeRoleWithWebIdentity" + ] + + principals { + type = "Federated" + identifiers = [ + aws_iam_openid_connect_provider.oidc_provider.arn + ] + } + + condition { + test = "StringEquals" + variable = "${aws_iam_openid_connect_provider.oidc_provider.arn}:aud" + values = [ + "sts.amazonaws.com" + ] + } + + condition { + test = "StringEquals" + variable = "${aws_iam_openid_connect_provider.oidc_provider.arn}:sub" + values = [ + "main/deploy-to-aws" + ] + } + } +} + +resource "aws_iam_role" "role" { + name = "s3_manager" + assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json +} + +data "aws_iam_policy_document" "manage_s3" { + statement { + sid = "ManageS3" + effect = "Allow" + + actions = [ + "s3:*" + ] + + resources = [ + "*" + ] + } +} + +resource "aws_iam_policy" "assumed_policy" { + name = "s3_manager_policy" + policy = data.aws_iam_policy_document.manage_s3.json +} + +resource "aws_iam_role_policy_attachment" "role_policy_attachment" { + role = aws_iam_role.role.name + policy_arn = aws_iam_policy.assumed_policy.arn +} + diff --git a/docs/examples/credentials/idtoken-aws.md b/docs/examples/credentials/idtoken-aws.md index e69de29bb..a196b8c9a 100644 --- a/docs/examples/credentials/idtoken-aws.md +++ b/docs/examples/credentials/idtoken-aws.md @@ -0,0 +1,93 @@ +--- +title: IDToken - AWS +--- + + +AWS +supports [federation with external identity providers](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers.html). +Using this, you can allow identities managed by an external identity provider to perform actions in your AWS account. + +In this scenario the external identity provider is Concourse and the identities are teams/pipelines/jobs. This means you +are able to grant a specific pipeline or job permission to perform actions in AWS (like deploying something), all +without managing IAM users or dealing with long-lived credentials. + +## Create OIDC Provider + +First you need +to [create an OpenID Connect identity provider](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html) +in your AWS Account: + +1. Set `url` to the external URL of your Concourse server (or the `--oidc-issuer-url` if you're using a separate OIDC + issuer - see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)). +2. For `client_id_list`, you can choose any string you like, but using a value like `sts.amazonaws.com` is recommended. + +!!! note + + You have to use the same string later in the configuration of your [`idtoken` var source](../../vars.md#id-token). + +```hcl linenums="1" +--8<-- "examples/credentials/assets/idtoken-aws-source.tf::35" +``` + +## Allow Role Assumption + +Next you will need +to [create an IAM-Role that can be assumed using your JWT](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html#idp_oidc_Create). + +First, create a policy document that has allows for the Provider to call `sts:AssumeRoleWithWebIdentity`. + +Add a condition on the sub-claim with type `StringEquals` and value `main/deploy-to-aws`. This will allow ONLY that +specific pipeline (and any instanced versions of it) to assume that IAM Role using a JWT. + +Add a second condition on the audience with type `StringEquals` and value `sts.amazonaws.com`. + +```hcl linenums="36" +--8<-- "examples/credentials/assets/idtoken-aws-source.tf:37:73" +``` + +## Allow for an Action + +Now, assign the assumed role a policy that the Pipeline can use. + +```hcl linenums="73" +--8<-- "examples/credentials/assets/idtoken-aws-source.tf:74:" +``` + +## Use within the Pipeline +Now you can use +the [AWS AssumeRoleWithWebIdentity API operation](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) +to assume your role via a JWT issued by Concourse. + +The easiest way is to do this is via +the [assume-role-with-web-identity AWS CLI command](https://docs.aws.amazon.com/cli/latest/reference/sts/assume-role-with-web-identity.html): + +```yaml linenums="1" +var_sources: + - name: awstoken + type: idtoken + config: + audience: + - "sts.amazonaws.com" + +jobs: + - name: aws-login + plan: + - task: print + config: + platform: linux + image_resource: + type: registry-image + source: + repository: amazon/aws-cli + run: + path: bash + args: + - -e + - -c + - | + aws sts assume-role-with-web-identity \ + --role-session-name Concourse \ + --role-arn arn:aws:iam:::role/s3_manager \ + --web-identity-token ((awstoken:token)) > creds.json + echo "Now do something with the temporary credentials in creds.json" +``` \ No newline at end of file diff --git a/docs/examples/credentials/idtoken-azure.md b/docs/examples/credentials/idtoken-azure.md index e69de29bb..13d3d956a 100644 --- a/docs/examples/credentials/idtoken-azure.md +++ b/docs/examples/credentials/idtoken-azure.md @@ -0,0 +1,54 @@ +--- +title: IDToken - Azure +--- + +Azure also supports a way to grant the holder of a JWT permissions in the Cloud. This is done via a feature +called [Federated Credentials](https://learn.microsoft.com/en-us/graph/api/resources/federatedidentitycredentials-overview?view=graph-rest-1.0). + +First, [create an EntraID App Registration](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app). +This app registration will be the service principal used by your pipeline. + +Now [create a federated credential](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp#other-identity-providers) +for the app registration you just created. + +For _Scenario_ select "Other". For Issuer set it to the external URL of your Concourse server (or the +`--oidc-issuer-url` if you're using a separate OIDC issuer - +see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)). +For _Type_ select "Explicit subject identifier" and set _Value_ to `/` of the pipeline that +should be able to use the identity. If you use the `subject_scope` setting to change the contents of your sub-claim, +change this setting here accordingly. + +You can now assign IAM permissions to the identity of the app registration, which define what the identity is allowed to +do in your Azure subscription. + +Your pipeline can now use the `az cli` to log in to Azure using a JWT generated by Concourse: + +```yaml +var_sources: + - name: azuretoken + type: idtoken + config: + audience: [ "api://AzureADTokenExchange" ] + +jobs: + - name: azure-deploy + plan: + - task: login + config: + platform: linux + image_resource: + type: registry-image + source: { repository: mcr.microsoft.com/azure-cli } + run: + path: bash + args: + - -e + - -c + - | + echo ((azuretoken:token)) + az login --service-principal \ + -u \ + --tenant \ + --federated-token ((azuretoken:token)) + echo "You are now authenticated with Azure. Do something with it!" +``` \ No newline at end of file diff --git a/docs/examples/credentials/idtoken-vault.md b/docs/examples/credentials/idtoken-vault.md index e69de29bb..0638b4784 100644 --- a/docs/examples/credentials/idtoken-vault.md +++ b/docs/examples/credentials/idtoken-vault.md @@ -0,0 +1,121 @@ +--- +title: IDToken - HashiCorp Vault +--- + +You can use JWTs to authenticate +with [HashiCorp Vault](https://developer.hashicorp.com/vault/docs/auth/jwt#jwt-authentication). This way your pipelines +can directly communicate with Vault and use all of its features, beyond what Concourse's native Vault-integration +offers. + +First enable the JWT auth method in your Vault Server: + +```shell +vault auth enable jwt +``` + +Now configure the JWT auth method to accept JWTs issued by your Concourse (use your `--oidc-issuer-url` if configured, +otherwise your external URL - see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)): + +```shell +vault write auth/jwt/config \ + oidc_discovery_url="https://" \ + default_role="demo" +``` + +Lastly, configure a role for JWT auth. Make sure to use the same value in your pipeline that you used for +_bound_audiences_ (the best would be the URL of your Vault). _bound_subject_ must be the sub-claim value of your JWT, if +you use the _subject_scope_ setting to change the contents of your sub-claim, adapt this accordingly! + +```shell +vault write auth/jwt/role/demo \ + role_type="jwt"\ + user_claim="sub" \ + bound_subject="main/your-pipeline" \ + bound_audiences="my-vault-server.com" \ + policies=webapps \ + ttl=1h +``` + +This role will allow the holder of a JWT with aud: "`my-vault-server.com`" and sub: "`main/your-pipeline`" to get a +Vault token with the Vault-policy `webapps`. If the policy you want to assign has a different name, simply change it in +the above example. Make sure to adapt the value for `bound_subject` according to your team and pipeline name. + +Pipelines can now do the following: + +```yaml +var_sources: + - name: vaulttoken + type: idtoken + config: + audience: [ "my-vault-server.com" ] + +jobs: + - name: vault-login + plan: + - task: login + config: + platform: linux + image_resource: + type: registry-image + source: { repository: hashicorp/vault } + run: + path: sh + args: + - -e + - -c + - | + export VAULT_ADDR=https://my-vault-server.com + vault write auth/jwt/login \ + role=demo \ + jwt=((vaulttoken:token)) \ + --format=json > vault-response.json + echo "Now do something with the token in vault-response.json" +``` + +You don't have to create a role and a policy for every single of your pipelines! You can use claims from the JWT with +Vault's [policy templating](https://developer.hashicorp.com/vault/tutorials/policies/policy-templating) feature. This +way you can define a policy that allows a pipeline read to all the secrets it would usually have access to using +Concourse's native Vault-integration: + +```hcl +path "concourse/metadata/{{ identity.entity.aliases..metadata.team }}" { + capabilities = ["list"] +} + +path "concourse/data/{{ identity.entity.aliases..metadata.team }}/+" { + capabilities = ["read"] +} + +path "concourse/metadata/{{ identity.entity.aliases..metadata.team }}/{{ identity.entity.aliases..metadata.pipeline }}" { + capabilities = ["list"] +} + +path "concourse/metadata/{{ identity.entity.aliases..metadata.team }}/{{ identity.entity.aliases..metadata.pipeline }}/*" { + capabilities = ["read", "list"] +} + +path "concourse/data/{{ identity.entity.aliases..metadata.team }}/{{ identity.entity.aliases..metadata.pipeline }}/*" { + capabilities = ["read", "list"] +} +``` + +!!! note + + Make sure to set `` to the actual mount-accessor value of your JWT Auth method! You can use `vault + auth list --format=json | jq -r '."jwt/".accessor'` to get the accessor for your jwt auth method. + +With a policy like this you don't need to configure `bound_subject` in your JWT auth role. Every single pipeline can +simply use the same role and the policy will take care that they can only access secrets meant for them. However, you +need to explicitly configure claim to metadata mapping: + +```shell +vault write auth/jwt/role/demo \ + role_type="jwt"\ + user_claim="sub" \ + bound_subject= \ + bound_audiences="my-vault-server.com" \ + policies=pipeline-new \ + claim_mappings='team=team' \ + claim_mappings='pipeline=pipeline' \ + ttl=1h +``` \ No newline at end of file diff --git a/docs/examples/templating/ytt-multi-file-pipeline.md b/docs/examples/templating/ytt-multi-file-pipeline.md index e69de29bb..3f71293fe 100644 --- a/docs/examples/templating/ytt-multi-file-pipeline.md +++ b/docs/examples/templating/ytt-multi-file-pipeline.md @@ -0,0 +1,19 @@ +--- +title: Templating - Multiple Files +--- + +```yaml linenums="1" +--8<-- "libs/examples/pipelines/templates/multiple-files/jobs.lib.yml" +``` + +```yaml linenums="1" +--8<-- "libs/examples/pipelines/templates/multiple-files/resources.lib.yml" +``` + +```yaml linenums="1" +--8<-- "libs/examples/pipelines/templates/multiple-files/template.yml" +``` + +```yaml linenums="1" +--8<-- "libs/examples/pipelines/templates/multiple-files/vars.yml" +``` \ No newline at end of file diff --git a/docs/examples/templating/ytt-single-pipeline.md b/docs/examples/templating/ytt-single-pipeline.md index e69de29bb..92e440ca0 100644 --- a/docs/examples/templating/ytt-single-pipeline.md +++ b/docs/examples/templating/ytt-single-pipeline.md @@ -0,0 +1,11 @@ +--- +title: Templating - Single File +--- + +```yaml linenums="1" +--8<-- "libs/examples/pipelines/templates/simple/template.yml" +``` + +```yaml linenums="1" +--8<-- "libs/examples/pipelines/templates/simple/vars.yml" +``` \ No newline at end of file From 81570ae88c624a45f31c8aeaf981cd6bf47cbbc5 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 21:59:48 -0400 Subject: [PATCH 5/9] other: switch to numbering --- docs/examples/credentials/idtoken-aws.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/examples/credentials/idtoken-aws.md b/docs/examples/credentials/idtoken-aws.md index a196b8c9a..93d0f2521 100644 --- a/docs/examples/credentials/idtoken-aws.md +++ b/docs/examples/credentials/idtoken-aws.md @@ -34,12 +34,13 @@ in your AWS Account: Next you will need to [create an IAM-Role that can be assumed using your JWT](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html#idp_oidc_Create). -First, create a policy document that has allows for the Provider to call `sts:AssumeRoleWithWebIdentity`. +1. Create a policy document that has allows for the Provider to call `sts:AssumeRoleWithWebIdentity` +2. Add a condition on the sub-claim with type `StringEquals` and value `main/deploy-to-aws` +3. Add a condition on the audience with type `StringEquals` and value `sts.amazonaws.com` -Add a condition on the sub-claim with type `StringEquals` and value `main/deploy-to-aws`. This will allow ONLY that -specific pipeline (and any instanced versions of it) to assume that IAM Role using a JWT. +!!! info -Add a second condition on the audience with type `StringEquals` and value `sts.amazonaws.com`. + This will allow ONLY that specific pipeline (and any instanced versions of it) to assume that IAM Role using a JWT. ```hcl linenums="36" --8<-- "examples/credentials/assets/idtoken-aws-source.tf:37:73" @@ -54,6 +55,7 @@ Now, assign the assumed role a policy that the Pipeline can use. ``` ## Use within the Pipeline + Now you can use the [AWS AssumeRoleWithWebIdentity API operation](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html) to assume your role via a JWT issued by Concourse. From a670992061afaf68ac21c3ebbba8bda26ef37993 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 22:42:36 -0400 Subject: [PATCH 6/9] docx: add GCP Example (#611) NOTE: This is very much untested as is the Azure example --- base.yml | 1 + docs/docs/operation/creds/id-token.md | 6 +- .../credentials/assets/idtoken-aws-source.tf | 6 +- .../assets/idtoken-azure-source.tf | 53 +++++++++ .../credentials/assets/idtoken-gcp-source.tf | 64 +++++++++++ docs/examples/credentials/idtoken-aws.md | 5 +- docs/examples/credentials/idtoken-azure.md | 72 +++++++++--- docs/examples/credentials/idtoken-gcp.md | 107 ++++++++++++++++++ docs/examples/credentials/idtoken-vault.md | 3 +- docs/examples/credentials/index.md | 13 ++- 10 files changed, 303 insertions(+), 27 deletions(-) create mode 100644 docs/examples/credentials/assets/idtoken-azure-source.tf create mode 100644 docs/examples/credentials/assets/idtoken-gcp-source.tf create mode 100644 docs/examples/credentials/idtoken-gcp.md diff --git a/base.yml b/base.yml index 2c7a8b1e4..c9bdb485f 100644 --- a/base.yml +++ b/base.yml @@ -276,6 +276,7 @@ nav: - examples/credentials/idtoken-vault.md - examples/credentials/idtoken-aws.md - examples/credentials/idtoken-azure.md + - examples/credentials/idtoken-gcp.md - Templating: - examples/templating/index.md - examples/templating/ytt-single-pipeline.md diff --git a/docs/docs/operation/creds/id-token.md b/docs/docs/operation/creds/id-token.md index 8e4d3e773..8e5ede513 100644 --- a/docs/docs/operation/creds/id-token.md +++ b/docs/docs/operation/creds/id-token.md @@ -176,6 +176,6 @@ while keeping your Concourse instance private. For end-to-end setup with a specific provider, see: -- [Authenticating to Vault via IDToken](../../../examples/credentials/idtoken-vault) -- [Authenticating to AWS via IDToken](../../../examples/credentials/idtoken-aws) -- [Authenticating to Azure via IDToken](../../../examples/credentials/idtoken-azure) +- [Authenticating to Vault via IDToken](../../../examples/credentials/idtoken-vault.md) +- [Authenticating to AWS via IDToken](../../../examples/credentials/idtoken-aws.md) +- [Authenticating to Azure via IDToken](../../../examples/credentials/idtoken-azure.md) diff --git a/docs/examples/credentials/assets/idtoken-aws-source.tf b/docs/examples/credentials/assets/idtoken-aws-source.tf index 5df4cfb07..06a40c8e3 100644 --- a/docs/examples/credentials/assets/idtoken-aws-source.tf +++ b/docs/examples/credentials/assets/idtoken-aws-source.tf @@ -11,7 +11,7 @@ terraform { } } -variable "concourse_url" { +variable "aws_concourse_url" { description = "Concourse URL" type = string @@ -19,11 +19,11 @@ variable "concourse_url" { } data "tls_certificate" "root_certificate" { - url = var.concourse_url + url = var.aws_concourse_url } resource "aws_iam_openid_connect_provider" "oidc_provider" { - url = var.concourse_url + url = var.aws_concourse_url client_id_list = [ "sts.amazonaws.com" diff --git a/docs/examples/credentials/assets/idtoken-azure-source.tf b/docs/examples/credentials/assets/idtoken-azure-source.tf new file mode 100644 index 000000000..7079b8179 --- /dev/null +++ b/docs/examples/credentials/assets/idtoken-azure-source.tf @@ -0,0 +1,53 @@ +terraform { + required_providers { + azuread = { + source = "hashicorp/azuread" + version = "3.0.2" + } + azurerm = { + source = "hashicorp/azurerm" + version = "4.15.0" + } + } +} + +provider "azurerm" { + features {} +} + +variable "azure_concourse_url" { + description = "Concourse URL" + type = string + + default = "https://ci.concourse-ci.org" +} + +resource "azuread_application_registration" "pipeline_app" { + display_name = "concourse-deploy" +} + +resource "azuread_service_principal" "pipeline_sp" { + client_id = azuread_application_registration.pipeline_app.client_id +} + +resource "azuread_application_federated_identity_credential" "concourse_federation" { + application_id = azuread_application_registration.pipeline_app.id + display_name = "concourse-oidc" + description = "Federated credential trusting JWTs issued by Concourse" + + issuer = var.azure_concourse_url + subject = "main/deploy-to-azure" + audiences = [ + "api://AzureADTokenExchange" + ] +} + +data "azurerm_resource_group" "target" { + name = "example-resources" +} + +resource "azurerm_role_assignment" "pipeline_access" { + scope = data.azurerm_resource_group.target.id + role_definition_name = "Contributor" + principal_id = azuread_service_principal.pipeline_sp.object_id +} diff --git a/docs/examples/credentials/assets/idtoken-gcp-source.tf b/docs/examples/credentials/assets/idtoken-gcp-source.tf new file mode 100644 index 000000000..1653dbc03 --- /dev/null +++ b/docs/examples/credentials/assets/idtoken-gcp-source.tf @@ -0,0 +1,64 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "6.9.0" + } + } +} + +variable "gcp_concourse_url" { + description = "Concourse URL" + type = string + + default = "https://ci.concourse-ci.org" +} + +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "project_number" { + description = "GCP Project Number" + type = string +} + +resource "google_iam_workload_identity_pool" "concourse_pool" { + workload_identity_pool_id = "concourse-pool" + display_name = "Concourse" + description = "Identity pool for Concourse pipelines" +} + +resource "google_iam_workload_identity_pool_provider" "concourse_provider" { + workload_identity_pool_id = google_iam_workload_identity_pool.concourse_pool.workload_identity_pool_id + workload_identity_pool_provider_id = "concourse-provider" + + attribute_mapping = { + "google.subject" = "assertion.sub" + } + + oidc { + issuer_uri = var.gcp_concourse_url + allowed_audiences = [ + "https://iam.googleapis.com/projects/${var.project_number}/locations/global/workloadIdentityPools/concourse-pool/providers/concourse-provider" + ] + } +} + +resource "google_service_account" "pipeline_sa" { + account_id = "concourse-deploy" + display_name = "Concourse Deploy Pipeline" +} + +resource "google_service_account_iam_member" "workload_identity_binding" { + service_account_id = google_service_account.pipeline_sa.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.concourse_pool.name}/subject/main/deploy-to-gcp" +} + +resource "google_project_iam_member" "storage_admin" { + project = var.project_id + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.pipeline_sa.email}" +} diff --git a/docs/examples/credentials/idtoken-aws.md b/docs/examples/credentials/idtoken-aws.md index 93d0f2521..3af056bd7 100644 --- a/docs/examples/credentials/idtoken-aws.md +++ b/docs/examples/credentials/idtoken-aws.md @@ -18,12 +18,13 @@ to [create an OpenID Connect identity provider](https://docs.aws.amazon.com/IAM/ in your AWS Account: 1. Set `url` to the external URL of your Concourse server (or the `--oidc-issuer-url` if you're using a separate OIDC - issuer - see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)). + issuer - + see [Configuring a Separate OIDC Issuer](../../docs/operation/creds/id-token.md#configuring-a-separate-oidc-issuer)). 2. For `client_id_list`, you can choose any string you like, but using a value like `sts.amazonaws.com` is recommended. !!! note - You have to use the same string later in the configuration of your [`idtoken` var source](../../vars.md#id-token). + You have to use the same string later in the configuration of your [`idtoken` var source](../../docs/vars.md#id-token). ```hcl linenums="1" --8<-- "examples/credentials/assets/idtoken-aws-source.tf::35" diff --git a/docs/examples/credentials/idtoken-azure.md b/docs/examples/credentials/idtoken-azure.md index 13d3d956a..1f73810c5 100644 --- a/docs/examples/credentials/idtoken-azure.md +++ b/docs/examples/credentials/idtoken-azure.md @@ -2,28 +2,69 @@ title: IDToken - Azure --- -Azure also supports a way to grant the holder of a JWT permissions in the Cloud. This is done via a feature -called [Federated Credentials](https://learn.microsoft.com/en-us/graph/api/resources/federatedidentitycredentials-overview?view=graph-rest-1.0). +Azure +supports [federation with external identity providers](https://learn.microsoft.com/en-us/graph/api/resources/federatedidentitycredentials-overview?view=graph-rest-1.0) +through a feature called Federated Credentials. Using this, you can allow identities managed by an external identity +provider to perform actions in your Azure subscription. -First, [create an EntraID App Registration](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app). -This app registration will be the service principal used by your pipeline. +In this scenario the external identity provider is Concourse and the identities are teams/pipelines/jobs. This means you +are able to grant a specific pipeline or job permission to perform actions in Azure (like deploying something), all +without managing service principal secrets or dealing with long-lived credentials. -Now [create a federated credential](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp#other-identity-providers) +## Create App Registration + +First, +[create an EntraID App Registration](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app) +along with the service principal that represents it. This app registration and its service principal are what your +pipeline will act as once it exchanges a Concourse-issued JWT for an Azure token. + +```hcl linenums="1" +--8<-- "examples/credentials/assets/idtoken-azure-source.tf::31" +``` + +## Create Federated Credential + +Next, +[create a federated credential](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp#other-identity-providers) for the app registration you just created. -For _Scenario_ select "Other". For Issuer set it to the external URL of your Concourse server (or the -`--oidc-issuer-url` if you're using a separate OIDC issuer - -see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)). -For _Type_ select "Explicit subject identifier" and set _Value_ to `/` of the pipeline that -should be able to use the identity. If you use the `subject_scope` setting to change the contents of your sub-claim, -change this setting here accordingly. +1. Set `issuer` to the external URL of your Concourse server (or the `--oidc-issuer-url` if you're using a separate OIDC + issuer - + see [Configuring a Separate OIDC Issuer](../../docs/operation/creds/id-token.md#configuring-a-separate-oidc-issuer)). +2. Set `subject` to `main/deploy-to-azure`. If you use the `subject_scope` setting to change the contents of your + sub-claim, change this value here accordingly. +3. For `audiences`, you can choose any string you like, but using a value like `api://AzureADTokenExchange` is + recommended. + +!!! note + + You have to use the same string later in the configuration of your [`idtoken` var source](../../docs/vars.md#id-token). + +```hcl linenums="33" +--8<-- "examples/credentials/assets/idtoken-azure-source.tf:33:43" +``` + +!!! info + + This will allow ONLY that specific pipeline (and any instanced versions of it) to obtain a token for this + identity using a JWT. + +## Assign Role Permissions + +Now, assign the identity of the app registration an RBAC role that defines what it's allowed to do in your Azure +subscription. + +```hcl linenums="45" +--8<-- "examples/credentials/assets/idtoken-azure-source.tf:45:" +``` -You can now assign IAM permissions to the identity of the app registration, which define what the identity is allowed to -do in your Azure subscription. +## Use within the Pipeline -Your pipeline can now use the `az cli` to log in to Azure using a JWT generated by Concourse: +Your pipeline can now use the `az` CLI to log in to Azure using a JWT issued by Concourse, via +[ +`az login`'s federated-token support](https://learn.microsoft.com/en-us/cli/azure/reference-index?view=azure-cli-latest#az-login): -```yaml +```yaml linenums="1" var_sources: - name: azuretoken type: idtoken @@ -45,7 +86,6 @@ jobs: - -e - -c - | - echo ((azuretoken:token)) az login --service-principal \ -u \ --tenant \ diff --git a/docs/examples/credentials/idtoken-gcp.md b/docs/examples/credentials/idtoken-gcp.md new file mode 100644 index 000000000..b1cec707f --- /dev/null +++ b/docs/examples/credentials/idtoken-gcp.md @@ -0,0 +1,107 @@ +--- +title: IDToken - GCP +--- + +GCP supports [workload identity federation](https://cloud.google.com/iam/docs/workload-identity-federation) with +external identity providers. Using this, you can allow identities managed by an external identity provider to +impersonate a service account in your GCP project. + +In this scenario the external identity provider is Concourse and the identities are teams/pipelines/jobs. This means you +are able to grant a specific pipeline or job permission to perform actions in GCP (like deploying something), all +without managing service account keys or dealing with long-lived credentials. + +## Create Workload Identity Pool and Provider + +First, [create a workload identity pool and provider](https://cloud.google.com/iam/docs/workload-identity-federation-with-other-providers#configure) +for your GCP project. + +1. Set the provider's `issuer_uri` to the external URL of your Concourse server (or the `--oidc-issuer-url` if you're + using a separate OIDC issuer - + see [Configuring a Separate OIDC Issuer](../../docs/operation/creds/id-token.md#configuring-a-separate-oidc-issuer)). +2. Map `google.subject` to `assertion.sub` so the pipeline's identity carries through to the impersonation check below. +3. For `allowed_audiences`, the full workload identity provider resource name is recommended, as shown below. + +!!! note + + You have to use the same audience string later in the configuration of your [`idtoken` var source](../../docs/vars.md#id-token). + +```hcl linenums="1" +--8<-- "examples/credentials/assets/idtoken-gcp-source.tf::47" +``` + +## Bind a Service Account + +Next, create the service account your pipeline will impersonate, and +[bind it to the workload identity pool](https://cloud.google.com/iam/docs/workload-identity-federation-with-other-providers#allow-impersonation) +with a condition on the sub-claim. + +```hcl linenums="49" +--8<-- "examples/credentials/assets/idtoken-gcp-source.tf:49:58" +``` + +!!! info + + This will allow ONLY that specific pipeline (and any instanced versions of it) to impersonate this service + account using a JWT. + +## Allow for an Action + +Now, grant the service account a role that defines what it's allowed to do in your GCP project. + +```hcl linenums="60" +--8<-- "examples/credentials/assets/idtoken-gcp-source.tf:60:" +``` + +## Use within the Pipeline + +Now you can use the [ +`gcloud` CLI's external account credential support](https://cloud.google.com/iam/docs/workload-identity-federation-with-other-providers#generate_a_credential_configuration) +to impersonate your service account via a JWT issued by Concourse. Since `gcloud` reads external credentials from a +config file rather than a single flag, the task writes the token to a file first, then assembles a small credential +config pointing at it: + +```yaml linenums="1" +var_sources: + - name: gcptoken + type: idtoken + config: + audience: + - "https://iam.googleapis.com/projects//locations/global/workloadIdentityPools/concourse-pool/providers/concourse-provider" + +jobs: + - name: gcp-deploy + plan: + - task: deploy + config: + platform: linux + image_resource: + type: registry-image + source: + repository: google/cloud-sdk + tag: slim + run: + path: bash + args: + - -e + - -c + - | + echo ((gcptoken:token)) > /tmp/gcp-token.jwt + + cat > /tmp/gcp-creds.json </locations/global/workloadIdentityPools/concourse-pool/providers/concourse-provider", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/concourse-deploy@.iam.gserviceaccount.com:generateAccessToken", + "credential_source": { + "file": "/tmp/gcp-token.jwt" + } + } + EOF + + export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-creds.json + gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" + gsutil ls gs://my-bucket + echo "Now do something with GCP using these impersonated credentials" +``` \ No newline at end of file diff --git a/docs/examples/credentials/idtoken-vault.md b/docs/examples/credentials/idtoken-vault.md index 0638b4784..4ee445152 100644 --- a/docs/examples/credentials/idtoken-vault.md +++ b/docs/examples/credentials/idtoken-vault.md @@ -14,7 +14,8 @@ vault auth enable jwt ``` Now configure the JWT auth method to accept JWTs issued by your Concourse (use your `--oidc-issuer-url` if configured, -otherwise your external URL - see [Configuring a Separate OIDC Issuer](#configuring-a-separate-oidc-issuer)): +otherwise your external URL - +see [Configuring a Separate OIDC Issuer](../../docs/operation/creds/id-token.md#configuring-a-separate-oidc-issuer)): ```shell vault write auth/jwt/config \ diff --git a/docs/examples/credentials/index.md b/docs/examples/credentials/index.md index bb2e52806..13e54c30c 100644 --- a/docs/examples/credentials/index.md +++ b/docs/examples/credentials/index.md @@ -1,9 +1,9 @@ --- title: Credentials & Identity Federation -hide: - - toc --- +# Credentials & Identity Federation + Guides for authenticating pipelines to external services using Concourse-issued JWTs (the [`idtoken` var source](../../docs/operation/creds/id-token.md)) instead of storing long-lived secrets. For general credential management — @@ -38,6 +38,15 @@ Vault, AWS Secrets Manager, and other cluster-wide credential managers — see [:octicons-arrow-right-24: View guide](idtoken-azure.md) +- :material-google-cloud:{ .lg .middle } __Authenticating to GCP via IDToken__ + + --- + + Impersonate a GCP service account from a pipeline using workload + identity federation. + + [:octicons-arrow-right-24: View guide](idtoken-gcp.md) +
For details on JWT claims, key rotation, and the `subject_scope` setting, see the From 307bdfa69631955c2a700aae670b6364fbd67284 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 23:00:20 -0400 Subject: [PATCH 7/9] docx: consolidate time triggered examples --- base.yml | 1 - .../time-triggered-pipelines.md | 120 ---------------- .../pipeline-patterns/time-triggered.md | 133 ++++++++++++++++-- 3 files changed, 124 insertions(+), 130 deletions(-) delete mode 100644 docs/examples/pipeline-patterns/time-triggered-pipelines.md diff --git a/base.yml b/base.yml index c9bdb485f..107beadaa 100644 --- a/base.yml +++ b/base.yml @@ -249,7 +249,6 @@ nav: - examples/pipeline-patterns/common-pipeline.md - examples/pipeline-patterns/task-inputs-outputs.md - examples/pipeline-patterns/gated-pipelines.md - - examples/pipeline-patterns/time-triggered-pipelines.md - examples/pipeline-patterns/manual-approval.md - Git Workflows: - examples/git-workflows/index.md diff --git a/docs/examples/pipeline-patterns/time-triggered-pipelines.md b/docs/examples/pipeline-patterns/time-triggered-pipelines.md deleted file mode 100644 index 94baf1bc7..000000000 --- a/docs/examples/pipeline-patterns/time-triggered-pipelines.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -title: Time Triggered Pipeline Patterns ---- - -The [time resource](https://github.com/concourse/time-resource/) produces a -new [version](../../docs/getting-started/resources.md#versions) for the time interval that was declared in its definition in -the pipeline configuration file. - -The two most common usages are having the time resource trigger on an interval: - -```yaml -resources: - - name: trigger-every-3-minutes - type: time - source: - interval: 3m -``` - -Or trigger once within a certain time range: - -```yaml -resources: - - name: trigger-daily-between-1am-and-2am - type: time - source: - start: 1:00 AM - stop: 2:00 AM - location: America/Toronto -``` - -Check the README of the [time resource](https://github.com/concourse/time-resource/) for more details. - -## 1) - Single Time Trigger - -The following is an example of a pipeline that is triggered by a time resource on a pre-determined interval. - -```yaml -resources: - - name: trigger-every-3-minutes - type: time - source: - interval: 3m - -jobs: - - name: run-forrest-run - plan: - - get: trigger-every-3-minutes - trigger: true - # can add other steps to run in this job - - - name: run-bubba-run - plan: - - get: trigger-every-3-minutes - trigger: true - passed: - - run-forrest-run - # can add other steps to run in this job -``` - -![](assets/time-triggered-pipelines-01.png) - -## 2) - Multiple Time Triggers - -As an enhancement to the previous sample with a single time trigger, this pipeline example implements two time resource -triggers and the ability to manually kick it off outside the time resources schedules. - -The first time you set up a pipeline like this you will need to manually trigger it in order to satisfy the passed -constraint of the `manual-trigger` resource. Once one version is available that satisfies the passed constraint all -future triggers by the other resources will work as expected. - -```yaml -resources: - - name: trigger-every-4-minutes - type: time - source: - interval: 4m - - name: trigger-every-10-minutes - type: time - source: - interval: 10m - - name: manual-trigger - type: time - source: - interval: 1m - -jobs: - - name: manual-trigger - plan: - - put: manual-trigger - - - name: run-forrest-run - plan: - - get: trigger-every-4-minutes - trigger: true - - get: trigger-every-10-minutes - trigger: true - - get: manual-trigger - trigger: true - passed: - - manual-trigger - # can add other steps to run in this job - - - name: run-bubba-run - plan: - - get: trigger-every-4-minutes - trigger: true - passed: - - run-forrest-run - - get: trigger-every-10-minutes - trigger: true - passed: - - run-forrest-run - - get: manual-trigger - trigger: true - passed: - - run-forrest-run - # can add other steps to run in this job -``` - -![](assets/time-triggered-pipelines-02.png) \ No newline at end of file diff --git a/docs/examples/pipeline-patterns/time-triggered.md b/docs/examples/pipeline-patterns/time-triggered.md index 29be3f707..75f6b0d19 100644 --- a/docs/examples/pipeline-patterns/time-triggered.md +++ b/docs/examples/pipeline-patterns/time-triggered.md @@ -1,14 +1,36 @@ --- -title: time-triggered job example -search: - exclude: true -hide: - - toc +title: Time-Triggered Pipelines --- -# `time`-triggered job example +The [`time` resource](https://github.com/concourse/time-resource) produces a new +[version](../../docs/getting-started/resources.md#versions) for the time interval declared in its definition, and +can be used to trigger a job on that schedule. -The [`time` resource](https://github.com/concourse/time-resource) can be used to trigger a job. +The two most common configurations are triggering on a fixed interval: + +```yaml +resources: + - name: trigger-every-3-minutes + type: time + source: + interval: 3m +``` + +or triggering once within a time window: + +```yaml +resources: + - name: trigger-daily-between-1am-and-2am + type: time + source: + start: 1:00 AM + stop: 2:00 AM + location: America/Toronto +``` + +Check the README of the [time resource](https://github.com/concourse/time-resource/) for the full set of options. + +See it running:
@@ -17,15 +39,108 @@ The [`time` resource](https://github.com/concourse/time-resource) can be used to
-## Pipeline Configuration +## Minimal Time-Triggered Pipeline + +The simplest possible version: one `time` resource on an interval, and one job that triggers off it. This is the +exact pipeline deployed above: ```yaml linenums="1" --8<-- "libs/examples/pipelines/time-triggered.yml" ``` +## Chaining Jobs off a Shared Trigger + +A common next step is having a second job run after the first, using the same time resource as its trigger and a +`passed` constraint to enforce ordering: + +```yaml +resources: + - name: trigger-every-3-minutes + type: time + source: + interval: 3m + +jobs: + - name: run-forrest-run + plan: + - get: trigger-every-3-minutes + trigger: true + # can add other steps to run in this job + + - name: run-bubba-run + plan: + - get: trigger-every-3-minutes + trigger: true + passed: + - run-forrest-run + # can add other steps to run in this job +``` + +![](assets/time-triggered-pipelines-01.png) + +## Multiple Time Triggers + +As an enhancement to the job-chaining pattern above, this example implements two time resource triggers plus the ability +to manually kick off the pipeline outside the time resources' schedules. + +The first time you set up a pipeline like this, you'll need to manually trigger it in order to satisfy the `passed` +constraint of the `manual-trigger` resource. Once one version is available that satisfies the constraint, all future +triggers by the other resources will work as expected. + +```yaml +resources: + - name: trigger-every-4-minutes + type: time + source: + interval: 4m + - name: trigger-every-10-minutes + type: time + source: + interval: 10m + - name: manual-trigger + type: time + source: + interval: 1m + +jobs: + - name: manual-trigger + plan: + - put: manual-trigger + + - name: run-forrest-run + plan: + - get: trigger-every-4-minutes + trigger: true + - get: trigger-every-10-minutes + trigger: true + - get: manual-trigger + trigger: true + passed: + - manual-trigger + # can add other steps to run in this job + + - name: run-bubba-run + plan: + - get: trigger-every-4-minutes + trigger: true + passed: + - run-forrest-run + - get: trigger-every-10-minutes + trigger: true + passed: + - run-forrest-run + - get: manual-trigger + trigger: true + passed: + - run-forrest-run + # can add other steps to run in this job +``` + +![](assets/time-triggered-pipelines-02.png) + ## References * [Resources](../../docs/resources/index.md) * [Jobs](../../docs/jobs.md) * [Steps](../../docs/steps/index.md) -* [Tasks](../../docs/tasks.md) +* [Tasks](../../docs/tasks.md) \ No newline at end of file From 27c864abd8dc36580c3d0e6478925c0e07280eec Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 23:04:43 -0400 Subject: [PATCH 8/9] other: add bullet for GCP on the idtoken page --- docs/docs/operation/creds/id-token.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/operation/creds/id-token.md b/docs/docs/operation/creds/id-token.md index 8e5ede513..5cdf8a046 100644 --- a/docs/docs/operation/creds/id-token.md +++ b/docs/docs/operation/creds/id-token.md @@ -179,3 +179,4 @@ For end-to-end setup with a specific provider, see: - [Authenticating to Vault via IDToken](../../../examples/credentials/idtoken-vault.md) - [Authenticating to AWS via IDToken](../../../examples/credentials/idtoken-aws.md) - [Authenticating to Azure via IDToken](../../../examples/credentials/idtoken-azure.md) +- [Authenticating to GCP via IDToken](../../../examples/credentials/idtoken-gcp.md) From 226da82349e6e099f27408b1f94a1dbe9886a0d7 Mon Sep 17 00:00:00 2001 From: Kevin Bimonte Date: Sat, 29 Aug 2026 23:22:06 -0400 Subject: [PATCH 9/9] other: little organization --- base.yml | 12 ++++++------ docs/examples/git-workflows/git-triggered.md | 6 ++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/base.yml b/base.yml index 107beadaa..e91a4f405 100644 --- a/base.yml +++ b/base.yml @@ -241,21 +241,21 @@ nav: - examples/fundamentals/set-pipeline.md - Pipeline Patterns: - examples/pipeline-patterns/index.md - - examples/pipeline-patterns/time-triggered.md - - examples/pipeline-patterns/serial-job.md - - examples/pipeline-patterns/pipeline-vars.md - - examples/pipeline-patterns/manually-triggered.md - - examples/pipeline-patterns/managing-pipeline-configs.md - examples/pipeline-patterns/common-pipeline.md + - examples/pipeline-patterns/managing-pipeline-configs.md - examples/pipeline-patterns/task-inputs-outputs.md - examples/pipeline-patterns/gated-pipelines.md - examples/pipeline-patterns/manual-approval.md + - examples/pipeline-patterns/time-triggered.md + - examples/pipeline-patterns/serial-job.md + - examples/pipeline-patterns/pipeline-vars.md + - examples/pipeline-patterns/manually-triggered.md - Git Workflows: - examples/git-workflows/index.md - - examples/git-workflows/git-triggered.md - examples/git-workflows/basic.md - examples/git-workflows/multi-branch.md - examples/git-workflows/monorepo.md + - examples/git-workflows/git-triggered.md - Container Images: - examples/container-images/index.md - examples/container-images/build-push.md diff --git a/docs/examples/git-workflows/git-triggered.md b/docs/examples/git-workflows/git-triggered.md index 21fe71b77..80a65fb25 100644 --- a/docs/examples/git-workflows/git-triggered.md +++ b/docs/examples/git-workflows/git-triggered.md @@ -1,12 +1,12 @@ --- -title: git-triggered job example +title: Git Triggered Job search: exclude: true hide: - toc --- -# `git`-triggered job example +# Git Triggered Job The [`git` resource](https://github.com/concourse/git-resource) can be used to trigger a job. @@ -19,8 +19,6 @@ The [`git` resource](https://github.com/concourse/git-resource) can be used to t ## Pipeline Configuration - - ```yaml linenums="1" --8<-- "libs/examples/pipelines/git-triggered.yml" ```