From 7bee2bf32562a0eaddbb94e29a56e6c8b438dda8 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Mon, 17 Aug 2026 16:05:46 +0545 Subject: [PATCH 1/4] feat(celery): add celery beat for scheduling new cronjobs - Introduce django-celery-beat - main/cronjobs.py: SCHEDULES registry, kombu queues (default/heavy/cronjob) feeding task_queues, per-job Sentry monitor config, and a beat_init hook that prunes PeriodicTask rows for removed jobs ("manual:" rows spared) - seed jobs: clear_expired_django_sessions (django_session was never pruned) and a per-queue celery_queue_uptime_check heartbeat - main/sentry.py: SentryConfig dataclass, every option passed to sentry_sdk.init() by name so an unrecognised key cannot reach it - SENTRY_DEBUG decouples Sentry verbosity from DJANGO_DEBUG - misc/dev: worker and beat entrypoints with a broker wait - tests in main/test_cronjobs.py, a django check for SCHEDULES task paths, and docs/cronjobs.md --- README.md | 7 +- api/management/commands/run_celery_dev.py | 5 +- api/management/commands/run_celery_prod.py | 4 +- api/tasks.py | 19 + .../templates/config/configmap.yaml | 59 ++++ deploy/helm/ifrcgo-helm/values.yaml | 333 ++++++++++++++++++ docker-compose.yml | 14 +- docs/cronjobs.md | 97 +++++ lang/tasks.py | 8 +- main/celery.py | 25 +- main/checks.py | 14 + main/cronjobs.py | 199 +++++++++++ main/lock.py | 9 +- main/sentry.py | 58 ++- main/settings.py | 41 ++- main/test_cronjobs.py | 123 +++++++ misc/dev/run_worker.sh | 7 + misc/dev/run_worker_beat.sh | 5 + misc/wait-for-broker.sh | 16 + pyproject.toml | 4 +- uv.lock | 51 ++- 21 files changed, 1036 insertions(+), 62 deletions(-) create mode 100644 deploy/helm/ifrcgo-helm/templates/config/configmap.yaml create mode 100644 deploy/helm/ifrcgo-helm/values.yaml create mode 100644 docs/cronjobs.md create mode 100644 main/cronjobs.py create mode 100644 main/test_cronjobs.py create mode 100755 misc/dev/run_worker.sh create mode 100755 misc/dev/run_worker_beat.sh create mode 100755 misc/wait-for-broker.sh diff --git a/README.md b/README.md index 4f546a374..d5cb9a4d3 100644 --- a/README.md +++ b/README.md @@ -190,9 +190,14 @@ For updating the index ```(bash) docker-compose exec serve bash python manage.py update_index ``` +## Cronjobs + +For more info checkout [Cronjobs](./docs/cronjobs.md) + ## Sentry -For updating the cron monitored tasks +For updating the cron monitored tasks (legacy k8s CronJobs only — celery beat +cronjobs are registered automatically) ```(bash) docker-compose exec serve bash ./manage.py cron_job_monitor ``` diff --git a/api/management/commands/run_celery_dev.py b/api/management/commands/run_celery_dev.py index a65a04a51..a0300d4f0 100644 --- a/api/management/commands/run_celery_dev.py +++ b/api/management/commands/run_celery_dev.py @@ -4,10 +4,7 @@ from django.core.management.base import BaseCommand from django.utils.autoreload import run_with_reloader -from main.celery import Queues - -all_queues = ",".join([q for q in Queues.DEV_QUEUES]) -CMD = f"celery -A main worker -Q {all_queues} --concurrency=2 -l info" +CMD = "celery -A main worker -E --concurrency=2 -l info" def restart_celery(): diff --git a/api/management/commands/run_celery_prod.py b/api/management/commands/run_celery_prod.py index 7278ec322..28dded685 100644 --- a/api/management/commands/run_celery_prod.py +++ b/api/management/commands/run_celery_prod.py @@ -5,9 +5,9 @@ from django.core.management.base import BaseCommand -from main.celery import Queues +from main.cronjobs import CeleryQueue -all_queues = ",".join([q for q in Queues.DEV_QUEUES]) +all_queues = ",".join([q.name for q in CeleryQueue.ALL_QUEUE]) # NOTE: Use a fixed concurrency to prevent the pod from being OOMKilled, # as Celery defaults to one worker per available CPU. diff --git a/api/tasks.py b/api/tasks.py index 411473b64..bad694ef8 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -2,10 +2,12 @@ from celery import shared_task from django.contrib.auth.models import User +from django.core import management from django.utils import timezone from rest_framework.authtoken.models import Token from api.playwright import render_pdf_from_url +from main.lock import RedisLockKey, redis_lock from main.utils import logger_context from .logger import logger @@ -59,3 +61,20 @@ def generate_export_pdf(export_id, title, set_user_language="en"): export.status = Export.ExportStatus.ERRORED export.save(update_fields=["status"]) logger.info(f"End export: {export.pk}") + + +# TODO(susilnem): Do we need this cron? +@shared_task +def clear_expired_django_sessions(): + """Purge expired django_session rows -- nothing else prunes them.""" + with redis_lock(RedisLockKey.CLEAR_EXPIRED_DJANGO_SESSIONS) as acquired: + if not acquired: + logger.warning("clear_expired_django_sessions: already running, skipping") + return + management.call_command("clearsessions", verbosity=0) + + +@shared_task +def celery_queue_uptime_check(queue: str) -> None: + """No-op heartbeat proving beat dispatches and that `queue` has a consumer.""" + logger.info("Celery queue '%s' is taking tasks", queue) diff --git a/deploy/helm/ifrcgo-helm/templates/config/configmap.yaml b/deploy/helm/ifrcgo-helm/templates/config/configmap.yaml new file mode 100644 index 000000000..ac95b4d68 --- /dev/null +++ b/deploy/helm/ifrcgo-helm/templates/config/configmap.yaml @@ -0,0 +1,59 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ template "ifrcgo-helm.fullname" . }}-api-configmap + labels: + component: api-deployment + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +data: + # Redis + {{- if .Values.redis.enabled }} + CELERY_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0" + CACHE_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/1" + {{- else }} + CELERY_REDIS_URL: {{ required "env.CELERY_REDIS_URL" .Values.env.CELERY_REDIS_URL | quote }} + CACHE_REDIS_URL: {{ required "env.CACHE_REDIS_URL" .Values.env.CACHE_REDIS_URL | quote }} + {{- end }} + + {{- if .Values.minio.enabled }} + AWS_S3_ENABLED: "true" + {{- else }} + AZURE_STORAGE_ENABLED: "true" + {{- end }} + + {{- if .Values.playwright.enabled }} + PLAYWRIGHT_SERVER_URL: "ws://{{ template "ifrcgo-helm.fullname" . }}-playwright:{{ .Values.playwright.containerPort }}/" + {{- else }} + PLAYWRIGHT_SERVER_URL: {{ required "env.PLAYWRIGHT_SERVER_URL" .Values.env.PLAYWRIGHT_SERVER_URL | quote }} + {{- end }} + + CACHE_MIDDLEWARE_SECONDS: {{ .Values.env.CACHE_MIDDLEWARE_SECONDS | quote }} + DJANGO_DEBUG: {{ .Values.env.DJANGO_DEBUG | quote }} + ELASTIC_SEARCH_HOST: {{ default (printf "elasticsearch://%s-elasticsearch:9200" (include "ifrcgo-helm.fullname" .)) .Values.env.ELASTIC_SEARCH_HOST | quote }} + ELASTIC_SEARCH_INDEX: {{ .Values.env.ELASTIC_SEARCH_INDEX | quote }} + DOCKER_HOST_IP: {{ .Values.env.DOCKER_HOST_IP | quote }} + DJANGO_ADDITIONAL_ALLOWED_HOSTS: {{ .Values.env.DJANGO_ADDITIONAL_ALLOWED_HOSTS | quote }} + ADDITIONAL_TRUSTED_ORIGINS: {{ .Values.env.ADDITIONAL_TRUSTED_ORIGINS | quote }} + SESSION_COOKIE_DOMAIN: {{ .Values.env.SESSION_COOKIE_DOMAIN | quote }} + CSRF_COOKIE_DOMAIN: {{ .Values.env.CSRF_COOKIE_DOMAIN | quote }} + GO_ENVIRONMENT: {{ .Values.env.GO_ENVIRONMENT | quote }} + API_FQDN: {{ .Values.env.API_FQDN | quote }} + FRONTEND_URL: {{ .Values.env.FRONTEND_URL | quote }} + DEBUG_EMAIL: {{ .Values.env.DEBUG_EMAIL | quote }} + IFRC_TRANSLATION_DOMAIN: {{ .Values.env.IFRC_TRANSLATION_DOMAIN | quote }} + AUTO_TRANSLATION_TRANSLATOR: {{ .Values.env.AUTO_TRANSLATION_TRANSLATOR | quote }} + DJANGO_READ_ONLY: {{ .Values.env.DJANGO_READ_ONLY | quote }} + SENTRY_SAMPLE_RATE: {{ .Values.env.SENTRY_SAMPLE_RATE | quote }} + SENTRY_DEBUG: {{ .Values.env.SENTRY_DEBUG | quote }} + SENTRY_DSN: {{ .Values.env.SENTRY_DSN | quote }} + SENTRY_MONITOR_CELERY_BEAT_TASKS: {{ .Values.env.SENTRY_MONITOR_CELERY_BEAT_TASKS | quote }} + OIDC_ENABLE: {{ .Values.env.OIDC_ENABLE | quote }} + + EOAPI_STAC_EXTERNAL_URL: {{ .Values.env.EOAPI_STAC_EXTERNAL_URL | quote }} + EOAPI_STAC_INTERNAL_URL: {{ .Values.env.EOAPI_STAC_INTERNAL_URL | quote }} + + # Additional configs + {{- range $name, $value := .Values.envAdditional }} + {{ $name }}: {{ $value | quote }} + {{- end }} diff --git a/deploy/helm/ifrcgo-helm/values.yaml b/deploy/helm/ifrcgo-helm/values.yaml new file mode 100644 index 000000000..f6543aeb3 --- /dev/null +++ b/deploy/helm/ifrcgo-helm/values.yaml @@ -0,0 +1,333 @@ +environment: dev + +env: + DJANGO_SECRET_KEY: '' + DJANGO_DB_NAME: '' + DJANGO_DB_USER: '' + DJANGO_DB_PASS: '' + DJANGO_DB_HOST: '' + DJANGO_DB_PORT: '' + AZURE_STORAGE_ACCOUNT: '' + AZURE_STORAGE_KEY: '' + EMAIL_API_ENDPOINT: '' + EMAIL_HOST: '' + EMAIL_PORT: '' + EMAIL_USER: '' + EMAIL_PASS: '' + TEST_EMAILS: '' + AWS_TRANSLATE_ACCESS_KEY: '' + AWS_TRANSLATE_SECRET_KEY: '' + AWS_TRANSLATE_REGION: '' + CELERY_REDIS_URL: '' + CACHE_MIDDLEWARE_SECONDS: '' + MOLNIX_API_BASE: '' + MOLNIX_USERNAME: '' + MOLNIX_PASSWORD: '' + ERP_API_ENDPOINT: '' + ERP_API_SUBSCRIPTION_KEY: '' + FDRS_CREDENTIAL: '' + HPC_CREDENTIAL: '' + APPLICATION_INSIGHTS_INSTRUMENTATION_KEY: '' + ELASTIC_SEARCH_HOST: '' + ELASTIC_SEARCH_INDEX: 'new_index' + GO_FTPHOST: '' + GO_FTPUSER: '' + GO_FTPPASS: '' + GO_DBPASS: '' + APPEALS_USER: '' + APPEALS_PASS: '' + DJANGO_DEBUG: '' + DOCKER_HOST_IP: '' + DJANGO_ADDITIONAL_ALLOWED_HOSTS: '' + ADDITIONAL_TRUSTED_ORIGINS: '' + SESSION_COOKIE_DOMAIN: '' + CSRF_COOKIE_DOMAIN: '' + GO_ENVIRONMENT: '' + API_FQDN: '' + FRONTEND_URL: '' + DEBUG_EMAIL: '' + SENTRY_DEBUG: false + SENTRY_DSN: '' + SENTRY_MONITOR_CELERY_BEAT_TASKS: true + SENTRY_SAMPLE_RATE: '' + DJANGO_READ_ONLY: '' + AUTO_TRANSLATION_TRANSLATOR: '' + IFRC_TRANSLATION_DOMAIN: '' + IFRC_TRANSLATION_HEADER_API_KEY: '' + FDRS_APIKEY: '' + NS_CONTACT_USERNAME: '' + NS_CONTACT_PASSWORD: '' + ACAPS_API_TOKEN: '' + NS_DOCUMENT_API_KEY: '' + NS_INITIATIVES_API_KEY: '' + NS_DOCUMENT_API_TOKEN: '' + JWT_PRIVATE_KEY_BASE64_ENCODED: '' + JWT_PUBLIC_KEY_BASE64_ENCODED: '' + JWT_EXPIRE_TIMESTAMP_DAYS: '' + AZURE_OPENAI_DEPLOYMENT_NAME: '' + AZURE_OPENAI_ENDPOINT: '' + AZURE_OPENAI_API_KEY: '' + OIDC_ENABLE: false + OIDC_RSA_PRIVATE_KEY_BASE64_ENCODED: + OIDC_RSA_PUBLIC_KEY_BASE64_ENCODED: + RELIEF_WEB_APP_NAME: '' + POWERBI_WORKSPACE_ID: '' + POWERBI_DATASET_IDS: '' + +# NOTE: Used to pass additional configs to api/worker containers +# NOTE: Not used by azure vault +envAdditional: + # Additional configs + # EXAMPLE: MY_CONFIG: "my-value" + +secrets: + API_TLS_CRT: '' + API_TLS_KEY: '' + API_ADDITIONAL_DOMAIN_TLS_CRT: '' + API_ADDITIONAL_DOMAIN_TLS_KEY: '' + +# NOTE: Used to pass additional secrets to api/worker containers +# NOTE: Not used by azure vault +secretsAdditional: + # Additional secrets + # EXAMPLE: MY_SECRET: "my-secret-value" + +redis: + enabled: true + architecture: standalone + fullnameOverride: go-redis + auth: + enabled: false + master: + persistence: + enabled: true + size: 1Gi + resources: + requests: + cpu: "0.5" + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + resources: + requests: + cpu: "0.5" + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + +# https://artifacthub.io/packages/helm/bitnami/minio +# extraEnvVars: https://github.com/bitnami/containers/blob/main/bitnami/minio/README.md#environment-variables +minio: + enabled: false # XXX: Used for alpha instances running outside Azure + disableWebUI: true + mode: standalone + fullnameOverride: go-minio + global: + defaultStorageClass: + apiIngress: + enabled: true + ingressClassName: + hostname: + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "50m" + auth: + forceNewKeys: True + rootUser: go + rootPassword: + persistence: + enabled: true + size: 1Gi + defaultBuckets: go-data,go-static + provisioning: + enabled: true + resourcesPreset: "nano" + cleanupAfterFinished: + enabled: true + extraCommands: + - "mc anonymous set download provisioning/go-static" + +postgresql: + enabled: false # XXX: Used for alpha instances running outside Azure + fullnameOverride: "go-pg" + architecture: standalone + primary: + persistence: + enabled: true + size: 8Gi + +playwright: + enabled: rue + replicaCount: 1 + containerPort: 3000 + image: + # NOTE: Make sure this matches with pyproject playwright dependency and root docker-compose + name: 'mcr.microsoft.com/playwright' + tag: 'v1.50.0-noble' + pullPolicy: 'IfNotPresent' + resources: + requests: + cpu: "0.1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +api: + domain: "go-staging.ifrc.org" + tls: + enabled: true + additionalDomain: "" + enabled: true + replicaCount: 1 + containerPort: 80 + image: + name: 'SET-BY-CICD-IMAGE' + tag: 'SET-BY-CICD-TAG' + pullPolicy: 'IfNotPresent' + resources: + requests: + cpu: "2" + memory: 4Gi + limits: + cpu: "2" + memory: 4Gi + +celery: + enabled: true + resources: + requests: + cpu: "0.5" + memory: 0.4Gi + limits: + cpu: "2" + memory: 4Gi + +argoHooksEnabled: false # FIXME: Remove this after go-api is moved to argoCD pipeline +argoHooks: + # NOTE: Make sure keys are lowercase + database-migration: + enabled: true + hook: PostSync + preserveHistory: true + command: ["./manage.py", "migrate"] + collect-static: + enabled: true + hook: PostSync + command: ["./manage.py", "collectstatic", "--noinput"] + +cronjobsDefaultResources: + requests: + cpu: 0.1 + memory: 1Gi + limits: + cpu: 4 + memory: 2Gi + +cronjobs: + - command: 'index_and_notify' + schedule: '*/5 * * * *' + - command: 'sync_molnix' + schedule: '10 */2 * * *' + resources: + requests: + memory: 1Gi + cpu: 0.1 + limits: + memory: 2Gi + cpu: 4 + - command: 'ingest_appeals' + schedule: '*/30 * * * *' + - command: 'sync_appealdocs' + schedule: '15 * * * *' + - command: 'revoke_staff_status' + schedule: '51 * * * *' + - command: 'update_project_status' + schedule: '1 3 * * *' + - command: 'user_registration_reminder' + schedule: '0 9 * * *' + - command: 'ingest_country_plan_file' + schedule: '1 0 * * *' + - command: 'fdrs_annual_income' + schedule: '0 0 * * 0' + - command: 'FDRS_INCOME' + schedule: '0 0 * * 0' + - command: 'ingest_acaps' + schedule: '0 1 * * 0' + - command: 'ingest_climate' + schedule: '0 0 * * 0' + - command: 'ingest_databank' + schedule: '0 0 * * 0' + - command: 'ingest_hdr' + schedule: '0 0 * * 0' + - command: 'ingest_unicef' + schedule: '0 0 * * 0' + - command: 'ingest_worldbank' + schedule: '0 2 * * 0' + - command: 'ingest_disaster_law' + schedule: '0 0 * * 0' + - command: 'ingest_ns_contact' + schedule: '0 0 * * 0' + - command: 'ingest_ns_capacity' + schedule: '0 0 * * 0' + - command: 'ingest_ns_directory' + schedule: '0 0 * * 0' + - command: 'ingest_ns_document' + schedule: '0 0 * * 0' + - command: 'ingest_ns_initiatives' + schedule: '0 0 * * 0' + - command: 'ingest_icrc' + schedule: '0 3 * * 0' + - command: 'notify_validators' + schedule: '0 0 * * *' + - command: 'poll_gdacs_cyclone' + schedule: '0 11 * * 0' + - command: 'poll_gdacs_flood' + schedule: '0 11 * * *' + - command: 'poll_usgs_earthquake' + schedule: '0 18 * * 0' + - command: 'alert_notification' + schedule: '0 */2 * * *' + # https://github.com/jazzband/django-oauth-toolkit/blob/master/docs/management_commands.rst#cleartokens + - command: 'oauth_cleartokens' + schedule: '0 1 * * *' + - command: 'eap_submission_reminder' + schedule: '0 0 * * *' + + +elasticsearch: + enabled: true + httpPort: 9200 + transportPort: 9300 + elasticsearchVersion: 7.16.2 + storageSize: 20Gi + storageClassName: managed-premium # FIXME: populate and use + disk: + name: my-disk-name + uri: https://mydisk.blob.core.windows.net/mycontainer/mydisk.vhd + resources: + requests: + cpu: "2" + memory: 4Gi + limits: + cpu: "2" + memory: 4Gi + +sshBastion: + enabled: true + resources: + requests: + cpu: "0.5" + memory: 0.5Gi + limits: + cpu: "1" + memory: 1Gi + keys: + zoltan.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPGAnkQdf5CIpVoqNVJ17AAzUb02gpTltJI5q5SRKxl8 zol@hp + daniel.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDU1XLLPq1J4kFvNyg5eUK8uuW8dtW1f3ALVnYr0nVhldxF0J59XtZbNFBLCVHYZL3NQxYQrucll6LbGaMGKbGsTwtqcxqd2fWlhg7nBnvhOzULYbAru3YfpkgnawGin6Y7qW/MQ3fYmqqm8MB7p5+G4sIL76S2yWbi7lcKWnd87yDTGEEoc8H6i6IwNNVHudvuMA4MzGkSgql7gIC2KuU+s2u9Y6fmE92G39BO454SUgAcCJfhuXukZhU4UN3RVYy+F0MxVeLc0hEJi4sCYcoPKREc0//srNyni7b8G8C+z6t02xrzhWwIORlb8Jr2kmbblp7PFMz4r2qRd8MvXAa5ta6kUvMDg0t52JaDMAGy0IjGZh9PznXbp1LYn7uS5NQh4C/t6Q3TXyJbEiaQaObcmjn6w/DWH6gI7ZRYkPGdlctlNm5MWnhjG9Q/FzRIxvaauSFqgs6bfIUGGaY9i1eNiowVSzDPlP7nH0gJpq+uS5Qdyg69m/XH1DqywPoZY7U= ifrcds\daniel.tovari@5CG41911RW + arun.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIERqaO+XlqTbvoh88Kuj9c377x77NChWhNP8VpbM1/hf ifrcds\arun.gandhi@5CG1355NPN + thenav56.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN/f/A3qkaTHSdbKn8Hv75YiJvRMEXvWTDdIiR7tyAjJ navin@nav-machine + david.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3FzrQdVh5Qwp5Y6KQGcpqHxKErxCW103iEECuutR/jBZe6X0xjD+cW7e+H8SrUsPQwj87fzOsMAc6v6n+3hdYFa6ekgRG/USEIUR5C/GD1Xjva3Xpp45PasBhJEtYt2ON+dlzwvRyOuv2hvqv2WHBO020ewIlVuQ4pU4Qj5ysvwWGj8GAv/jITiVERmjLTStbFwxeIDT3jQEbwnfV1zZZKiGxIecB/y51nk6oIQ00ZGrYEo5ieWsUSVfLHOX0/lZ0mtrdqxDEgMaCbNaUbICAimsJPamNpoirKc7FoKIKKrLQsK8qE1lClWQEecbW+dgSiwxracooKeWhHq+BkKUCNgEL/C0ff2l9e8sJcLmYZUdPtDCdtUDC8BAlELA5HR6tdCTfFcc0nXltclSSODMnZkQohh5/2fixJTwN5p5csEfBLzbdrturKtT/TbYSoaodg4muPqY4YE5jiJfrHVAGS1DVWz/cRcm1vOxT2V4iW2SNvo8fS2PZOpU5furrvbM= ifrcds\david.muchatiza@5CG41911S1 + paola.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGql4RrbxSQTW5QrTh+P+94jGCXOCeZgc23hxL9zFCYQrzL0SMw1F53Z5SFZimIhJswYPqV2pT8L4oTRqIrTCM+looWi7b9/9u+m/KmA+FWbo3u6uRrckkA3nVIKsKHvlOucX2GxE6i+tXdeXEisW49ZpMtuvxMLJ3Eg4MK10d/2d3FKuzTsrxCTlJn8FAE3yOsVow0jdu+381IrkAqRE2GINeQ87hVlQpbo+bL2N/2QZmNjDhBBQkRJLDisW0+UNgo+S9wN7HbpV5LheSJS9wGN7LlmcqlpZFrDO/lVyoMxEQ0588wUI8BVfqAZDEBJPdGtzq513r+5iXEX/9A1Mendlvxfl6ANNRcH9PVZHkRN1dxY3rckQ+Lk3qqIjjfYFYvl5Gybidb1BM2VNWHAuzaDDQzJpeTHIbQnDt7Ke4oX2xWYgyu+kVhqz0HnAV28qMXbMEsrMIrtwl7IjcrorgdduHghZvWFbaJZNtXOfgnf1IYNXkZ9eWPS+Bz9nWMhE= ifrcds\paola.yela@5CG41911RT + ranjan.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJA0ec4Gavc+m1MjEZGoUce51yWouMTRTYJZV3s/jgD rsh@rsh-XPS-15-9510 diff --git a/docker-compose.yml b/docker-compose.yml index c8149845a..1caec4b54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,7 +145,19 @@ services: # For development only celery: <<: *base_server_setup - command: python manage.py run_celery_dev + restart: unless-stopped + command: ./misc/dev/run_worker.sh + healthcheck: + test: ["CMD-SHELL", "celery -A main inspect ping -d celery@$$HOSTNAME || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + celery-beat: + <<: *base_server_setup + restart: unless-stopped + command: ./misc/dev/run_worker_beat.sh # ------------------ Helper CLI Commands # Usage: `docker compose run --rm ` diff --git a/docs/cronjobs.md b/docs/cronjobs.md new file mode 100644 index 000000000..b604869fb --- /dev/null +++ b/docs/cronjobs.md @@ -0,0 +1,97 @@ +# Cronjobs + +There are **two** cronjob mechanisms. Use celery beat for new cronjobs. + +A job belongs to one mechanism or the other, **never both** — +`SentryMonitor.validate_config()` asserts that the enum matches `values.yaml`, +so mixing them breaks it. + +## 1. Celery beat — use this for new cronjobs + +Schedules are declared in [`main/cronjobs.py`](../main/cronjobs.py) and synced +into `django_celery_beat` `PeriodicTask` rows when beat starts. `SCHEDULES` is +the source of truth: remove an entry and its row is deleted on the next start. +Rows named `manual:*` are left alone, as an escape hatch for one-off tasks +created through the admin. + +Adding one takes two files, with no helm change and no `cron_job_monitor` run: + +**1. Write the task in `/tasks.py`** + +```python +@shared_task(soft_time_limit=..., time_limit=...) +def my_new_job(): + with redis_lock(RedisLockKey.MY_NEW_JOB) as acquired: + if not acquired: + return + ... +``` + +- The lock matters: `CELERY_ACKS_LATE` is on, so a task can be redelivered to + another worker if the one running it dies. +- Time limits go **on the decorator**. `DatabaseScheduler` silently discards + `time_limit` / `soft_time_limit` from a schedule entry's options. +- Don't set `queue` here — it belongs in the schedule entry below. + +**2. Add a `CronJob` entry to `SCHEDULES` in `main/cronjobs.py`** + +```python +"my_new_job": CronJob( + task="myapp.tasks.my_new_job", + schedule=TimeConstants.EVERY_DAY, + options=CronJobOption( + expire_seconds=TimeConstants.SECONDS_IN_A_DAY, + queue=CeleryQueue.cronjob.name, + ), + sentry_config=CronJobSentryConfig(max_runtime=10), +), +``` + +`options` only supports the keys `ModelEntry._unpack_options` keeps — `queue`, +`exchange`, `routing_key`, `priority`, `headers`, `expire_seconds`. Anything +else is dropped without warning. `expire_seconds` stops a backlog accumulating +while workers are down. + +Sentry cron monitoring is automatic, controlled by +`SENTRY_MONITOR_CELERY_BEAT_TASKS` (default on). `CronJobSentryConfig` sets each +job's grace period, max runtime and thresholds next to its schedule. + +### Queues + +`CeleryQueue` in `main/cronjobs.py` declares which queues exist (`default`, +`heavy`, `cronjob`) and feeds `app.conf.task_queues`. A worker started without +`-Q` consumes all of them, which is the dev setup. + +A queue that is routed to but not declared here is a black hole: the task is +accepted and then never consumed by anything. + +### Running locally + +```bash +docker-compose up celery celery-beat +``` + +Worker and beat entrypoints live in `misc/dev/`. + +### Not deployed yet + +**Beat currently runs in local development only.** There is no beat Deployment +in `deploy/helm/`, so nothing in `SCHEDULES` fires in alpha/staging/prod until +one is added. Still to do: + +- A beat Deployment with **`replicas: 1`** and `strategy: Recreate` — two beat + processes fire every cronjob twice — plus a `celeryBeat` block in + `values.yaml`. It needs the same `envFrom` secret + configmap as the celery + worker. +- Beat needs the `django_celery_beat` tables, which `manage.py migrate` creates + on the API pod. If beat starts first it crashloops until migrations have run. + +## 2. Kubernetes CronJobs — the legacy set + +The pre-existing cronjobs run as k8s CronJob resources listed under `cronjobs:` +in `deploy/helm/ifrcgo-helm/values.yaml`, one pod per run, monitored via +`SentryMonitor` in `main/sentry.py`. Their Sentry monitors are registered with: + +```bash +docker-compose exec serve bash ./manage.py cron_job_monitor +``` diff --git a/lang/tasks.py b/lang/tasks.py index e27947ffd..bf0481659 100644 --- a/lang/tasks.py +++ b/lang/tasks.py @@ -12,7 +12,7 @@ from modeltranslation.translator import translator from modeltranslation.utils import build_localized_fieldname -from main.celery import Queues +from main.cronjobs import CeleryQueue from main.lock import RedisLockKey, redis_lock from main.translation import ( TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME, @@ -217,7 +217,7 @@ def run(self, batch_size=None, only_models: typing.Optional[typing.List[models.M index += 1 -@shared_task(queue=Queues.CRONJOB) +@shared_task(queue=CeleryQueue.cronjob.name) def translate_remaining_models_fields(): # Disabled in DEBUG/Development if settings.DEBUG: @@ -226,7 +226,7 @@ def translate_remaining_models_fields(): ModelTranslator().run(batch_size=100) -@shared_task(queue=Queues.DEFAULT) +@shared_task(queue=CeleryQueue.default.name) def translate_model_fields(model_name, pk): model = django_apps.get_model(model_name) obj = model.objects.get(pk=pk) @@ -239,7 +239,7 @@ def translate_model_fields(model_name, pk): logger.info(f"Translation success for {model_name} with pk={pk}.") -@shared_task(queue=Queues.HEAVY) +@shared_task(queue=CeleryQueue.heavy.name) def translate_model_fields_in_bulk(model_name, pks): model = django_apps.get_model(model_name) qs = model.objects.filter( diff --git a/main/celery.py b/main/celery.py index 33e4f6956..2309cbe3f 100644 --- a/main/celery.py +++ b/main/celery.py @@ -1,19 +1,17 @@ +import dataclasses import os import celery from banjo_utils.celery_health.worker import setup_worker_heartbeat from django.conf import settings -from main import sentry +from main.cronjobs import BEAT_SCHEDULES, CeleryQueue class CustomCeleryApp(celery.Celery): def on_configure(self): if settings.SENTRY_DSN: - sentry.init_sentry( - app_type="WORKER", - **settings.SENTRY_CONFIG, - ) + dataclasses.replace(settings.SENTRY_CONFIG, app_type="WORKER").init_sentry() # set the default Django settings module for the 'celery' program. @@ -34,19 +32,12 @@ def on_configure(self): app.autodiscover_tasks() -class Queues: - DEFAULT = "default" - HEAVY = "heavy" - CRONJOB = "cronjob" +app.conf.task_default_queue = CeleryQueue.default.name +app.conf.task_queues = CeleryQueue.ALL_QUEUE - DEV_QUEUES = ( - DEFAULT, - HEAVY, - CRONJOB, - ) - - -app.conf.task_default_queue = Queues.DEFAULT +# Cronjobs scheduled through celery beat. See main/cronjobs.py -- note that this +# is separate from the legacy k8s CronJob resources in values.yaml:cronjobs. +app.conf.beat_schedule = BEAT_SCHEDULES @app.task(bind=True) diff --git a/main/checks.py b/main/checks.py index 39c2bf38a..e21c8cfb7 100644 --- a/main/checks.py +++ b/main/checks.py @@ -1,7 +1,21 @@ +from pydoc import locate + from django.conf import settings from django.core.checks import Error, Tags, register +@register(Tags.compatibility) +def celery_beat_tasks(app_configs, **kwargs): + """Catch a typo'd SCHEDULES task path now, not on beat's first tick.""" + from main.cronjobs import SCHEDULES + + errors = [] + for name, config in SCHEDULES.items(): + if locate(config.task) is None: + errors.append(Error(f"Celery beat <{name}> task is incorrect: {config.task}")) + return errors + + @register(Tags.compatibility) def oauth2_check(app_configs, **kwargs): if not settings.OIDC_ENABLE: diff --git a/main/cronjobs.py b/main/cronjobs.py new file mode 100644 index 000000000..334a12d92 --- /dev/null +++ b/main/cronjobs.py @@ -0,0 +1,199 @@ +""" +Registry for cronjobs scheduled by celery beat. For NEW cronjobs only. + +The legacy cronjobs are k8s CronJobs listed in values.yaml:cronjobs and +monitored via main.sentry.SentryMonitor. A job belongs to one mechanism or the +other, never both -- SentryMonitor.validate_config() asserts the enum matches +values.yaml. See docs/cronjobs.md for how to add one. +""" + +import functools +import logging +import operator +import typing + +from celery import signals +from celery.schedules import crontab +from django.conf import settings +from django.db import models +from kombu import Queue +from sentry_sdk.integrations.celery import beat as sentry_celery_beat + +if typing.TYPE_CHECKING: + from celery import Celery + from sentry_sdk._types import MonitorConfig + +logger = logging.getLogger(__name__) + + +class TimeConstants: + SECONDS_IN_A_MINUTE = 60 + SECONDS_IN_A_HOUR = 60 * 60 + SECONDS_IN_A_DAY = 24 * 60 * 60 + SECONDS_IN_A_WEEK = 7 * 24 * 60 * 60 + + EVERY_WEEK = crontab(minute="1", hour="1", day_of_week="1") + EVERY_DAY = crontab(minute="1", hour="1") + EVERY_HOUR = crontab(minute="0", hour="*") + EVERY_2_MINUTES = crontab(minute="*/2") + EVERY_1_MINUTES = crontab(minute="*/1") + + +class CeleryQueue: + # NOTE: Names must be lowercase (used as-is in k8s). + default = Queue("default") + heavy = Queue("heavy") + cronjob = Queue("cronjob") + + # Feeds app.conf.task_queues. A worker started without -Q consumes all of these. + ALL_QUEUE = ( + default, + heavy, + cronjob, + ) + + +class CronJobOption(typing.TypedDict, total=False): + """Per-job options for django_celery_beat. + + WARNING: only queue/exchange/routing_key/priority/headers/expire_seconds + survive ModelEntry._unpack_options; anything else is silently dropped. Put + time_limit/soft_time_limit on the task decorator instead. + """ + + expire_seconds: float + """Task will not run if picked up later than this. Avoids a backlog when workers are down.""" + + queue: str + """Queue the task is sent to. Required -- see test_cronjobs.py.""" + + +class CeleryBeatSchedule(typing.TypedDict): + task: str + schedule: crontab + options: CronJobOption + args: tuple[typing.Any, ...] + + +class CronJobSentryConfig(typing.NamedTuple): + checkin_margin: int = 5 + """Minutes of grace before a missed check-in is flagged.""" + + max_runtime: int = 30 + """Minutes before Sentry considers the job failed.""" + + failure_issue_threshold: int = 1 + """Consecutive failures before an issue is created.""" + + recovery_threshold: int = 1 + """Consecutive successes before an issue is resolved.""" + + def as_dict(self) -> "MonitorConfig": + return { + "checkin_margin": self.checkin_margin, + "max_runtime": self.max_runtime, + "failure_issue_threshold": self.failure_issue_threshold, + "recovery_threshold": self.recovery_threshold, + } + + +class CronJob(typing.NamedTuple): + task: str + schedule: crontab + args: tuple[typing.Any, ...] = () + sentry_config: CronJobSentryConfig = CronJobSentryConfig() + options: CronJobOption = {} + + +# NOTE: Removing an entry here deletes its PeriodicTask row (see update_periodic_tasks). +SCHEDULES: dict[str, CronJob] = { + "clear_expired_django_sessions": CronJob( + task="api.tasks.clear_expired_django_sessions", + schedule=TimeConstants.EVERY_WEEK, + options=CronJobOption(expire_seconds=TimeConstants.SECONDS_IN_A_WEEK), + ), + **{ + f"celery_queue_uptime_{celery_queue.name}": CronJob( + task="api.tasks.celery_queue_uptime_check", + args=(celery_queue.name,), + schedule=TimeConstants.EVERY_HOUR, + options=CronJobOption( + expire_seconds=TimeConstants.SECONDS_IN_A_HOUR, + queue=celery_queue.name, + ), + sentry_config=CronJobSentryConfig( + checkin_margin=10, + max_runtime=2, + failure_issue_threshold=2, + ), + ) + for celery_queue in CeleryQueue.ALL_QUEUE + }, +} + +BEAT_SCHEDULES: dict[str, CeleryBeatSchedule] = { + name: { + "task": config.task, + "args": config.args, + "schedule": config.schedule, + "options": config.options, + } + for name, config in SCHEDULES.items() +} + + +_get_monitor_config = sentry_celery_beat._get_monitor_config + + +class SentryMonkeyPatch: + @staticmethod + def custom__get_monitor_config(celery_schedule: typing.Any, app: "Celery", monitor_name: str) -> "MonitorConfig": + """Get configuration for sentry monitoring. + + https://github.com/getsentry/sentry-python/blob/5715734eac1c5fb4b6ec61ef459080c74fa777b5/sentry_sdk/integrations/celery/beat.py#L59 + """ + config = _get_monitor_config(celery_schedule, app, monitor_name) + job_config = SCHEDULES.get(monitor_name) + if job_config: + config.update(job_config.sentry_config.as_dict()) + return config + + +sentry_celery_beat._get_monitor_config = SentryMonkeyPatch.custom__get_monitor_config + + +def get_obsolete_periodic_task_qs(): + """Our PeriodicTask rows that are no longer in SCHEDULES. + + Scoped to GO_APPS so third-party rows are never touched. `manual:` rows are + spared as an escape hatch for one-off tasks created in the admin. + """ + from django_celery_beat.models import PeriodicTask + + ours = functools.reduce( + operator.or_, + [models.Q(task__startswith=f"{app_name}.") for app_name in settings.GO_APPS], + ) + + return PeriodicTask.objects.filter(ours).exclude(name__in=list(BEAT_SCHEDULES.keys())).exclude(name__startswith="manual:") + + +@signals.beat_init.connect +def update_periodic_tasks(**_): + """Drop rows for cronjobs no longer in SCHEDULES -- git is the source of truth.""" + logger.info("Cronjob sync: Start") + try: + obsolete_tasks_qs = get_obsolete_periodic_task_qs() + + obsolete_task_names = list(obsolete_tasks_qs.values_list("name", flat=True)) + if not obsolete_task_names: + logger.info("Cronjob sync - Obsolete tasks: Nothing to do") + return + + for task_name in obsolete_task_names: + logger.warning("Cronjob sync - Obsolete tasks: Task <%s> will be deleted", task_name) + + deleted_count, _ = obsolete_tasks_qs.delete() + logger.warning("Cronjob sync - Obsolete tasks: Deleted %s tasks", deleted_count) + except Exception: + logger.error("Cronjob sync: Failed to sync PeriodicTasks", exc_info=True) diff --git a/main/lock.py b/main/lock.py index 9de5b7aaf..0de91cf87 100644 --- a/main/lock.py +++ b/main/lock.py @@ -21,22 +21,27 @@ class RedisLockKey(models.TextChoices): OPERATION_LEARNING_SUMMARY_EXPORT = _BASE + "-operation-learning-summary-export-{0}" MODEL_TRANSLATION = _BASE + "-{model_name}-translation-{id}" DREF_SUMMARY = _BASE + "-dref-summary-{0}" + CLEAR_EXPIRED_DJANGO_SESSIONS = _BASE + "-clear-expired-django-sessions" @contextmanager def redis_lock( key: RedisLockKey, - id: typing.Union[int, str], + id: typing.Union[int, str, None] = None, model_name: typing.Optional[str] = None, lock_expire: int = settings.REDIS_DEFAULT_LOCK_EXPIRE, ): """ Locking mechanism using Redis + + `id` is optional, for singleton locks whose key has no placeholder. """ if model_name and id: lock_id = key.format(model_name=model_name, id=id) - else: + elif id is not None: lock_id = key.format(id) + else: + lock_id = str(key) timeout_at = time.monotonic() + lock_expire - 3 # cache.add fails if the key already exists status = cache.add(lock_id, 1, lock_expire) diff --git a/main/sentry.py b/main/sentry.py index 5b5966f55..6c0453f2c 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -1,3 +1,4 @@ +import dataclasses import logging import os import typing @@ -82,21 +83,48 @@ def fetch_git_sha(path, head=None): return str(fh.read()).strip() -def init_sentry(app_type, tags={}, **config): - integrations = [ - CeleryIntegration(), - DjangoIntegration(), - RedisIntegration(), - ] - sentry_sdk.init( - **config, - ignore_errors=IGNORED_ERRORS, - integrations=integrations, - ) - with sentry_sdk.configure_scope() as scope: - scope.set_tag("app_type", app_type) - for tag, value in tags.items(): - scope.set_tag(tag, value) +@dataclasses.dataclass +class SentryConfig: + """Typed sentry options. + + Every option is passed to sentry_sdk.init() by name, so a config key it does + not recognise can no longer reach it. Use dataclasses.replace() to vary a + field per process (see main/celery.py). + """ + + dsn: str + release: str | None + environment: str + send_default_pii: bool + traces_sample_rate: float + enable_tracing: bool + debug: bool + # Custom configs + monitor_celery_beat_tasks: bool + app_type: str + tags: dict[str, str] + + def init_sentry(self): + integrations = [ + DjangoIntegration(), + RedisIntegration(), + CeleryIntegration(monitor_beat_tasks=self.monitor_celery_beat_tasks), + ] + sentry_sdk.init( + ignore_errors=IGNORED_ERRORS, + integrations=integrations, + dsn=self.dsn, + release=self.release, + environment=self.environment, + send_default_pii=self.send_default_pii, + traces_sample_rate=self.traces_sample_rate, + enable_tracing=self.enable_tracing, + debug=self.debug, + ) + with sentry_sdk.configure_scope() as scope: + scope.set_tag("app_type", self.app_type) + for tag, value in self.tags.items(): + scope.set_tag(tag, value) class SentryMonitor(models.TextChoices): diff --git a/main/settings.py b/main/settings.py index 3dfffd1be..a32dbcce0 100644 --- a/main/settings.py +++ b/main/settings.py @@ -1,3 +1,4 @@ +# type: ignore[reportAttributeAccessIssue] import base64 import logging import os @@ -122,9 +123,12 @@ APPEALS_USER=(str, None), APPEALS_PASS=(str, None), # Sentry + SENTRY_DEBUG=(bool, False), SENTRY_DSN=(str, None), SENTRY_SAMPLE_RATE=(float, 0.2), SENTRY_RELEASE=(str, None), + # Sentry cron monitors for celery beat tasks only; legacy k8s CronJobs use SentryMonitor. + SENTRY_MONITOR_CELERY_BEAT_TASKS=(bool, True), # Maintenance mode DJANGO_READ_ONLY=(bool, False), # Misc @@ -283,6 +287,7 @@ def parse_domain(*env_keys: str) -> str: # GO Apps *GO_APPS, # Utils Apps + "django_celery_beat", "oauth2_provider", "tinymce", "admin_auto_filters", @@ -849,6 +854,8 @@ def log_render_extra_context(record): CELERY_RESULT_BACKEND = CELERY_REDIS_URL CELERY_TIMEZONE = TIME_ZONE CELERY_ACKS_LATE = True +# Schedules come from main/cronjobs.py, synced into PeriodicTask rows on beat startup. +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" RETRY_STRATEGY = Retry(total=3, status_forcelist=[429, 500, 502, 503, 504], allowed_methods=["HEAD", "GET", "OPTIONS"]) @@ -896,9 +903,12 @@ def log_render_extra_context(record): LAST_GIT_TAG = 0 # Sentry Config +SENTRY_DEBUG = env("SENTRY_DEBUG") SENTRY_DSN = env("SENTRY_DSN") SENTRY_SAMPLE_RATE = env("SENTRY_SAMPLE_RATE") +SENTRY_MONITOR_CELERY_BEAT_TASKS = env("SENTRY_MONITOR_CELERY_BEAT_TASKS") + SENTRY_RELEASE = env("SENTRY_RELEASE") if not SENTRY_RELEASE: try: @@ -907,24 +917,27 @@ def log_render_extra_context(record): # .git is missing, unreadable, or a gitfile pointer (submodule/worktree checkout) SENTRY_RELEASE = "unknown" -SENTRY_CONFIG = { - "dsn": SENTRY_DSN, - "send_default_pii": True, +SENTRY_CONFIG = sentry.SentryConfig( + dsn=SENTRY_DSN, + release=SENTRY_RELEASE, + environment=GO_ENVIRONMENT, + send_default_pii=True, # Drop k8s health-probe transactions from tracing (they fire every few seconds). - "traces_sampler": make_sentry_traces_sampler_with_health_probe_ignore(SENTRY_SAMPLE_RATE), - "enable_tracing": True, - "release": SENTRY_RELEASE, - "environment": GO_ENVIRONMENT, - "debug": DEBUG, - "tags": { + traces_sampler=make_sentry_traces_sampler_with_health_probe_ignore(SENTRY_SAMPLE_RATE), + enable_tracing=True, + debug=SENTRY_DEBUG, + # Custom configs + monitor_celery_beat_tasks=SENTRY_MONITOR_CELERY_BEAT_TASKS, + # TODO: changes it to env! + app_type="API", + tags={ "site": GO_API_URL, }, -} +) + if SENTRY_DSN: - sentry.init_sentry( - app_type="API", - **SENTRY_CONFIG, - ) + SENTRY_CONFIG.init_sentry() + # Required for Django HayStack HAYSTACK_CONNECTIONS = { "default": { diff --git a/main/test_cronjobs.py b/main/test_cronjobs.py new file mode 100644 index 000000000..0d1ea52ae --- /dev/null +++ b/main/test_cronjobs.py @@ -0,0 +1,123 @@ +import dataclasses +import inspect +from pydoc import locate + +import pytest +from django.conf import settings + +from main import cronjobs +from main.cronjobs import ( + BEAT_SCHEDULES, + SCHEDULES, + CeleryQueue, + SentryMonkeyPatch, + get_obsolete_periodic_task_qs, +) + +ALL_QUEUE_NAMES = {queue.name for queue in CeleryQueue.ALL_QUEUE} + +# SentryConfig fields that are ours, not sentry_sdk.init() options. +CUSTOM_SENTRY_FIELDS = {"monitor_celery_beat_tasks", "app_type", "tags"} + +# The only keys ModelEntry._unpack_options keeps; anything else is silently dropped. +SUPPORTED_OPTION_KEYS = { + "queue", + "exchange", + "routing_key", + "priority", + "headers", + "expire_seconds", +} + + +def test_sentry_config_fields_are_all_accounted_for(): + """Each SentryConfig field must be a real sentry_sdk option or a known custom one. + + sentry_sdk.init() raises `TypeError: Unknown option` for anything it does not + recognise, and that only happens once SENTRY_DSN is set -- never locally. + A field that is neither must be handled by hand in init_sentry(). + """ + from sentry_sdk.consts import DEFAULT_OPTIONS + + from main.sentry import SentryConfig + + fields = {f.name for f in dataclasses.fields(SentryConfig)} + unaccounted = fields - set(DEFAULT_OPTIONS) - CUSTOM_SENTRY_FIELDS + assert not unaccounted, f"SentryConfig fields sentry_sdk.init() does not know: {unaccounted}" + + +def test_sentry_config_still_carries_beat_monitoring(): + assert isinstance(settings.SENTRY_CONFIG.monitor_celery_beat_tasks, bool) + assert dataclasses.replace(settings.SENTRY_CONFIG, app_type="WORKER").app_type == "WORKER" + + +def test_schedules_tasks_are_importable_celery_tasks(): + for name, config in SCHEDULES.items(): + task = locate(config.task) + assert task is not None, f"{name}: task path does not exist: {config.task}" + assert hasattr(task, "delay"), f"{name}: {config.task} is not a celery task" + + +def test_schedules_set_the_queue_in_config_not_on_the_task(): + """A queue is optional -- without one the task uses task_default_queue. When + it is set it belongs in the SCHEDULES entry, never on the task decorator. + """ + for name, config in SCHEDULES.items(): + task_queue = getattr(locate(config.task), "queue", None) + assert task_queue is None, f"{name}: set queue in SCHEDULES, not on the {config.task} decorator" + + queue = config.options.get("queue") + if queue is not None: + assert queue in ALL_QUEUE_NAMES, f"{name}: queue {queue!r} is not one of {ALL_QUEUE_NAMES}" + + +def test_schedules_declare_an_expiry(): + """Without expire_seconds a backlog accumulates while workers are down.""" + missing = [name for name, config in SCHEDULES.items() if "expire_seconds" not in config.options] + assert not missing, f"Cronjobs missing expire_seconds: {missing}" + + +def test_schedules_only_use_options_the_scheduler_honours(): + """Guard against silently-dropped options (eg. time_limit belongs on the task).""" + for name, entry in BEAT_SCHEDULES.items(): + unsupported = set(entry["options"]) - SUPPORTED_OPTION_KEYS + assert not unsupported, f"{name}: options {unsupported} are ignored by DatabaseScheduler" + + +def test_prune_filter_matches_how_our_tasks_are_named(): + """A task path not starting with a GO_APPS label is invisible to pruning.""" + for name, config in SCHEDULES.items(): + app_label = config.task.split(".")[0] + assert app_label in settings.GO_APPS, f"{name}: {config.task} does not start with a GO_APPS label" + + +def test_sentry_monkeypatch_is_applied_and_signature_still_matches(): + """Fails if a sentry-sdk upgrade moves or re-signatures the private hook.""" + from sentry_sdk.integrations.celery import beat as sentry_celery_beat + + assert sentry_celery_beat._get_monitor_config is SentryMonkeyPatch.custom__get_monitor_config + + expected = ["celery_schedule", "app", "monitor_name"] + assert list(inspect.signature(cronjobs._get_monitor_config).parameters) == expected + assert list(inspect.signature(SentryMonkeyPatch.custom__get_monitor_config).parameters) == expected + + +@pytest.mark.django_db +def test_prune_targets_only_obsolete_rows_of_ours(): + from django_celery_beat.models import IntervalSchedule, PeriodicTask + + schedule, _ = IntervalSchedule.objects.get_or_create(every=1, period=IntervalSchedule.DAYS) + + PeriodicTask.objects.create(name="obsolete_job", task="api.tasks.removed_job", interval=schedule) + # Spared: deliberate admin-created escape hatch. + PeriodicTask.objects.create(name="manual:adhoc_job", task="api.tasks.removed_job", interval=schedule) + # Spared: not one of our apps. + PeriodicTask.objects.create(name="vendor_job", task="some_vendor.tasks.thing", interval=schedule) + # Spared: still declared in SCHEDULES. + PeriodicTask.objects.create( + name="clear_expired_django_sessions", + task="api.tasks.clear_expired_django_sessions", + interval=schedule, + ) + + assert set(get_obsolete_periodic_task_qs().values_list("name", flat=True)) == {"obsolete_job"} diff --git a/misc/dev/run_worker.sh b/misc/dev/run_worker.sh new file mode 100755 index 000000000..0553b971c --- /dev/null +++ b/misc/dev/run_worker.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e +# Uses the management command for Django's autoreloader. + +# TODO(susilnem): Use wait_for_resources once banjo-stack is implemented +./misc/wait-for-broker.sh + +exec ./manage.py run_celery_dev diff --git a/misc/dev/run_worker_beat.sh b/misc/dev/run_worker_beat.sh new file mode 100755 index 000000000..addd48af6 --- /dev/null +++ b/misc/dev/run_worker_beat.sh @@ -0,0 +1,5 @@ +#!/bin/bash -e + +./misc/wait-for-broker.sh + +exec celery -A main beat -l INFO diff --git a/misc/wait-for-broker.sh b/misc/wait-for-broker.sh new file mode 100755 index 000000000..5c1f0cea1 --- /dev/null +++ b/misc/wait-for-broker.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e +# Wait for the celery broker. main/entrypoint.sh waits for the db, not the broker. +# +# TODO: Replace with wait_for_resources once banjo-stack is implemented + +BROKER_HOST_PORT=$(python -c " +import os +from urllib.parse import urlparse + +url = urlparse(os.environ['CELERY_REDIS_URL']) +print(f'{url.hostname}:{url.port or 6379}') +") + +echo "Waiting for celery broker at ${BROKER_HOST_PORT}..." +wait-for-it "${BROKER_HOST_PORT}" +>&2 echo "Celery broker is up - continuing..." diff --git a/pyproject.toml b/pyproject.toml index 8102b0b62..7d46ee7c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,9 @@ dependencies = [ "xmltodict==0.11.0", "xhtml2pdf==0.2.17", "reportlab", # XXX: Used by xhtml2pdf reportlab==3.6.7 breaks for now - "celery[redis]>=5.2.2", + # NOTE: floor is >=5.2.3 because django-celery-beat requires celery>=5.2.3,<6.0 + "celery[redis]>=5.2.3", + "django-celery-beat>=2.9.0", "django-redis==5.0.0", "sentry-sdk", "django-haystack[elasticsearch]", diff --git a/uv.lock b/uv.lock index 9597a3fff..232120d24 100644 --- a/uv.lock +++ b/uv.lock @@ -465,6 +465,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/5a/234e8fadf85c3cc48cb31c247b9e8e0c7f06ece80f5b29f9b8c241f9da4c/coverage-7.16.0-py3-none-any.whl", hash = "sha256:245f7de6d023a5bba375dbec9f2e0869bfa26ac0cc639bbb7b4c814884000b73", size = 214977, upload-time = "2026-08-28T21:54:35.189Z" }, ] +[[package]] +name = "cron-descriptor" +version = "1.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/83/70bd410dc6965e33a5460b7da84cf0c5a7330a68d6d5d4c3dfdb72ca117e/cron_descriptor-1.4.5.tar.gz", hash = "sha256:f51ce4ffc1d1f2816939add8524f206c376a42c87a5fca3091ce26725b3b1bca", size = 30666, upload-time = "2024-08-24T18:16:48.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/20/2cfe598ead23a715a00beb716477cfddd3e5948cf203c372d02221e5b0c6/cron_descriptor-1.4.5-py3-none-any.whl", hash = "sha256:736b3ae9d1a99bc3dbfc5b55b5e6e7c12031e7ba5de716625772f8b02dcd6013", size = 50370, upload-time = "2024-08-24T18:16:46.783Z" }, +] + [[package]] name = "cryptography" version = "50.0.0" @@ -574,6 +583,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/18/35bd4e459bfd387c67f1439fafb7e923fa1df620d41a5eae148fa9f5b551/django_admin_list_filter_dropdown-1.0.3-py3-none-any.whl", hash = "sha256:bf1b48bab9772dad79db71efef17e78782d4f2421444d5e49bb10e0da71cd6bb", size = 3813, upload-time = "2019-10-14T10:21:25.963Z" }, ] +[[package]] +name = "django-celery-beat" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "celery" }, + { name = "cron-descriptor" }, + { name = "django" }, + { name = "django-timezone-field" }, + { name = "python-crontab" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/45/fc97bc1d9af8e7dc07f1e37044d9551a30e6793249864cef802341e2e3a8/django_celery_beat-2.9.0.tar.gz", hash = "sha256:92404650f52fcb44cf08e2b09635cb1558327c54b1a5d570f0e2d3a22130934c", size = 177667, upload-time = "2026-02-28T16:45:34.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/ae/9befa7ae37f5e5c41be636a254fcf47ff30dd5c88bd115070e252f6b9162/django_celery_beat-2.9.0-py3-none-any.whl", hash = "sha256:4a9e5ebe26d6f8d7215e1fc5c46e466016279dc102435a28141108649bdf2157", size = 105013, upload-time = "2026-02-28T16:45:32.822Z" }, +] + [[package]] name = "django-cors-headers" version = "3.11.0" @@ -840,6 +866,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/41/10b8d576651af4f1dc6bdfb0d8a203e8b3a55d60ae36ba305d0edcdb594f/django_stubs_ext-6.1.0-py3-none-any.whl", hash = "sha256:57273506823274700a707c8f404dfb52a12c4554daf1468dfb2bff857bb22074", size = 10404, upload-time = "2026-08-12T10:54:42.979Z" }, ] +[[package]] +name = "django-timezone-field" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/0b/22654abc2355f3b84e3e5b9d26569639c15d04b96d7186f3a477ec45c1ef/django_timezone_field-7.2.2.tar.gz", hash = "sha256:a004d0b19fe10bf5964cb21a65b36324b16a61879e4711c0dafdf8d6253e8ebc", size = 13158, upload-time = "2026-06-06T05:28:23.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/5f/c8dfb67105c4ef391a3e9d0bcd35b7eae4c4c9d023d612c3999ae1cb32ef/django_timezone_field-7.2.2-py3-none-any.whl", hash = "sha256:30354d0f37462a0b9b5c289e271580a6be9b58dea30e7bf88435372882c8fa7a", size = 13322, upload-time = "2026-06-06T05:28:22.454Z" }, +] + [[package]] name = "django-tinymce" version = "4.1.0" @@ -1038,6 +1076,7 @@ dependencies = [ { name = "django" }, { name = "django-admin-autocomplete-filter" }, { name = "django-admin-list-filter-dropdown" }, + { name = "django-celery-beat" }, { name = "django-cors-headers" }, { name = "django-coverage" }, { name = "django-enumfield" }, @@ -1126,7 +1165,7 @@ requires-dist = [ { name = "banjo-utils", git = "https://github.com/toggle-corp/banjo-utils?tag=v0.2.0" }, { name = "beautifulsoup4", specifier = ">=4.12.3,<5.0" }, { name = "boto3", specifier = ">=1.34.0,<2.0.0" }, - { name = "celery", extras = ["redis"], specifier = ">=5.2.2" }, + { name = "celery", extras = ["redis"], specifier = ">=5.2.3" }, { name = "choicesenum", specifier = ">=0.7.0,<1.0" }, { name = "colorlog" }, { name = "coverage", specifier = ">=7.4.0,<8.0" }, @@ -1134,6 +1173,7 @@ requires-dist = [ { name = "django", specifier = ">=5.2,<5.3" }, { name = "django-admin-autocomplete-filter" }, { name = "django-admin-list-filter-dropdown" }, + { name = "django-celery-beat", specifier = ">=2.9.0" }, { name = "django-cors-headers", specifier = "==3.11.0" }, { name = "django-coverage", specifier = "==1.2.4" }, { name = "django-enumfield", specifier = "==2.0.2" }, @@ -2398,6 +2438,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/f0/f6b9d17e3426e7b54c18d05d917982647a976233ed9348a2ef40f1a17f85/python_bidi-0.6.11-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:cf4e88a6fec81b7155a487cbbea7753a3d9a76dc4d391b4f8958b37227ef2c12", size = 505204, upload-time = "2026-06-30T14:23:41.222Z" }, ] +[[package]] +name = "python-crontab" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/99/7f/c54fb7e70b59844526aa4ae321e927a167678660ab51dda979955eafb89a/python_crontab-3.3.0.tar.gz", hash = "sha256:007c8aee68dddf3e04ec4dce0fac124b93bd68be7470fc95d2a9617a15de291b", size = 57626, upload-time = "2025-07-13T20:05:35.535Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/42/bb4afa5b088f64092036221843fc989b7db9d9d302494c1f8b024ee78a46/python_crontab-3.3.0-py3-none-any.whl", hash = "sha256:739a778b1a771379b75654e53fd4df58e5c63a9279a63b5dfe44c0fcc3ee7884", size = 27533, upload-time = "2025-07-13T20:05:34.266Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From bb4b32c1b4cb401e5a81f765ff3debf4d02030ad Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 9 Sep 2026 12:24:42 +0545 Subject: [PATCH 2/4] fix(sentry): build traces_sampler inside SentryConfig.init_sentry The banjo stack passes traces_sampler= to SENTRY_CONFIG, but the SentryConfig dataclass only had traces_sample_rate, so settings import raised TypeError: unexpected keyword argument 'traces_sampler' in every process where SENTRY_DSN is set. Keep the rate on the dataclass and build banjo's health-probe-ignoring sampler inside init_sentry(), so the config stays plain data and dataclasses.replace() does not carry a closure. --- .../templates/config/configmap.yaml | 59 ---- deploy/helm/ifrcgo-helm/values.yaml | 333 ------------------ main/sentry.py | 5 +- main/settings.py | 8 +- misc/wait-for-broker.sh | 16 - 5 files changed, 6 insertions(+), 415 deletions(-) delete mode 100644 deploy/helm/ifrcgo-helm/templates/config/configmap.yaml delete mode 100644 deploy/helm/ifrcgo-helm/values.yaml delete mode 100755 misc/wait-for-broker.sh diff --git a/deploy/helm/ifrcgo-helm/templates/config/configmap.yaml b/deploy/helm/ifrcgo-helm/templates/config/configmap.yaml deleted file mode 100644 index ac95b4d68..000000000 --- a/deploy/helm/ifrcgo-helm/templates/config/configmap.yaml +++ /dev/null @@ -1,59 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: {{ template "ifrcgo-helm.fullname" . }}-api-configmap - labels: - component: api-deployment - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -data: - # Redis - {{- if .Values.redis.enabled }} - CELERY_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0" - CACHE_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/1" - {{- else }} - CELERY_REDIS_URL: {{ required "env.CELERY_REDIS_URL" .Values.env.CELERY_REDIS_URL | quote }} - CACHE_REDIS_URL: {{ required "env.CACHE_REDIS_URL" .Values.env.CACHE_REDIS_URL | quote }} - {{- end }} - - {{- if .Values.minio.enabled }} - AWS_S3_ENABLED: "true" - {{- else }} - AZURE_STORAGE_ENABLED: "true" - {{- end }} - - {{- if .Values.playwright.enabled }} - PLAYWRIGHT_SERVER_URL: "ws://{{ template "ifrcgo-helm.fullname" . }}-playwright:{{ .Values.playwright.containerPort }}/" - {{- else }} - PLAYWRIGHT_SERVER_URL: {{ required "env.PLAYWRIGHT_SERVER_URL" .Values.env.PLAYWRIGHT_SERVER_URL | quote }} - {{- end }} - - CACHE_MIDDLEWARE_SECONDS: {{ .Values.env.CACHE_MIDDLEWARE_SECONDS | quote }} - DJANGO_DEBUG: {{ .Values.env.DJANGO_DEBUG | quote }} - ELASTIC_SEARCH_HOST: {{ default (printf "elasticsearch://%s-elasticsearch:9200" (include "ifrcgo-helm.fullname" .)) .Values.env.ELASTIC_SEARCH_HOST | quote }} - ELASTIC_SEARCH_INDEX: {{ .Values.env.ELASTIC_SEARCH_INDEX | quote }} - DOCKER_HOST_IP: {{ .Values.env.DOCKER_HOST_IP | quote }} - DJANGO_ADDITIONAL_ALLOWED_HOSTS: {{ .Values.env.DJANGO_ADDITIONAL_ALLOWED_HOSTS | quote }} - ADDITIONAL_TRUSTED_ORIGINS: {{ .Values.env.ADDITIONAL_TRUSTED_ORIGINS | quote }} - SESSION_COOKIE_DOMAIN: {{ .Values.env.SESSION_COOKIE_DOMAIN | quote }} - CSRF_COOKIE_DOMAIN: {{ .Values.env.CSRF_COOKIE_DOMAIN | quote }} - GO_ENVIRONMENT: {{ .Values.env.GO_ENVIRONMENT | quote }} - API_FQDN: {{ .Values.env.API_FQDN | quote }} - FRONTEND_URL: {{ .Values.env.FRONTEND_URL | quote }} - DEBUG_EMAIL: {{ .Values.env.DEBUG_EMAIL | quote }} - IFRC_TRANSLATION_DOMAIN: {{ .Values.env.IFRC_TRANSLATION_DOMAIN | quote }} - AUTO_TRANSLATION_TRANSLATOR: {{ .Values.env.AUTO_TRANSLATION_TRANSLATOR | quote }} - DJANGO_READ_ONLY: {{ .Values.env.DJANGO_READ_ONLY | quote }} - SENTRY_SAMPLE_RATE: {{ .Values.env.SENTRY_SAMPLE_RATE | quote }} - SENTRY_DEBUG: {{ .Values.env.SENTRY_DEBUG | quote }} - SENTRY_DSN: {{ .Values.env.SENTRY_DSN | quote }} - SENTRY_MONITOR_CELERY_BEAT_TASKS: {{ .Values.env.SENTRY_MONITOR_CELERY_BEAT_TASKS | quote }} - OIDC_ENABLE: {{ .Values.env.OIDC_ENABLE | quote }} - - EOAPI_STAC_EXTERNAL_URL: {{ .Values.env.EOAPI_STAC_EXTERNAL_URL | quote }} - EOAPI_STAC_INTERNAL_URL: {{ .Values.env.EOAPI_STAC_INTERNAL_URL | quote }} - - # Additional configs - {{- range $name, $value := .Values.envAdditional }} - {{ $name }}: {{ $value | quote }} - {{- end }} diff --git a/deploy/helm/ifrcgo-helm/values.yaml b/deploy/helm/ifrcgo-helm/values.yaml deleted file mode 100644 index f6543aeb3..000000000 --- a/deploy/helm/ifrcgo-helm/values.yaml +++ /dev/null @@ -1,333 +0,0 @@ -environment: dev - -env: - DJANGO_SECRET_KEY: '' - DJANGO_DB_NAME: '' - DJANGO_DB_USER: '' - DJANGO_DB_PASS: '' - DJANGO_DB_HOST: '' - DJANGO_DB_PORT: '' - AZURE_STORAGE_ACCOUNT: '' - AZURE_STORAGE_KEY: '' - EMAIL_API_ENDPOINT: '' - EMAIL_HOST: '' - EMAIL_PORT: '' - EMAIL_USER: '' - EMAIL_PASS: '' - TEST_EMAILS: '' - AWS_TRANSLATE_ACCESS_KEY: '' - AWS_TRANSLATE_SECRET_KEY: '' - AWS_TRANSLATE_REGION: '' - CELERY_REDIS_URL: '' - CACHE_MIDDLEWARE_SECONDS: '' - MOLNIX_API_BASE: '' - MOLNIX_USERNAME: '' - MOLNIX_PASSWORD: '' - ERP_API_ENDPOINT: '' - ERP_API_SUBSCRIPTION_KEY: '' - FDRS_CREDENTIAL: '' - HPC_CREDENTIAL: '' - APPLICATION_INSIGHTS_INSTRUMENTATION_KEY: '' - ELASTIC_SEARCH_HOST: '' - ELASTIC_SEARCH_INDEX: 'new_index' - GO_FTPHOST: '' - GO_FTPUSER: '' - GO_FTPPASS: '' - GO_DBPASS: '' - APPEALS_USER: '' - APPEALS_PASS: '' - DJANGO_DEBUG: '' - DOCKER_HOST_IP: '' - DJANGO_ADDITIONAL_ALLOWED_HOSTS: '' - ADDITIONAL_TRUSTED_ORIGINS: '' - SESSION_COOKIE_DOMAIN: '' - CSRF_COOKIE_DOMAIN: '' - GO_ENVIRONMENT: '' - API_FQDN: '' - FRONTEND_URL: '' - DEBUG_EMAIL: '' - SENTRY_DEBUG: false - SENTRY_DSN: '' - SENTRY_MONITOR_CELERY_BEAT_TASKS: true - SENTRY_SAMPLE_RATE: '' - DJANGO_READ_ONLY: '' - AUTO_TRANSLATION_TRANSLATOR: '' - IFRC_TRANSLATION_DOMAIN: '' - IFRC_TRANSLATION_HEADER_API_KEY: '' - FDRS_APIKEY: '' - NS_CONTACT_USERNAME: '' - NS_CONTACT_PASSWORD: '' - ACAPS_API_TOKEN: '' - NS_DOCUMENT_API_KEY: '' - NS_INITIATIVES_API_KEY: '' - NS_DOCUMENT_API_TOKEN: '' - JWT_PRIVATE_KEY_BASE64_ENCODED: '' - JWT_PUBLIC_KEY_BASE64_ENCODED: '' - JWT_EXPIRE_TIMESTAMP_DAYS: '' - AZURE_OPENAI_DEPLOYMENT_NAME: '' - AZURE_OPENAI_ENDPOINT: '' - AZURE_OPENAI_API_KEY: '' - OIDC_ENABLE: false - OIDC_RSA_PRIVATE_KEY_BASE64_ENCODED: - OIDC_RSA_PUBLIC_KEY_BASE64_ENCODED: - RELIEF_WEB_APP_NAME: '' - POWERBI_WORKSPACE_ID: '' - POWERBI_DATASET_IDS: '' - -# NOTE: Used to pass additional configs to api/worker containers -# NOTE: Not used by azure vault -envAdditional: - # Additional configs - # EXAMPLE: MY_CONFIG: "my-value" - -secrets: - API_TLS_CRT: '' - API_TLS_KEY: '' - API_ADDITIONAL_DOMAIN_TLS_CRT: '' - API_ADDITIONAL_DOMAIN_TLS_KEY: '' - -# NOTE: Used to pass additional secrets to api/worker containers -# NOTE: Not used by azure vault -secretsAdditional: - # Additional secrets - # EXAMPLE: MY_SECRET: "my-secret-value" - -redis: - enabled: true - architecture: standalone - fullnameOverride: go-redis - auth: - enabled: false - master: - persistence: - enabled: true - size: 1Gi - resources: - requests: - cpu: "0.5" - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - resources: - requests: - cpu: "0.5" - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - -# https://artifacthub.io/packages/helm/bitnami/minio -# extraEnvVars: https://github.com/bitnami/containers/blob/main/bitnami/minio/README.md#environment-variables -minio: - enabled: false # XXX: Used for alpha instances running outside Azure - disableWebUI: true - mode: standalone - fullnameOverride: go-minio - global: - defaultStorageClass: - apiIngress: - enabled: true - ingressClassName: - hostname: - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "50m" - auth: - forceNewKeys: True - rootUser: go - rootPassword: - persistence: - enabled: true - size: 1Gi - defaultBuckets: go-data,go-static - provisioning: - enabled: true - resourcesPreset: "nano" - cleanupAfterFinished: - enabled: true - extraCommands: - - "mc anonymous set download provisioning/go-static" - -postgresql: - enabled: false # XXX: Used for alpha instances running outside Azure - fullnameOverride: "go-pg" - architecture: standalone - primary: - persistence: - enabled: true - size: 8Gi - -playwright: - enabled: rue - replicaCount: 1 - containerPort: 3000 - image: - # NOTE: Make sure this matches with pyproject playwright dependency and root docker-compose - name: 'mcr.microsoft.com/playwright' - tag: 'v1.50.0-noble' - pullPolicy: 'IfNotPresent' - resources: - requests: - cpu: "0.1" - memory: 1Gi - limits: - cpu: "2" - memory: 2Gi - -api: - domain: "go-staging.ifrc.org" - tls: - enabled: true - additionalDomain: "" - enabled: true - replicaCount: 1 - containerPort: 80 - image: - name: 'SET-BY-CICD-IMAGE' - tag: 'SET-BY-CICD-TAG' - pullPolicy: 'IfNotPresent' - resources: - requests: - cpu: "2" - memory: 4Gi - limits: - cpu: "2" - memory: 4Gi - -celery: - enabled: true - resources: - requests: - cpu: "0.5" - memory: 0.4Gi - limits: - cpu: "2" - memory: 4Gi - -argoHooksEnabled: false # FIXME: Remove this after go-api is moved to argoCD pipeline -argoHooks: - # NOTE: Make sure keys are lowercase - database-migration: - enabled: true - hook: PostSync - preserveHistory: true - command: ["./manage.py", "migrate"] - collect-static: - enabled: true - hook: PostSync - command: ["./manage.py", "collectstatic", "--noinput"] - -cronjobsDefaultResources: - requests: - cpu: 0.1 - memory: 1Gi - limits: - cpu: 4 - memory: 2Gi - -cronjobs: - - command: 'index_and_notify' - schedule: '*/5 * * * *' - - command: 'sync_molnix' - schedule: '10 */2 * * *' - resources: - requests: - memory: 1Gi - cpu: 0.1 - limits: - memory: 2Gi - cpu: 4 - - command: 'ingest_appeals' - schedule: '*/30 * * * *' - - command: 'sync_appealdocs' - schedule: '15 * * * *' - - command: 'revoke_staff_status' - schedule: '51 * * * *' - - command: 'update_project_status' - schedule: '1 3 * * *' - - command: 'user_registration_reminder' - schedule: '0 9 * * *' - - command: 'ingest_country_plan_file' - schedule: '1 0 * * *' - - command: 'fdrs_annual_income' - schedule: '0 0 * * 0' - - command: 'FDRS_INCOME' - schedule: '0 0 * * 0' - - command: 'ingest_acaps' - schedule: '0 1 * * 0' - - command: 'ingest_climate' - schedule: '0 0 * * 0' - - command: 'ingest_databank' - schedule: '0 0 * * 0' - - command: 'ingest_hdr' - schedule: '0 0 * * 0' - - command: 'ingest_unicef' - schedule: '0 0 * * 0' - - command: 'ingest_worldbank' - schedule: '0 2 * * 0' - - command: 'ingest_disaster_law' - schedule: '0 0 * * 0' - - command: 'ingest_ns_contact' - schedule: '0 0 * * 0' - - command: 'ingest_ns_capacity' - schedule: '0 0 * * 0' - - command: 'ingest_ns_directory' - schedule: '0 0 * * 0' - - command: 'ingest_ns_document' - schedule: '0 0 * * 0' - - command: 'ingest_ns_initiatives' - schedule: '0 0 * * 0' - - command: 'ingest_icrc' - schedule: '0 3 * * 0' - - command: 'notify_validators' - schedule: '0 0 * * *' - - command: 'poll_gdacs_cyclone' - schedule: '0 11 * * 0' - - command: 'poll_gdacs_flood' - schedule: '0 11 * * *' - - command: 'poll_usgs_earthquake' - schedule: '0 18 * * 0' - - command: 'alert_notification' - schedule: '0 */2 * * *' - # https://github.com/jazzband/django-oauth-toolkit/blob/master/docs/management_commands.rst#cleartokens - - command: 'oauth_cleartokens' - schedule: '0 1 * * *' - - command: 'eap_submission_reminder' - schedule: '0 0 * * *' - - -elasticsearch: - enabled: true - httpPort: 9200 - transportPort: 9300 - elasticsearchVersion: 7.16.2 - storageSize: 20Gi - storageClassName: managed-premium # FIXME: populate and use - disk: - name: my-disk-name - uri: https://mydisk.blob.core.windows.net/mycontainer/mydisk.vhd - resources: - requests: - cpu: "2" - memory: 4Gi - limits: - cpu: "2" - memory: 4Gi - -sshBastion: - enabled: true - resources: - requests: - cpu: "0.5" - memory: 0.5Gi - limits: - cpu: "1" - memory: 1Gi - keys: - zoltan.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPGAnkQdf5CIpVoqNVJ17AAzUb02gpTltJI5q5SRKxl8 zol@hp - daniel.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDU1XLLPq1J4kFvNyg5eUK8uuW8dtW1f3ALVnYr0nVhldxF0J59XtZbNFBLCVHYZL3NQxYQrucll6LbGaMGKbGsTwtqcxqd2fWlhg7nBnvhOzULYbAru3YfpkgnawGin6Y7qW/MQ3fYmqqm8MB7p5+G4sIL76S2yWbi7lcKWnd87yDTGEEoc8H6i6IwNNVHudvuMA4MzGkSgql7gIC2KuU+s2u9Y6fmE92G39BO454SUgAcCJfhuXukZhU4UN3RVYy+F0MxVeLc0hEJi4sCYcoPKREc0//srNyni7b8G8C+z6t02xrzhWwIORlb8Jr2kmbblp7PFMz4r2qRd8MvXAa5ta6kUvMDg0t52JaDMAGy0IjGZh9PznXbp1LYn7uS5NQh4C/t6Q3TXyJbEiaQaObcmjn6w/DWH6gI7ZRYkPGdlctlNm5MWnhjG9Q/FzRIxvaauSFqgs6bfIUGGaY9i1eNiowVSzDPlP7nH0gJpq+uS5Qdyg69m/XH1DqywPoZY7U= ifrcds\daniel.tovari@5CG41911RW - arun.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIERqaO+XlqTbvoh88Kuj9c377x77NChWhNP8VpbM1/hf ifrcds\arun.gandhi@5CG1355NPN - thenav56.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN/f/A3qkaTHSdbKn8Hv75YiJvRMEXvWTDdIiR7tyAjJ navin@nav-machine - david.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3FzrQdVh5Qwp5Y6KQGcpqHxKErxCW103iEECuutR/jBZe6X0xjD+cW7e+H8SrUsPQwj87fzOsMAc6v6n+3hdYFa6ekgRG/USEIUR5C/GD1Xjva3Xpp45PasBhJEtYt2ON+dlzwvRyOuv2hvqv2WHBO020ewIlVuQ4pU4Qj5ysvwWGj8GAv/jITiVERmjLTStbFwxeIDT3jQEbwnfV1zZZKiGxIecB/y51nk6oIQ00ZGrYEo5ieWsUSVfLHOX0/lZ0mtrdqxDEgMaCbNaUbICAimsJPamNpoirKc7FoKIKKrLQsK8qE1lClWQEecbW+dgSiwxracooKeWhHq+BkKUCNgEL/C0ff2l9e8sJcLmYZUdPtDCdtUDC8BAlELA5HR6tdCTfFcc0nXltclSSODMnZkQohh5/2fixJTwN5p5csEfBLzbdrturKtT/TbYSoaodg4muPqY4YE5jiJfrHVAGS1DVWz/cRcm1vOxT2V4iW2SNvo8fS2PZOpU5furrvbM= ifrcds\david.muchatiza@5CG41911S1 - paola.pub: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGql4RrbxSQTW5QrTh+P+94jGCXOCeZgc23hxL9zFCYQrzL0SMw1F53Z5SFZimIhJswYPqV2pT8L4oTRqIrTCM+looWi7b9/9u+m/KmA+FWbo3u6uRrckkA3nVIKsKHvlOucX2GxE6i+tXdeXEisW49ZpMtuvxMLJ3Eg4MK10d/2d3FKuzTsrxCTlJn8FAE3yOsVow0jdu+381IrkAqRE2GINeQ87hVlQpbo+bL2N/2QZmNjDhBBQkRJLDisW0+UNgo+S9wN7HbpV5LheSJS9wGN7LlmcqlpZFrDO/lVyoMxEQ0588wUI8BVfqAZDEBJPdGtzq513r+5iXEX/9A1Mendlvxfl6ANNRcH9PVZHkRN1dxY3rckQ+Lk3qqIjjfYFYvl5Gybidb1BM2VNWHAuzaDDQzJpeTHIbQnDt7Ke4oX2xWYgyu+kVhqz0HnAV28qMXbMEsrMIrtwl7IjcrorgdduHghZvWFbaJZNtXOfgnf1IYNXkZ9eWPS+Bz9nWMhE= ifrcds\paola.yela@5CG41911RT - ranjan.pub: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGJA0ec4Gavc+m1MjEZGoUce51yWouMTRTYJZV3s/jgD rsh@rsh-XPS-15-9510 diff --git a/main/sentry.py b/main/sentry.py index 6c0453f2c..013cbd72d 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -6,6 +6,7 @@ import sentry_sdk import yaml +from banjo_utils.health import make_sentry_traces_sampler_with_health_probe_ignore # Celery Terminated Exception: The worker processing a job has been terminated by user request. from billiard.exceptions import Terminated @@ -117,7 +118,9 @@ def init_sentry(self): release=self.release, environment=self.environment, send_default_pii=self.send_default_pii, - traces_sample_rate=self.traces_sample_rate, + # Never sample k8s /healthz/* probe transactions (they hit the WSGI + # layer outside Django middleware); defer everything else to the rate. + traces_sampler=make_sentry_traces_sampler_with_health_probe_ignore(self.traces_sample_rate), enable_tracing=self.enable_tracing, debug=self.debug, ) diff --git a/main/settings.py b/main/settings.py index a32dbcce0..fd862925f 100644 --- a/main/settings.py +++ b/main/settings.py @@ -9,10 +9,7 @@ import environ import pytz from azure.identity import DefaultAzureCredential -from banjo_utils.health import ( - is_health_probe_path, - make_sentry_traces_sampler_with_health_probe_ignore, -) +from banjo_utils.health import is_health_probe_path from corsheaders.defaults import default_headers from django.utils.translation import gettext_lazy as _ from urllib3.util.retry import Retry @@ -922,8 +919,7 @@ def log_render_extra_context(record): release=SENTRY_RELEASE, environment=GO_ENVIRONMENT, send_default_pii=True, - # Drop k8s health-probe transactions from tracing (they fire every few seconds). - traces_sampler=make_sentry_traces_sampler_with_health_probe_ignore(SENTRY_SAMPLE_RATE), + traces_sample_rate=SENTRY_SAMPLE_RATE, enable_tracing=True, debug=SENTRY_DEBUG, # Custom configs diff --git a/misc/wait-for-broker.sh b/misc/wait-for-broker.sh deleted file mode 100755 index 5c1f0cea1..000000000 --- a/misc/wait-for-broker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -e -# Wait for the celery broker. main/entrypoint.sh waits for the db, not the broker. -# -# TODO: Replace with wait_for_resources once banjo-stack is implemented - -BROKER_HOST_PORT=$(python -c " -import os -from urllib.parse import urlparse - -url = urlparse(os.environ['CELERY_REDIS_URL']) -print(f'{url.hostname}:{url.port or 6379}') -") - -echo "Waiting for celery broker at ${BROKER_HOST_PORT}..." -wait-for-it "${BROKER_HOST_PORT}" ->&2 echo "Celery broker is up - continuing..." From 1a1f92cd43ba81ad70fb05c4308c4021a143266f Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 9 Sep 2026 12:25:48 +0545 Subject: [PATCH 3/4] feat(helm): run celery beat as a worker addon Adds app.worker.addons.beat so the cronjobs in main/cronjobs.py actually run in deployed environments; previously beat existed only in docker-compose. - replicaCount: 1 with strategy: Recreate -- two beat processes would fire every cronjob twice - --scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler so banjo-celery-probe can check liveness - SENTRY_DEBUG and SENTRY_MONITOR_CELERY_BEAT_TASKS added to app.env - snapshots regenerated for alpha/staging/production --- deploy/helm/snapshots/alpha.yaml | 101 ++++++++++++++++++++-- deploy/helm/snapshots/production.yaml | 116 ++++++++++++++++++++++++-- deploy/helm/snapshots/staging.yaml | 116 ++++++++++++++++++++++++-- deploy/helm/values.yaml | 46 +++++++++- docs/cronjobs.md | 40 +++++---- 5 files changed, 386 insertions(+), 33 deletions(-) diff --git a/deploy/helm/snapshots/alpha.yaml b/deploy/helm/snapshots/alpha.yaml index 756e7edee..065eecbce 100644 --- a/deploy/helm/snapshots/alpha.yaml +++ b/deploy/helm/snapshots/alpha.yaml @@ -42,6 +42,8 @@ data: ELASTIC_SEARCH_HOST: "http://go-api-elasticsearch.example.com:9200" FRONTEND_URL: "https://go.alpha.example.com" GO_ENVIRONMENT: "ALPHA" + SENTRY_DEBUG: "false" + SENTRY_MONITOR_CELERY_BEAT_TASKS: "true" SENTRY_SAMPLE_RATE: "0.2" SESSION_COOKIE_DOMAIN: ".example.com" @@ -107,7 +109,7 @@ spec: metadata: annotations: checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d - checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92 + checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449 labels: app: go-api component: api @@ -169,6 +171,95 @@ spec: sleep: seconds: 8 +--- +# Source: ifrcgo-helm/charts/app/templates/worker-addons/deployment.yaml +# Addon: beat +apiVersion: apps/v1 +kind: Deployment +metadata: + name: go-api-worker-beat + annotations: + argocd.argoproj.io/sync-wave: "30" + reloader.stakater.com/auto: "true" + labels: + app: go-api + component: worker-addon + addon: beat + environment: ALPHA + release: release-name +spec: + replicas: 1 + strategy: + type: Recreate + revisionHistoryLimit: 1 + selector: + matchLabels: + app: go-api + component: worker-addon + addon: beat + template: + metadata: + annotations: + checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d + checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449 + labels: + app: go-api + component: worker-addon + addon: beat + spec: + volumes: + - emptyDir: + medium: Memory + sizeLimit: 1Mi + name: celery-heartbeat + containers: + - name: beat + image: "local.example.com/ifrcgo/go-api:develop.xxxxxxxx" + imagePullPolicy: IfNotPresent + command: + - celery + - -A + - main + - beat + - -l + - INFO + - --scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler + livenessProbe: + exec: + command: + - banjo-celery-probe + - --max-age + - "60" + failureThreshold: 3 + periodSeconds: 60 + timeoutSeconds: 5 + resources: + limits: + memory: 512Mi + requests: + cpu: "0.1" + memory: 256Mi + envFrom: + - secretRef: + name: go-api-secret + - configMapRef: + name: go-api-env-name + env: + - name: DJANGO_APP_TYPE + value: "worker" + - name: CACHE_REDIS_URL + value: redis://go-api-dragonfly:6379/1 + - name: CELERY_REDIS_URL + value: redis://go-api-dragonfly:6379/0 + - name: PLAYWRIGHT_SERVER_URL + value: ws://go-api-playwright:3000/ + + - name: BANJO_CELERY_HEARTBEAT_FILE + value: "/var/run/celery/beat_heartbeat" + volumeMounts: + - mountPath: /var/run/celery + name: celery-heartbeat + --- # Source: ifrcgo-helm/charts/app/templates/worker/deployment.yaml # Queue: default @@ -196,7 +287,7 @@ spec: metadata: annotations: checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d - checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92 + checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449 labels: app: go-api component: worker @@ -326,7 +417,7 @@ spec: metadata: annotations: checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d - checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92 + checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449 labels: app: go-api component: hook @@ -384,7 +475,7 @@ spec: metadata: annotations: checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d - checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92 + checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449 labels: app: go-api component: hook @@ -440,7 +531,7 @@ spec: metadata: annotations: checksum/secret: dd5e71c25a35366086506b396869b06a501d1ec22e83806a15738e9f3ac3a03d - checksum/configmap: 0313dd0020b57c2897db8d61fed2b2611c7bcb7840d7dbc1f37fe90f32881f92 + checksum/configmap: a69dd58e3dc31298d8e3f2ee16870920ad8ef2c9ad466f5e1ac7d802a9b00449 labels: app: go-api component: hook diff --git a/deploy/helm/snapshots/production.yaml b/deploy/helm/snapshots/production.yaml index 7f29bc691..de1c3ebdd 100644 --- a/deploy/helm/snapshots/production.yaml +++ b/deploy/helm/snapshots/production.yaml @@ -39,6 +39,8 @@ data: HEALTH_CHECK_DISK_USAGE_MAX: "none" JWT_EXPIRE_TIMESTAMP_DAYS: "365" OIDC_ENABLE: "true" + SENTRY_DEBUG: "false" + SENTRY_MONITOR_CELERY_BEAT_TASKS: "true" SENTRY_SAMPLE_RATE: "0.2" SESSION_COOKIE_DOMAIN: ".example.com" @@ -123,7 +125,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 8338ba490298cb58350d749c6fe8334b174149408ad4fec592f1a7360a91b4c5 + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 labels: app: go-api component: api @@ -200,6 +202,108 @@ spec: mountPath: /mnt/secrets-store readOnly: true +--- +# Source: ifrcgo-helm/charts/app/templates/worker-addons/deployment.yaml +# Addon: beat +apiVersion: apps/v1 +kind: Deployment +metadata: + name: go-api-worker-beat + annotations: + argocd.argoproj.io/sync-wave: "30" + reloader.stakater.com/auto: "true" + labels: + app: go-api + component: worker-addon + addon: beat + environment: production + release: release-name +spec: + replicas: 1 + strategy: + type: Recreate + revisionHistoryLimit: 1 + selector: + matchLabels: + app: go-api + component: worker-addon + addon: beat + template: + metadata: + annotations: + checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 + labels: + app: go-api + component: worker-addon + addon: beat + azure.workload.identity/use: "true" + spec: + volumes: + - name: go-api-secret + csi: + driver: "secrets-store.csi.k8s.io" + readOnly: true + volumeAttributes: + secretProviderClass: go-api-secret-provider + - emptyDir: + medium: Memory + sizeLimit: 1Mi + name: celery-heartbeat + serviceAccountName: service-token-reader + containers: + - name: beat + image: "SET-BY-CICD-IMAGE:SET-BY-CICD-TAG" + imagePullPolicy: IfNotPresent + command: + - celery + - -A + - main + - beat + - -l + - INFO + - --scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler + livenessProbe: + exec: + command: + - banjo-celery-probe + - --max-age + - "60" + failureThreshold: 3 + periodSeconds: 60 + timeoutSeconds: 5 + resources: + limits: + memory: 512Mi + requests: + cpu: "0.1" + memory: 256Mi + envFrom: + - secretRef: + name: go-api-secret + - configMapRef: + name: go-api-env-name + env: + - name: DJANGO_APP_TYPE + value: "worker" + - name: CACHE_REDIS_URL + value: redis://go-api-dragonfly:6379/1 + - name: CELERY_REDIS_URL + value: redis://go-api-dragonfly:6379/0 + - name: ELASTIC_SEARCH_HOST + value: http://go-api-elasticsearch-es-http:9200 + - name: PLAYWRIGHT_SERVER_URL + value: ws://go-api-playwright:3000/ + + - name: BANJO_CELERY_HEARTBEAT_FILE + value: "/var/run/celery/beat_heartbeat" + volumeMounts: + - name: go-api-secret + mountPath: /mnt/secrets-store + readOnly: true + - mountPath: /var/run/celery + name: celery-heartbeat + --- # Source: ifrcgo-helm/charts/app/templates/worker/deployment.yaml # Queue: default @@ -227,7 +331,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 8338ba490298cb58350d749c6fe8334b174149408ad4fec592f1a7360a91b4c5 + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 labels: app: go-api component: worker @@ -370,7 +474,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 8338ba490298cb58350d749c6fe8334b174149408ad4fec592f1a7360a91b4c5 + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 labels: app: go-api component: hook @@ -444,7 +548,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 8338ba490298cb58350d749c6fe8334b174149408ad4fec592f1a7360a91b4c5 + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 labels: app: go-api component: hook @@ -517,7 +621,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 8338ba490298cb58350d749c6fe8334b174149408ad4fec592f1a7360a91b4c5 + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 labels: app: go-api component: hook @@ -596,7 +700,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 8338ba490298cb58350d749c6fe8334b174149408ad4fec592f1a7360a91b4c5 + checksum/configmap: bd46cc3d30ba7d2fe19e56ce7943519f3a7e5f908a0159decb6b70d664896ff6 labels: app: go-api component: hook diff --git a/deploy/helm/snapshots/staging.yaml b/deploy/helm/snapshots/staging.yaml index 160960cd2..8dfdf7c3d 100644 --- a/deploy/helm/snapshots/staging.yaml +++ b/deploy/helm/snapshots/staging.yaml @@ -39,6 +39,8 @@ data: HEALTH_CHECK_DISK_USAGE_MAX: "none" JWT_EXPIRE_TIMESTAMP_DAYS: "365" OIDC_ENABLE: "true" + SENTRY_DEBUG: "false" + SENTRY_MONITOR_CELERY_BEAT_TASKS: "true" SENTRY_SAMPLE_RATE: "0.3" SESSION_COOKIE_DOMAIN: ".example.com" @@ -123,7 +125,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 3be62c602f9e6651c5d7889f2bef452a571a23bdbe335aad3762f40eda5da3a5 + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a labels: app: go-api component: api @@ -200,6 +202,108 @@ spec: mountPath: /mnt/secrets-store readOnly: true +--- +# Source: ifrcgo-helm/charts/app/templates/worker-addons/deployment.yaml +# Addon: beat +apiVersion: apps/v1 +kind: Deployment +metadata: + name: go-api-worker-beat + annotations: + argocd.argoproj.io/sync-wave: "30" + reloader.stakater.com/auto: "true" + labels: + app: go-api + component: worker-addon + addon: beat + environment: staging + release: release-name +spec: + replicas: 1 + strategy: + type: Recreate + revisionHistoryLimit: 1 + selector: + matchLabels: + app: go-api + component: worker-addon + addon: beat + template: + metadata: + annotations: + checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a + labels: + app: go-api + component: worker-addon + addon: beat + azure.workload.identity/use: "true" + spec: + volumes: + - name: go-api-secret + csi: + driver: "secrets-store.csi.k8s.io" + readOnly: true + volumeAttributes: + secretProviderClass: go-api-secret-provider + - emptyDir: + medium: Memory + sizeLimit: 1Mi + name: celery-heartbeat + serviceAccountName: service-token-reader + containers: + - name: beat + image: "SET-BY-CICD-IMAGE:SET-BY-CICD-TAG" + imagePullPolicy: IfNotPresent + command: + - celery + - -A + - main + - beat + - -l + - INFO + - --scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler + livenessProbe: + exec: + command: + - banjo-celery-probe + - --max-age + - "60" + failureThreshold: 3 + periodSeconds: 60 + timeoutSeconds: 5 + resources: + limits: + memory: 512Mi + requests: + cpu: "0.1" + memory: 256Mi + envFrom: + - secretRef: + name: go-api-secret + - configMapRef: + name: go-api-env-name + env: + - name: DJANGO_APP_TYPE + value: "worker" + - name: CACHE_REDIS_URL + value: redis://go-api-dragonfly:6379/1 + - name: CELERY_REDIS_URL + value: redis://go-api-dragonfly:6379/0 + - name: ELASTIC_SEARCH_HOST + value: http://go-api-elasticsearch-es-http:9200 + - name: PLAYWRIGHT_SERVER_URL + value: ws://go-api-playwright:3000/ + + - name: BANJO_CELERY_HEARTBEAT_FILE + value: "/var/run/celery/beat_heartbeat" + volumeMounts: + - name: go-api-secret + mountPath: /mnt/secrets-store + readOnly: true + - mountPath: /var/run/celery + name: celery-heartbeat + --- # Source: ifrcgo-helm/charts/app/templates/worker/deployment.yaml # Queue: default @@ -227,7 +331,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 3be62c602f9e6651c5d7889f2bef452a571a23bdbe335aad3762f40eda5da3a5 + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a labels: app: go-api component: worker @@ -370,7 +474,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 3be62c602f9e6651c5d7889f2bef452a571a23bdbe335aad3762f40eda5da3a5 + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a labels: app: go-api component: hook @@ -444,7 +548,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 3be62c602f9e6651c5d7889f2bef452a571a23bdbe335aad3762f40eda5da3a5 + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a labels: app: go-api component: hook @@ -517,7 +621,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 3be62c602f9e6651c5d7889f2bef452a571a23bdbe335aad3762f40eda5da3a5 + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a labels: app: go-api component: hook @@ -596,7 +700,7 @@ spec: metadata: annotations: checksum/secret: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/configmap: 3be62c602f9e6651c5d7889f2bef452a571a23bdbe335aad3762f40eda5da3a5 + checksum/configmap: e6f35c66012291b459eb96a466f917bdaf746967a108511a092aea7328f8058a labels: app: go-api component: hook diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index dd0568fc1..daf57848f 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -66,8 +66,46 @@ app: default: enabled: true celeryArgs: ["-Q", "default,heavy,cronjob", "--concurrency", "2"] - # No beat: go-api has no in-code schedule / django-celery-beat; scheduled work - # runs as CronJobs below (Sentry-monitored via main/sentry.py). + addons: + # Scheduler for the cronjobs declared in main/cronjobs.py. The legacy jobs + # still run as CronJobs below (Sentry-monitored via main/sentry.py); see + # docs/cronjobs.md for which mechanism a job belongs to. + beat: + enabled: true + # MUST stay 1 -- two beat processes fire every cronjob twice. + replicaCount: 1 + strategy: {type: Recreate} + # settings.py sets CELERY_BEAT_SCHEDULER to django-celery-beat's + # DatabaseScheduler. HeartbeatDatabaseScheduler is that same scheduler + # plus a heartbeat file each tick, so banjo-celery-probe can check liveness. + command: + - "celery" + - "-A" + - "main" + - "beat" + - "-l" + - "INFO" + - "--scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler" + extraEnv: + BANJO_CELERY_HEARTBEAT_FILE: /var/run/celery/beat_heartbeat + volumes: + celery-heartbeat: {emptyDir: {medium: Memory, sizeLimit: 1Mi}} + volumeMounts: + celery-heartbeat: {mountPath: /var/run/celery} + probes: + enabled: true + liveness: + # DatabaseScheduler ticks every ~5s, so a tight --max-age is fine. + exec: {command: [banjo-celery-probe, --max-age, "60"]} + periodSeconds: 60 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: "0.1" + memory: 256Mi + limits: + memory: 512Mi hooks: enabled: true @@ -226,6 +264,10 @@ app: # Object storage: Azure Blob is the deployment backend (pod filesystem is ephemeral) AZURE_STORAGE_ENABLED: "true" SENTRY_SAMPLE_RATE: "0.2" + SENTRY_DEBUG: "false" + # Auto-creates Sentry cron monitors for celery beat cronjobs. The legacy + # CronJobs are registered separately via `manage.py cron_job_monitor`. + SENTRY_MONITOR_CELERY_BEAT_TASKS: "true" secrets: {} diff --git a/docs/cronjobs.md b/docs/cronjobs.md index b604869fb..6d8200910 100644 --- a/docs/cronjobs.md +++ b/docs/cronjobs.md @@ -71,26 +71,38 @@ accepted and then never consumed by anything. docker-compose up celery celery-beat ``` -Worker and beat entrypoints live in `misc/dev/`. +Worker and beat entrypoints live in `misc/dev/`. Both wait for the database and +the broker first via banjo's `manage.py wait_for_resources`. -### Not deployed yet +### Deployment -**Beat currently runs in local development only.** There is no beat Deployment -in `deploy/helm/`, so nothing in `SCHEDULES` fires in alpha/staging/prod until -one is added. Still to do: +Beat runs as the `beat` worker addon in +[`deploy/helm/values.yaml`](../deploy/helm/values.yaml) (`app.worker.addons.beat`), +rendered by banjo-helm as a `-worker-beat` Deployment. -- A beat Deployment with **`replicas: 1`** and `strategy: Recreate` — two beat - processes fire every cronjob twice — plus a `celeryBeat` block in - `values.yaml`. It needs the same `envFrom` secret + configmap as the celery - worker. -- Beat needs the `django_celery_beat` tables, which `manage.py migrate` creates - on the API pod. If beat starts first it crashloops until migrations have run. +Two properties there are load-bearing: + +- **`replicaCount: 1` with `strategy: Recreate`.** Two beat processes fire every + cronjob twice, so this must never be scaled up. +- **`--scheduler=banjo_utils.celery_health.database.HeartbeatDatabaseScheduler`** — + django-celery-beat's `DatabaseScheduler` plus a heartbeat file each tick, which + is what the `banjo-celery-probe` liveness check reads. + +Beat needs the `django_celery_beat` tables, created by `manage.py migrate` in the +deploy hook. If beat starts first it crashloops until migrations have run. + +After changing `SCHEDULES`, regenerate the chart snapshots: + +```bash +cd deploy/helm && ./update-snapshots.sh +``` ## 2. Kubernetes CronJobs — the legacy set -The pre-existing cronjobs run as k8s CronJob resources listed under `cronjobs:` -in `deploy/helm/ifrcgo-helm/values.yaml`, one pod per run, monitored via -`SentryMonitor` in `main/sentry.py`. Their Sentry monitors are registered with: +The pre-existing cronjobs run as k8s CronJob resources listed under +`app.cronjobs.jobs` in `deploy/helm/values.yaml`, one pod per run, monitored via +`SentryMonitor` in `main/sentry.py`. Their Sentry monitors are **not** created +automatically — they must be registered per environment with: ```bash docker-compose exec serve bash ./manage.py cron_job_monitor From 522af1f06270733c1f3635501546daecdc356805 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 9 Sep 2026 12:26:02 +0545 Subject: [PATCH 4/4] refactor(celery): wait for resources via banjo Replaces misc/wait-for-broker.sh with banjo's `manage.py wait_for_resources --db --celery-broker`, which also waits for the database instead of only the broker. --- api/tasks.py | 1 - api/views.py | 2 +- misc/dev/run_worker.sh | 3 +-- misc/dev/run_worker_beat.sh | 4 +++- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/tasks.py b/api/tasks.py index bad694ef8..077a99a1c 100644 --- a/api/tasks.py +++ b/api/tasks.py @@ -63,7 +63,6 @@ def generate_export_pdf(export_id, title, set_user_language="en"): logger.info(f"End export: {export.pk}") -# TODO(susilnem): Do we need this cron? @shared_task def clear_expired_django_sessions(): """Purge expired django_session rows -- nothing else prunes them.""" diff --git a/api/views.py b/api/views.py index 9cb82580e..001084894 100644 --- a/api/views.py +++ b/api/views.py @@ -558,7 +558,7 @@ def get(cls, request): "cronjob_err": c, "maintenance_mode": settings.DJANGO_READ_ONLY, "git_last_tag": settings.LAST_GIT_TAG, - "git_last_commit": settings.SENTRY_CONFIG["release"][0:8], + "git_last_commit": settings.SENTRY_CONFIG.release[0:8], } return JsonResponse(res, safe=False) diff --git a/misc/dev/run_worker.sh b/misc/dev/run_worker.sh index 0553b971c..50b9e1a8c 100755 --- a/misc/dev/run_worker.sh +++ b/misc/dev/run_worker.sh @@ -1,7 +1,6 @@ #!/bin/bash -e # Uses the management command for Django's autoreloader. -# TODO(susilnem): Use wait_for_resources once banjo-stack is implemented -./misc/wait-for-broker.sh +./manage.py wait_for_resources --db --celery-broker exec ./manage.py run_celery_dev diff --git a/misc/dev/run_worker_beat.sh b/misc/dev/run_worker_beat.sh index addd48af6..27a50fe38 100755 --- a/misc/dev/run_worker_beat.sh +++ b/misc/dev/run_worker_beat.sh @@ -1,5 +1,7 @@ #!/bin/bash -e -./misc/wait-for-broker.sh +./manage.py wait_for_resources --db --celery-broker +# Prod uses HeartbeatDatabaseScheduler so banjo-celery-probe can check liveness +# (see deploy/helm/values.yaml); locally the plain DatabaseScheduler is enough. exec celery -A main beat -l INFO