From c7a3a2a5df4ae0c4ec50b0807b113190be275662 Mon Sep 17 00:00:00 2001 From: Mohammad Atallah Date: Fri, 21 Aug 2026 06:31:02 -0400 Subject: [PATCH] fix(cli): make agent autoscaling opt-in instead of a hardcoded default A crash-looping pod's restart CPU burn reads as CPU load, so the HPA this CLI attached to every deployment scaled the failure to maxReplicas, then froze there once the metrics went missing. Keep the autoscaling block so an environments.yaml opt-in still deep-merges onto the existing replica bounds and CPU target. Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/cli/handlers/deploy_handlers.py | 7 +- .../default-claude-code/environments.yaml.j2 | 3 + .../default-codex/environments.yaml.j2 | 3 + .../default-langgraph/environments.yaml.j2 | 3 + .../environments.yaml.j2 | 3 + .../default-pydantic-ai/environments.yaml.j2 | 3 + .../templates/default/environments.yaml.j2 | 3 + .../sync-claude-code/environments.yaml.j2 | 3 + .../templates/sync-codex/environments.yaml.j2 | 3 + .../sync-langgraph/environments.yaml.j2 | 3 + .../environments.yaml.j2 | 3 + .../sync-openai-agents/environments.yaml.j2 | 3 + .../sync-pydantic-ai/environments.yaml.j2 | 3 + .../cli/templates/sync/environments.yaml.j2 | 3 + .../temporal-claude-code/environments.yaml.j2 | 8 +- .../temporal-codex/environments.yaml.j2 | 8 +- .../temporal-langgraph/environments.yaml.j2 | 8 +- .../environments.yaml.j2 | 8 +- .../temporal-pydantic-ai/environments.yaml.j2 | 8 +- .../templates/temporal/environments.yaml.j2 | 8 +- tests/lib/cli/test_deploy_handlers.py | 98 +++++++++++++++++++ 21 files changed, 182 insertions(+), 10 deletions(-) create mode 100644 tests/lib/cli/test_deploy_handlers.py diff --git a/src/agentex/lib/cli/handlers/deploy_handlers.py b/src/agentex/lib/cli/handlers/deploy_handlers.py index 605d91709..4586e4967 100644 --- a/src/agentex/lib/cli/handlers/deploy_handlers.py +++ b/src/agentex/lib/cli/handlers/deploy_handlers.py @@ -292,9 +292,9 @@ def merge_deployment_configs( "memory": manifest.deployment.global_config.resources.limits.memory, }, }, - # Enable autoscaling by default for production deployments + # Opt-in only: a crash loop's restart CPU burn reads as load and scales it to max. "autoscaling": { - "enabled": True, + "enabled": False, "minReplicas": 1, "maxReplicas": 10, "targetCPUUtilizationPercentage": 50, @@ -307,9 +307,8 @@ def merge_deployment_configs( if temporal_config: helm_values[TEMPORAL_WORKER_KEY] = { "enabled": True, - # Enable autoscaling for temporal workers as well "autoscaling": { - "enabled": True, + "enabled": False, "minReplicas": 1, "maxReplicas": 10, "targetCPUUtilizationPercentage": 50, diff --git a/src/agentex/lib/cli/templates/default-claude-code/environments.yaml.j2 b/src/agentex/lib/cli/templates/default-claude-code/environments.yaml.j2 index f802776f0..991a6485f 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/environments.yaml.j2 @@ -51,6 +51,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal: enabled: false diff --git a/src/agentex/lib/cli/templates/default-codex/environments.yaml.j2 b/src/agentex/lib/cli/templates/default-codex/environments.yaml.j2 index f802776f0..991a6485f 100644 --- a/src/agentex/lib/cli/templates/default-codex/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/default-codex/environments.yaml.j2 @@ -51,6 +51,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal: enabled: false diff --git a/src/agentex/lib/cli/templates/default-langgraph/environments.yaml.j2 b/src/agentex/lib/cli/templates/default-langgraph/environments.yaml.j2 index f802776f0..991a6485f 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/environments.yaml.j2 @@ -51,6 +51,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal: enabled: false diff --git a/src/agentex/lib/cli/templates/default-openai-agents/environments.yaml.j2 b/src/agentex/lib/cli/templates/default-openai-agents/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/environments.yaml.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/environments.yaml.j2 index f802776f0..991a6485f 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/environments.yaml.j2 @@ -51,6 +51,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal: enabled: false diff --git a/src/agentex/lib/cli/templates/default/environments.yaml.j2 b/src/agentex/lib/cli/templates/default/environments.yaml.j2 index f802776f0..991a6485f 100644 --- a/src/agentex/lib/cli/templates/default/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/default/environments.yaml.j2 @@ -51,6 +51,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal: enabled: false diff --git a/src/agentex/lib/cli/templates/sync-claude-code/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync-claude-code/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/sync-codex/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync-codex/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync-codex/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/sync-langgraph/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync-langgraph/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/sync/environments.yaml.j2 b/src/agentex/lib/cli/templates/sync/environments.yaml.j2 index 73924abdd..d1de34603 100644 --- a/src/agentex/lib/cli/templates/sync/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/sync/environments.yaml.j2 @@ -50,4 +50,7 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/environments.yaml.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/environments.yaml.j2 index a3df5e228..0390a04b4 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/environments.yaml.j2 @@ -52,6 +52,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal-worker: enabled: true replicaCount: 2 @@ -61,4 +64,7 @@ environments: memory: "1Gi" limits: cpu: "1000m" - memory: "2Gi" \ No newline at end of file + memory: "2Gi" + # Autoscaling here is separate from the block above, so opt in twice. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/temporal-codex/environments.yaml.j2 b/src/agentex/lib/cli/templates/temporal-codex/environments.yaml.j2 index a3df5e228..0390a04b4 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/environments.yaml.j2 @@ -52,6 +52,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal-worker: enabled: true replicaCount: 2 @@ -61,4 +64,7 @@ environments: memory: "1Gi" limits: cpu: "1000m" - memory: "2Gi" \ No newline at end of file + memory: "2Gi" + # Autoscaling here is separate from the block above, so opt in twice. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/environments.yaml.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/environments.yaml.j2 index a3df5e228..0390a04b4 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/environments.yaml.j2 @@ -52,6 +52,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal-worker: enabled: true replicaCount: 2 @@ -61,4 +64,7 @@ environments: memory: "1Gi" limits: cpu: "1000m" - memory: "2Gi" \ No newline at end of file + memory: "2Gi" + # Autoscaling here is separate from the block above, so opt in twice. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/environments.yaml.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/environments.yaml.j2 index a3df5e228..0390a04b4 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/environments.yaml.j2 @@ -52,6 +52,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal-worker: enabled: true replicaCount: 2 @@ -61,4 +64,7 @@ environments: memory: "1Gi" limits: cpu: "1000m" - memory: "2Gi" \ No newline at end of file + memory: "2Gi" + # Autoscaling here is separate from the block above, so opt in twice. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/environments.yaml.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/environments.yaml.j2 index a3df5e228..0390a04b4 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/environments.yaml.j2 @@ -52,6 +52,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal-worker: enabled: true replicaCount: 2 @@ -61,4 +64,7 @@ environments: memory: "1Gi" limits: cpu: "1000m" - memory: "2Gi" \ No newline at end of file + memory: "2Gi" + # Autoscaling here is separate from the block above, so opt in twice. + # autoscaling: + # enabled: true diff --git a/src/agentex/lib/cli/templates/temporal/environments.yaml.j2 b/src/agentex/lib/cli/templates/temporal/environments.yaml.j2 index a3df5e228..0390a04b4 100644 --- a/src/agentex/lib/cli/templates/temporal/environments.yaml.j2 +++ b/src/agentex/lib/cli/templates/temporal/environments.yaml.j2 @@ -52,6 +52,9 @@ environments: limits: cpu: "1000m" memory: "2Gi" + # Autoscaling is opt-in. Unset keys inherit maxReplicas 10 and a 50% CPU target. + # autoscaling: + # enabled: true temporal-worker: enabled: true replicaCount: 2 @@ -61,4 +64,7 @@ environments: memory: "1Gi" limits: cpu: "1000m" - memory: "2Gi" \ No newline at end of file + memory: "2Gi" + # Autoscaling here is separate from the block above, so opt in twice. + # autoscaling: + # enabled: true diff --git a/tests/lib/cli/test_deploy_handlers.py b/tests/lib/cli/test_deploy_handlers.py new file mode 100644 index 000000000..1727cee67 --- /dev/null +++ b/tests/lib/cli/test_deploy_handlers.py @@ -0,0 +1,98 @@ +"""Tests for helm value merging in deploy_handlers.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from agentex.config.agent_config import AgentConfig +from agentex.config.build_config import BuildConfig, BuildContext +from agentex.config.agent_configs import TemporalConfig, TemporalWorkflowConfig +from agentex.config.agent_manifest import AgentManifest +from agentex.config.deployment_config import ImageConfig, DeploymentConfig +from agentex.config.environment_config import AgentAuthConfig, AgentEnvironmentConfig +from agentex.lib.cli.handlers.deploy_handlers import ( + TEMPORAL_WORKER_KEY, + InputDeployOverrides, + merge_deployment_configs, +) + + +def _manifest(*, temporal: bool) -> AgentManifest: + return AgentManifest( + build=BuildConfig(context=BuildContext(root=".", dockerfile="Dockerfile", dockerignore=None)), + agent=AgentConfig( + name="test-agent", + acp_type="async", + description="An AgentEx agent", + temporal=TemporalConfig( + enabled=True, + workflows=[TemporalWorkflowConfig(name="test-agent", queue_name="test_agent_queue")], + ) + if temporal + else None, + ), + deployment=DeploymentConfig(image=ImageConfig(repository="registry.example.com/test-agent", tag="v1")), + ) + + +def _env_config(helm_overrides: dict[str, Any] | None = None) -> AgentEnvironmentConfig: + return AgentEnvironmentConfig( + auth=AgentAuthConfig(principal={"user_id": "test-user", "account_id": "test-account"}), + helm_overrides=helm_overrides or {}, + ) + + +def _merge(*, temporal: bool = False, env_config: AgentEnvironmentConfig | None = None) -> dict[str, Any]: + return merge_deployment_configs( + manifest=_manifest(temporal=temporal), + agent_env_config=env_config, + deploy_overrides=InputDeployOverrides(), + manifest_path="manifest.yaml", + ) + + +class TestAutoscalingDefaults: + """Autoscaling is opt-in, and the bounds stay available for whoever opts in.""" + + def test_disabled_by_default(self): + assert _merge()["autoscaling"]["enabled"] is False + + def test_disabled_by_default_for_temporal_worker(self): + helm_values = _merge(temporal=True) + assert helm_values[TEMPORAL_WORKER_KEY]["autoscaling"]["enabled"] is False + + @pytest.mark.parametrize( + "key,expected", [("minReplicas", 1), ("maxReplicas", 10), ("targetCPUUtilizationPercentage", 50)] + ) + def test_bounds_survive_an_opt_in(self, key: str, expected: int): + """An opt-in that sets only `enabled` must still inherit the CPU target and replica bounds.""" + helm_values = _merge(env_config=_env_config({"autoscaling": {"enabled": True}})) + + assert helm_values["autoscaling"]["enabled"] is True + assert helm_values["autoscaling"][key] == expected + + def test_temporal_worker_opts_in_separately(self): + """The worker HPA is its own block, so opting the agent in leaves the worker off.""" + helm_values = _merge( + temporal=True, + env_config=_env_config({"autoscaling": {"enabled": True}}), + ) + + assert helm_values["autoscaling"]["enabled"] is True + assert helm_values[TEMPORAL_WORKER_KEY]["autoscaling"]["enabled"] is False + + opted_in = _merge( + temporal=True, + env_config=_env_config({TEMPORAL_WORKER_KEY: {"autoscaling": {"enabled": True}}}), + ) + + assert opted_in[TEMPORAL_WORKER_KEY]["autoscaling"]["enabled"] is True + assert opted_in[TEMPORAL_WORKER_KEY]["autoscaling"]["targetCPUUtilizationPercentage"] == 50 + + def test_opt_in_overrides_win_over_the_base(self): + helm_values = _merge(env_config=_env_config({"autoscaling": {"enabled": True, "maxReplicas": 3}})) + + assert helm_values["autoscaling"]["maxReplicas"] == 3 + assert helm_values["autoscaling"]["targetCPUUtilizationPercentage"] == 50