Add Cluster Policy support to bundle resources

.nextchanges/bundles/cluster-policies.md

@@ -0,0 +1 @@
+Add support for the `cluster_policies` resource type in Declarative Automation Bundles. Cluster policies are only supported in direct deployment mode.

acceptance/bundle/invariant/configs/cluster_policy.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    foo:
+      name: test-cluster-policy-$UNIQUE_NAME
+      definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}'

acceptance/bundle/invariant/continue_293/test.toml

@@ -16,6 +16,9 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 # instance_pools resource is not supported on v0.293.0
 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
 
+# cluster_policies resource is not supported on v0.293.0
+EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
+
 # job_runs resource is not supported on v0.293.0
 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
 

acceptance/bundle/invariant/migrate/test.toml

@@ -14,6 +14,8 @@ EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmp
 EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 # Instance pools are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
+# Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/test.toml

@@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [
     "catalog_optional_fields.yml.tmpl",
     "cluster.yml.tmpl",
     "cluster_apply_policy_default_values.yml.tmpl",
+    "cluster_policy.yml.tmpl",
     "dashboard.yml.tmpl",
     "job_apply_policy_default_values_job_cluster.yml.tmpl",
     "job_apply_policy_default_values_task_cluster.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -279,6 +279,40 @@ resources.catalogs.*.grants[*]	catalog.PrivilegeAssignment	ALL
 resources.catalogs.*.grants[*].principal	string	ALL
 resources.catalogs.*.grants[*].privileges	[]catalog.Privilege	ALL
 resources.catalogs.*.grants[*].privileges[*]	catalog.Privilege	ALL
+resources.cluster_policies.*.created_at_timestamp	int64	REMOTE
+resources.cluster_policies.*.creator_user_name	string	REMOTE
+resources.cluster_policies.*.definition	any	INPUT
+resources.cluster_policies.*.definition	string	ALL
+resources.cluster_policies.*.description	string	ALL
+resources.cluster_policies.*.id	string	INPUT
+resources.cluster_policies.*.is_default	bool	REMOTE
+resources.cluster_policies.*.libraries	[]compute.Library	ALL
+resources.cluster_policies.*.libraries[*]	compute.Library	ALL
+resources.cluster_policies.*.libraries[*].cran	*compute.RCranLibrary	ALL
+resources.cluster_policies.*.libraries[*].cran.package	string	ALL
+resources.cluster_policies.*.libraries[*].cran.repo	string	ALL
+resources.cluster_policies.*.libraries[*].egg	string	ALL
+resources.cluster_policies.*.libraries[*].jar	string	ALL
+resources.cluster_policies.*.libraries[*].maven	*compute.MavenLibrary	ALL
+resources.cluster_policies.*.libraries[*].maven.coordinates	string	ALL
+resources.cluster_policies.*.libraries[*].maven.exclusions	[]string	ALL
+resources.cluster_policies.*.libraries[*].maven.exclusions[*]	string	ALL
+resources.cluster_policies.*.libraries[*].maven.repo	string	ALL
+resources.cluster_policies.*.libraries[*].pypi	*compute.PythonPyPiLibrary	ALL
+resources.cluster_policies.*.libraries[*].pypi.package	string	ALL
+resources.cluster_policies.*.libraries[*].pypi.repo	string	ALL
+resources.cluster_policies.*.libraries[*].requirements	string	ALL
+resources.cluster_policies.*.libraries[*].whl	string	ALL
+resources.cluster_policies.*.lifecycle	resources.Lifecycle	INPUT
+resources.cluster_policies.*.lifecycle.prevent_destroy	bool	INPUT
+resources.cluster_policies.*.max_clusters_per_user	int64	ALL
+resources.cluster_policies.*.modified_status	string	INPUT
+resources.cluster_policies.*.name	string	ALL
+resources.cluster_policies.*.policy_family_definition_overrides	any	INPUT
+resources.cluster_policies.*.policy_family_definition_overrides	string	ALL
+resources.cluster_policies.*.policy_family_id	string	ALL
+resources.cluster_policies.*.policy_id	string	REMOTE
+resources.cluster_policies.*.url	string	INPUT
 resources.clusters.*.apply_policy_default_values	bool	ALL
 resources.clusters.*.autoscale	*compute.AutoScale	ALL
 resources.clusters.*.autoscale.max_workers	int	ALL

acceptance/bundle/resources/cluster_policies/backend_normalization/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: cluster_policy_backend_normalization
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    # Definition as a compact JSON string.
+    json_string:
+      name: probe_json_string-$UNIQUE_NAME
+      definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}'
+
+    # Definition as a multiline block-scalar JSON string.
+    multiline:
+      name: probe_multiline-$UNIQUE_NAME
+      definition: |-
+        {
+          "spark_version": {
+            "type": "fixed",
+            "value": "13.3.x-scala2.12"
+          }
+        }
+
+    # Definition as native YAML (normalized to compact JSON by the mutator before deploy).
+    yaml:
+      name: probe_yaml-$UNIQUE_NAME
+      definition:
+        spark_version:
+          type: fixed
+          value: 13.3.x-scala2.12
+        autotermination_minutes:
+          type: range
+          maxValue: 120

acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt

@@ -0,0 +1,39 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.json_string
+Created cluster_policies.multiline
+Created cluster_policies.yaml
+Files: 5 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+=== [json_string] definition returned by the backend (GET)
+>>> [CLI] cluster-policies get [JSON_STRING_ID] -o json
+{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}
+
+=== [multiline] definition returned by the backend (GET)
+>>> [CLI] cluster-policies get [MULTILINE_ID] -o json
+{
+  "spark_version": {
+    "type": "fixed",
+    "value": "13.3.x-scala2.12"
+  }
+}
+
+=== [yaml] definition returned by the backend (GET)
+>>> [CLI] cluster-policies get [YAML_ID] -o json
+{"autotermination_minutes":{"maxValue":120,"type":"range"},"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}
+
+=== Plan after deploy: any drift means the backend reformatted a definition
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.json_string
+  delete resources.cluster_policies.multiline
+  delete resources.cluster_policies.yaml
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 3 deleted

acceptance/bundle/resources/cluster_policies/backend_normalization/script

@@ -0,0 +1,21 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+# For each authoring style, print the definition string the backend returns on GET.
+# Compare it against what we sent to see whether the backend reformats the JSON.
+for name in json_string multiline yaml; do
+    pol_id=$(read_id.py "$name")
+    title "[$name] definition returned by the backend (GET)"
+    trace $CLI cluster-policies get "$pol_id" -o json | jq -r '.definition'
+done
+
+# yaml is normalized to compact JSON by the mutator, matching the string the backend
+# stores and returns, so it does not drift (plan below is a no-op).
+title "Plan after deploy: any drift means the backend reformatted a definition"
+trace $CLI bundle plan

acceptance/bundle/resources/cluster_policies/backend_normalization/test.toml

@@ -0,0 +1,6 @@
+RecordRequests = false
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]

acceptance/bundle/resources/cluster_policies/basic/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: test_cluster_policy
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    test_cluster_policy:
+      name: my_cluster_policy-$UNIQUE_NAME
+      definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}'

acceptance/bundle/resources/cluster_policies/basic/output.txt

@@ -0,0 +1,125 @@
+
+>>> [CLI] bundle validate
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Validation OK!
+
+>>> [CLI] bundle validate -o json
+{
+  "test_cluster_policy": {
+    "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_cluster_policy-[UNIQUE_NAME]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy-[UNIQUE_NAME]
+      URL:  (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.test_cluster_policy
+Files: 5 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Verify the create request
+>>> print_requests.py //policies/clusters/create
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/create",
+  "body": {
+    "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_cluster_policy-[UNIQUE_NAME]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy-[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID]
+
+=== Update the cluster policy name
+>>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Updated cluster_policies.test_cluster_policy
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+=== Verify the update request
+>>> print_requests.py //policies/clusters/edit
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/edit",
+  "body": {
+    "definition": "{\"spark_version\":{\"type\":\"fixed\",\"value\":\"13.3.x-scala2.12\"}}",
+    "name": "my_cluster_policy_2-[UNIQUE_NAME]",
+    "policy_id": "[TEST_CLUSTER_POLICY_ID]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy_2-[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[NUMID]
+
+=== Destroy the cluster policy
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.test_cluster_policy
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted
+
+=== Verify the destroy request
+>>> print_requests.py //policies/clusters/delete
+{
+  "method": "POST",
+  "path": "/api/2.0/policies/clusters/delete",
+  "body": {
+    "policy_id": "[TEST_CLUSTER_POLICY_ID]"
+  }
+}
+
+>>> [CLI] bundle summary
+Name: test_cluster_policy
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+Resources:
+  Cluster Policies:
+    test_cluster_policy:
+      Name: my_cluster_policy_2-[UNIQUE_NAME]
+      URL:  (not deployed)
+
+>>> [CLI] bundle destroy --auto-approve
+No active deployment found to destroy!

acceptance/bundle/resources/cluster_policies/basic/script

@@ -0,0 +1,39 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+trace $CLI bundle validate
+trace $CLI bundle validate -o json | jq ".resources.cluster_policies"
+
+trace $CLI bundle summary
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+trace $CLI bundle deploy
+
+# Mask the server-assigned policy id: the real backend returns a hex id that the
+# built-in UUID replacement misses, so register it explicitly for cloud parity.
+read_id.py test_cluster_policy > /dev/null
+
+title "Verify the create request"
+trace print_requests.py //policies/clusters/create
+
+trace $CLI bundle summary
+
+title "Update the cluster policy name"
+trace update_file.py databricks.yml my_cluster_policy my_cluster_policy_2
+trace $CLI bundle deploy
+
+title "Verify the update request"
+trace print_requests.py //policies/clusters/edit
+
+trace $CLI bundle summary
+
+title "Destroy the cluster policy"
+trace $CLI bundle destroy --auto-approve
+
+title "Verify the destroy request"
+trace print_requests.py //policies/clusters/delete
+
+trace $CLI bundle summary