diff --git a/acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl new file mode 100644 index 0000000000..c63034835c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/databricks.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + pipelines: + my_pipeline: + name: test-pipeline-$UNIQUE_NAME + root_path: ./pipeline_root + libraries: + - notebook: + path: /Users/{{workspace_user_name}}/notebook + + jobs: + my_job: + tasks: + - task_key: main + notebook_task: + notebook_path: ./src/notebook.py + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + +targets: + default: + mode: development diff --git a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml new file mode 100644 index 0000000000..c4500f378d --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false + +[GOOS] + windows = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt new file mode 100644 index 0000000000..c4ce29efa0 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -0,0 +1,60 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Set correct paths and add git info remotely +=== Break local paths to simulate stale config +=== Sync with broken local paths +Detected changes in 2 resource(s): + +Resource: resources.jobs.my_job + git_source: add + tasks[task_key='main'].notebook_task.notebook_path: replace + +Resource: resources.pipelines.my_pipeline + root_path: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -6,5 +6,5 @@ + my_pipeline: + name: test-pipeline-[UNIQUE_NAME] +- root_path: ./pipeline_root ++ root_path: ./pipeline_root_v2 + libraries: + - notebook: +@@ -16,9 +16,13 @@ + - task_key: main + notebook_task: +- notebook_path: ./src/notebook.py ++ notebook_path: /Users/[USERNAME]/notebook + new_cluster: + spark_version: 13.3.x-snapshot-scala2.12 + node_type_id: [NODE_TYPE_ID] + num_workers: 1 ++ git_source: ++ git_branch: main ++ git_provider: gitHub ++ git_url: https://github.com/databricks/databricks-sdk-go.git + + targets: + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/validation_errors/script b/acceptance/bundle/config-remote-sync/validation_errors/script new file mode 100644 index 0000000000..828ec20644 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/validation_errors/script @@ -0,0 +1,48 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# Create valid paths so that initial deploy succeeds +mkdir -p pipeline_root src +echo '# Databricks notebook source' > src/notebook.py + +cleanup() { + # Restore valid paths for destroy to work (includes pipeline_root_v2 from sync) + mkdir -p pipeline_root pipeline_root_v2 src + echo '# Databricks notebook source' > src/notebook.py + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" +pipeline_id="$(read_id.py my_pipeline)" + + +title "Set correct paths and add git info remotely" +edit_resource.py pipelines $pipeline_id <