From 93406f8b74cca6d1adb6239a4de4b3f3299adb19 Mon Sep 17 00:00:00 2001 From: AnOtterGithubUser Date: Fri, 10 Mar 2023 18:55:31 +0100 Subject: [PATCH 1/4] [#1466] Add --includeAllTags flag to filter actions which include all tags in --tag option --- api/commands/prune.ts | 16 +++++++++++++--- cli/index.ts | 13 ++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/api/commands/prune.ts b/api/commands/prune.ts index 4da4d39bc..d3c85913f 100644 --- a/api/commands/prune.ts +++ b/api/commands/prune.ts @@ -47,6 +47,7 @@ function computeIncludedActionNames( const hasActionSelector = runConfig.actions?.length > 0; const hasTagSelector = runConfig.tags?.length > 0; + const hasIncludeAllTagsSelector = runConfig.includeAllTags; // If no selectors, return all actions. if (!hasActionSelector && !hasTagSelector) { @@ -64,9 +65,18 @@ function computeIncludedActionNames( // Determine actions selected with --tag option and update applicable actions if (hasTagSelector) { - allActions - .filter(action => action.tags.some(tag => runConfig.tags.includes(tag))) - .forEach(action => includedActionNames.add(targetAsReadableString(action.target))); + // Keep actions wich include every tags in --tag option + if (hasIncludeAllTagsSelector) { + allActions + .filter(action => runConfig.tags.every(tag => action.tags.includes(tag))) + .forEach(action => includedActionNames.add(targetAsReadableString(action.target))); + } + // Keep actions with include at least one tag in --tag option + else { + allActions + .filter(action => action.tags.some(tag => runConfig.tags.includes(tag))) + .forEach(action => includedActionNames.add(targetAsReadableString(action.target))); + } } // Compute all transitive dependencies. diff --git a/cli/index.ts b/cli/index.ts index fc8f59d6a..c8e7ae7f4 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -98,6 +98,15 @@ const tagsOption: INamedOption = { } }; +const IncludeAllTagsOption: INamedOption = { + name: "include-all-tags", + option: { + describe: "Filter actions which include all user-specified tags", + type: "boolean", + default: false + } +} + const includeDepsOption: INamedOption = { name: "include-deps", option: { @@ -581,6 +590,7 @@ export function runCli() { fullRefreshOption, actionsOption, tagsOption, + IncludeAllTagsOption, includeDepsOption, includeDependentsOption, schemaSuffixOverrideOption, @@ -628,7 +638,8 @@ export function runCli() { actions: argv[actionsOption.name], includeDependencies: argv[includeDepsOption.name], includeDependents: argv[includeDependentsOption.name], - tags: argv[tagsOption.name] + tags: argv[tagsOption.name], + includeAllTags: argv[IncludeAllTagsOption.name] }, dbadapter ); From 676f151f198f576136889cc428d7c66202995af2 Mon Sep 17 00:00:00 2001 From: AnOtterGithubUser Date: Fri, 10 Mar 2023 19:12:42 +0100 Subject: [PATCH 2/4] [#1466] Add includeAllTags to execution.proto --- api/commands/prune.ts | 2 +- protos/execution.proto | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/commands/prune.ts b/api/commands/prune.ts index d3c85913f..9a59a9e37 100644 --- a/api/commands/prune.ts +++ b/api/commands/prune.ts @@ -47,7 +47,7 @@ function computeIncludedActionNames( const hasActionSelector = runConfig.actions?.length > 0; const hasTagSelector = runConfig.tags?.length > 0; - const hasIncludeAllTagsSelector = runConfig.includeAllTags; + const hasIncludeAllTagsSelector = runConfig.includeAllTags?.valueOf(); // If no selectors, return all actions. if (!hasActionSelector && !hasTagSelector) { diff --git a/protos/execution.proto b/protos/execution.proto index c7d1e2fad..cc644e603 100644 --- a/protos/execution.proto +++ b/protos/execution.proto @@ -10,6 +10,7 @@ option go_package = "github.com/dataform-co/dataform/protos/dataform"; message RunConfig { repeated string actions = 1; repeated string tags = 5; + bool include_all_tags = 10; bool include_dependencies = 3; bool include_dependents = 11; bool full_refresh = 2; From b3cfb1ba1e493014f102d32a8f6179b49de95679 Mon Sep 17 00:00:00 2001 From: AnOtterGithubUser Date: Sat, 11 Mar 2023 13:09:54 +0100 Subject: [PATCH 3/4] [#1466] Lowercase option and change description --- cli/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/index.ts b/cli/index.ts index c8e7ae7f4..5d82cec0a 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -98,10 +98,10 @@ const tagsOption: INamedOption = { } }; -const IncludeAllTagsOption: INamedOption = { +const includeAllTagsOption: INamedOption = { name: "include-all-tags", option: { - describe: "Filter actions which include all user-specified tags", + describe: "If set, only actions that include every user-specified tags will be run", type: "boolean", default: false } @@ -590,7 +590,7 @@ export function runCli() { fullRefreshOption, actionsOption, tagsOption, - IncludeAllTagsOption, + includeAllTagsOption, includeDepsOption, includeDependentsOption, schemaSuffixOverrideOption, @@ -639,7 +639,7 @@ export function runCli() { includeDependencies: argv[includeDepsOption.name], includeDependents: argv[includeDependentsOption.name], tags: argv[tagsOption.name], - includeAllTags: argv[IncludeAllTagsOption.name] + includeAllTags: argv[includeAllTagsOption.name] }, dbadapter ); From a05c4f8b7dc1ff3f2ff7c325ad899064e41d4cd1 Mon Sep 17 00:00:00 2001 From: AnOtterGithubUser Date: Sat, 11 Mar 2023 13:48:09 +0100 Subject: [PATCH 4/4] [#1466] Add test case for new flag --- tests/api/api.spec.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/api/api.spec.ts b/tests/api/api.spec.ts index 42a3842f2..b571ee913 100644 --- a/tests/api/api.spec.ts +++ b/tests/api/api.spec.ts @@ -291,6 +291,11 @@ suite("@dataform/api", () => { target: { schema: "schema", name: "op_d" }, tags: ["tag3"], queries: ["create or replace view schema.someview as select 1 as test"] + }, + { + target: { schema: "schema", name: "op_e" }, + tags: ["tag1", "tag2"], + queries: ["create or replace view schema.someview as select 1 as test"] } ], tables: [ @@ -395,6 +400,23 @@ suite("@dataform/api", () => { expect(actionNames).includes("schema.tab_a"); }); + test("prune actions with --tags (with include all tags)", () => { + const prunedGraph = prune(TEST_GRAPH_WITH_TAGS, { + tags: ["tag1", "tag2"], + includeAllTags: true + }); + const actionNames = [ + ...prunedGraph.tables.map(action => targetAsReadableString(action.target)), + ...prunedGraph.operations.map(action => targetAsReadableString(action.target)) + ]; + expect(actionNames).includes("schema.op_e"); + expect(actionNames).includes("schema.tab_a"); + expect(actionNames).not.includes("schema.op_a"); + expect(actionNames).not.includes("schema.op_b"); + expect(actionNames).not.includes("schema.op_c"); + expect(actionNames).not.includes("schema.op_d"); + }); + test("prune actions with --actions with dependencies", () => { const prunedGraph = prune(TEST_GRAPH, { actions: ["schema.a"], includeDependencies: true }); const actionNames = [