new: Add more task input types. (#2128)

* Add new input type.

* Implement affected.

* Fix tests.

* Add affected tests.

* Update docs.

* Added ^ support.

* Rename enum.

* Start on group input.

* Rename file variant.

* Rename file variant.

* Fixes.

* Polish.

* Add expander tests.

Author: milesj

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 ## Unreleased
 
+#### 🚀 Updates
+
+- Added a new task input type that allows you to depend on changes to a project's files directly,
+  instead of depending on a project's task.
+  - Added URI support: `project://<id>`
+  - Added object support: `project: '<id>'`
+  - Can filter with globs or by file group.
+  - Can reference all project dependencies with `^`.
+- Added a new task input type for referencing file groups within the owning project.
+  - Added URI support: `group://<name>`
+  - Added object support: `group: '<name>'`
+  - This is similar to the `@files`, `@globs`, etc, token functions.
+
 #### ⚙️ Internal
 
 - Updated proto to [v0.53.0](https://github.com/moonrepo/proto/releases/tag/v0.53.0) (from 0.52.3).

Cargo.lock

@@ -71,7 +71,7 @@ starbase_archive = { version = "0.11.6", default-features = false, features = [
 ] }
 starbase_console = { version = "0.6.13", features = ["miette"] }
 starbase_events = "0.7.3"
-starbase_id = { version = "0.2.2", features = ["miette", "schematic"] }
+starbase_id = { version = "0.2.3", features = ["miette", "schematic"] }
 starbase_sandbox = "0.9.7"
 starbase_shell = "0.10.2"
 starbase_styles = { version = "0.6.1", features = ["relative-path"] }

Cargo.toml

@@ -14,11 +14,11 @@ use moon_common::{Id, color};
 use moon_config::{PipelineActionSwitch, TaskDependencyConfig};
 use moon_pdk_api::{DefineRequirementsInput, LocateDependenciesRootInput};
 use moon_platform::{PlatformManager, Runtime, ToolchainSpec};
-use moon_project::Project;
+use moon_project::{Project, ProjectError};
 use moon_query::{Criteria, build_query};
 use moon_task::{Target, TargetError, TargetLocator, TargetScope, Task};
 use moon_task_args::parse_task_args;
-use moon_workspace_graph::{GraphConnections, WorkspaceGraph, tasks::TaskGraphError};
+use moon_workspace_graph::{GraphConnections, WorkspaceGraph};
 use petgraph::prelude::*;
 use rustc_hash::{FxHashMap, FxHashSet};
 use std::mem;
@@ -627,7 +627,11 @@ impl<'query> ActionGraphBuilder<'query> {
 
                 // Don't allow internal tasks to be ran
                 if !allow_internal && task.is_internal() {
-                    return Err(TaskGraphError::UnconfiguredTarget(task.target.clone()).into());
+                    return Err(ProjectError::UnknownTask {
+                        task_id: task.id.to_string(),
+                        project_id: project_id.to_string(),
+                    }
+                    .into());
                 }
 
                 tasks.push(task);

crates/action-graph/src/action_graph_builder.rs

@@ -10,6 +10,7 @@ publish = false
 
 [dependencies]
 moon_common = { path = "../common" }
+moon_config = { path = "../config" }
 moon_env_var = { path = "../env-var" }
 moon_project = { path = "../project" }
 moon_target = { path = "../target" }

crates/affected/Cargo.toml

@@ -120,6 +120,9 @@ pub struct AffectedTaskState {
     #[serde(skip_serializing_if = "FxHashSet::is_empty")]
     pub files: FxHashSet<WorkspaceRelativePathBuf>,
 
+    #[serde(skip_serializing_if = "FxHashSet::is_empty")]
+    pub projects: FxHashSet<Id>,
+
     #[serde(skip_serializing_if = "FxHashSet::is_empty")]
     pub upstream: FxHashSet<Target>,
 
@@ -147,6 +150,9 @@ impl AffectedTaskState {
                 AffectedBy::UpstreamTask(target) => {
                     state.upstream.insert(target);
                 }
+                AffectedBy::UpstreamProject(id) => {
+                    state.projects.insert(id);
+                }
                 _ => {
                     state.other = true;
                 }

crates/affected/src/affected.rs

@@ -1,12 +1,14 @@
 use crate::affected::*;
 use moon_common::path::WorkspaceRelativePathBuf;
 use moon_common::{Id, color};
+use moon_config::Input;
 use moon_env_var::GlobalEnvBag;
 use moon_project::Project;
 use moon_task::{Target, Task, TaskOptionRunInCI};
 use moon_workspace_graph::{GraphConnections, WorkspaceGraph};
 use rustc_hash::{FxHashMap, FxHashSet};
 use starbase_utils::fs;
+use starbase_utils::glob::GlobSet;
 use std::fmt;
 use std::sync::Arc;
 use tracing::{debug, trace};
@@ -74,10 +76,11 @@ impl AffectedTracker {
             debug!(
                 env = ?state.env.iter().collect::<Vec<_>>(),
                 files = ?state.files.iter().collect::<Vec<_>>(),
+                projects = ?state.projects.iter().map(|id| id.as_str()).collect::<Vec<_>>(),
                 upstream = ?state.upstream.iter().map(|target| target.as_str()).collect::<Vec<_>>(),
                 downstream = ?state.downstream.iter().map(|target| target.as_str()).collect::<Vec<_>>(),
                 other = state.other,
-                "Task {} is affected by", color::label(&target),
+                "Task {} is affected by", color::id(&target),
             );
 
             affected.tasks.insert(target, state);
@@ -161,6 +164,59 @@ impl AffectedTracker {
         }
     }
 
+    pub fn is_project_affected_using_file_group(
+        &self,
+        project: &Project,
+        file_group_id: &Id,
+    ) -> miette::Result<Option<AffectedBy>> {
+        let group = project.get_file_group(file_group_id)?;
+
+        if !group.files.is_empty() {
+            for file in &group.files {
+                if self.touched_files.contains(file) {
+                    return Ok(Some(AffectedBy::TouchedFile(file.to_owned())));
+                }
+            }
+        }
+
+        if !group.globs.is_empty() {
+            return self.is_project_affected_using_globs(project, &group.globs);
+        }
+
+        Ok(None)
+    }
+
+    pub fn is_project_affected_using_globs<G: AsRef<str>>(
+        &self,
+        project: &Project,
+        base_globs: &[G],
+    ) -> miette::Result<Option<AffectedBy>> {
+        let mut globs = vec![];
+
+        // Ensure they are relative from the project root
+        for glob in base_globs {
+            let glob = glob.as_ref();
+
+            if glob.starts_with(project.source.as_str()) {
+                globs.push(glob.to_owned());
+            } else {
+                globs.push(format!("{}/{glob}", project.source));
+            }
+        }
+
+        if !globs.is_empty() {
+            let globset = GlobSet::new(&globs)?;
+
+            for file in &self.touched_files {
+                if globset.matches(file.as_str()) {
+                    return Ok(Some(AffectedBy::TouchedFile(file.to_owned())));
+                }
+            }
+        }
+
+        Ok(None)
+    }
+
     pub fn is_project_marked(&self, project: &Project) -> bool {
         self.projects.contains_key(&project.id)
     }
@@ -332,6 +388,7 @@ impl AffectedTracker {
             return Ok(Some(AffectedBy::AlreadyMarked));
         }
 
+        // Special CI handling
         if self.ci {
             match &task.options.run_in_ci {
                 TaskOptionRunInCI::Always => {
@@ -342,11 +399,12 @@ impl AffectedTracker {
             };
         }
 
-        // inputs: []
+        // Never affected
         if task.state.empty_inputs {
             return Ok(None);
         }
 
+        // By env vars
         if !task.input_env.is_empty() {
             let bag = GlobalEnvBag::instance();
 
@@ -359,32 +417,77 @@ impl AffectedTracker {
             }
         }
 
+        // By files
         let globset = task.create_globset()?;
 
         for file in self.touched_files.iter() {
-            let mut affected = false;
-
-            if let Some(params) = task.input_files.get(file) {
-                affected = match &params.content {
+            let affected = if let Some(params) = task.input_files.get(file) {
+                match &params.content {
                     Some(matcher) => {
-                        let content =
-                            fs::read_file(file.to_logical_path(&self.workspace_graph.root))?;
+                        let abs_file = file.to_logical_path(&self.workspace_graph.root);
 
-                        matcher.is_match(&content)
+                        if abs_file.exists() {
+                            matcher.is_match(&fs::read_file(abs_file)?)
+                        } else {
+                            false
+                        }
                     }
                     None => true,
-                };
-            }
-
-            if !affected {
-                affected = globset.matches(file.as_str());
-            }
+                }
+            } else {
+                globset.matches(file.as_str())
+            };
 
             if affected {
                 return Ok(Some(AffectedBy::TouchedFile(file.to_owned())));
             }
         }
 
+        // By other inputs
+        let mut has_all_project_sources = false;
+
+        for input in &task.inputs {
+            match input {
+                Input::Project(inner) => {
+                    if has_all_project_sources {
+                        continue;
+                    }
+
+                    let projects = if inner.project == "^" {
+                        has_all_project_sources = true;
+
+                        let parent = self
+                            .workspace_graph
+                            .get_project(task.target.get_project_id()?)?;
+
+                        self.workspace_graph
+                            .get_projects_by_id(parent.dependencies.iter().map(|dep| &dep.id))?
+                    } else {
+                        vec![self.workspace_graph.get_project(&inner.project)?]
+                    };
+
+                    for project in projects {
+                        let affected = if let Some(group_id) = &inner.group {
+                            self.is_project_affected_using_file_group(&project, group_id)?
+                                .is_some()
+                        } else if !inner.filter.is_empty() {
+                            self.is_project_affected_using_globs(&project, &inner.filter)?
+                                .is_some()
+                        } else {
+                            self.is_project_affected(&project).is_some()
+                        };
+
+                        if affected {
+                            return Ok(Some(AffectedBy::UpstreamProject(project.id.clone())));
+                        }
+                    }
+                }
+                _ => {
+                    // Skip
+                }
+            };
+        }
+
         Ok(None)
     }
 

crates/affected/src/affected_tracker.rs

@@ -0,0 +1,3 @@
+fileGroups:
+  tests:
+    - 'tests/**/*'

crates/affected/tests/__fixtures__/tasks/dep-test/moon.yml

@@ -0,0 +1,5 @@
+fileGroups:
+  sources:
+    - 'src/**/*'
+  tests:
+    - 'tests/**/*'

crates/affected/tests/__fixtures__/tasks/dep/moon.yml

@@ -0,0 +1,19 @@
+dependsOn:
+  - 'dep'
+  - 'dep-test'
+
+tasks:
+  by-any:
+    inputs:
+      - 'project://dep'
+  by-deps:
+    inputs:
+      - project: '^'
+        group: 'tests'
+  by-group:
+    inputs:
+      - project: 'dep'
+        group: 'sources'
+  by-filter:
+    inputs:
+      - 'project://dep?filter=tests/**/*'