feat: dynamic autoplan via OUTPUT_MODIFIED_FILES_FILE in pre-workflow hooks

runatlantis.io/docs/pre-workflow-hooks.md

@@ -58,6 +58,37 @@ repos:
           description: Generating configs
 ```
 
+### Dynamic Autoplan with Uncommitted Files
+
+Some workflows (e.g. [Atmos](https://atmos.tools)) generate Terraform variable files
+from higher-level stack YAML configuration. When the stack YAML changes, the affected
+`tfvars` files are regenerated but are not committed — Atlantis would therefore not
+detect them as modified files and would not trigger a plan.
+
+To handle this, a pre-workflow hook can write extra file paths to the
+`OUTPUT_MODIFIED_FILES_FILE`. Atlantis will append those paths to the list of modified
+files before performing project discovery. This allows a custom script to determine
+which projects are affected by uncommitted changes and trigger plans for only those
+projects.
+
+```yaml
+repos:
+    - id: /.*/
+      pre_workflow_hooks:
+        - run: |
+            # Run a tool that computes affected stacks and writes their tfvars
+            # paths (relative to the repo root) to OUTPUT_MODIFIED_FILES_FILE,
+            # one path per line. Atlantis will treat those files as modified.
+            atmos describe affected --output-path "$OUTPUT_MODIFIED_FILES_FILE"
+          description: Compute affected stacks
+          commands: plan
+```
+
+The `OUTPUT_MODIFIED_FILES_FILE` is evaluated **after** all pre-workflow hooks have
+run. Each non-empty line of the file is treated as a repo-relative file path. This
+works together with each project's `autoplan.when_modified` patterns, so only the
+projects whose patterns match the extra paths will be planned.
+
 ## Customizing the Shell
 
 By default, the command will be run using the 'sh' shell with an argument of '-c'. This
@@ -114,4 +145,8 @@ command](custom-workflows.md#custom-run-command).
       every character is escaped, ex. `atlantis plan -- arg1 arg2` will result in `COMMENT_ARGS=\a\r\g\1,\a\r\g\2`.
   * `COMMAND_NAME` - The name of the command that is being executed, i.e. `plan`, `apply` etc.
   * `OUTPUT_STATUS_FILE` - An output file to customize the success or failure status. ex. `echo 'failure' > $OUTPUT_STATUS_FILE`.
+  * `OUTPUT_MODIFIED_FILES_FILE` - An output file to specify extra modified file paths (relative to the repo root) that
+      Atlantis should treat as modified when determining which projects to plan. Write one file path per line.
+      This is useful for dynamically triggering plans for projects affected by uncommitted changes (e.g. generated
+      `tfvars` files). ex. `echo 'components/s3/ue1-dev/terraform.tfvars' >> $OUTPUT_MODIFIED_FILES_FILE`.
 :::

server/core/runtime/pre_workflow_hook_runner.go

@@ -25,28 +25,30 @@ type DefaultPreWorkflowHookRunner struct {
 
 func (wh DefaultPreWorkflowHookRunner) Run(ctx models.WorkflowHookCommandContext, command string, shell string, shellArgs string, path string) (string, string, error) {
 	outputFilePath := filepath.Join(path, "OUTPUT_STATUS_FILE")
+	outputModifiedFilesFilePath := filepath.Join(path, "OUTPUT_MODIFIED_FILES_FILE")
 
 	shellArgsSlice := append(strings.Split(shellArgs, " "), command)
 	cmd := exec.Command(shell, shellArgsSlice...) // #nosec
 	cmd.Dir = path
 
 	baseEnvVars := os.Environ()
 	customEnvVars := map[string]string{
-		"BASE_BRANCH_NAME":   ctx.Pull.BaseBranch,
-		"BASE_REPO_NAME":     ctx.BaseRepo.Name,
-		"BASE_REPO_OWNER":    ctx.BaseRepo.Owner,
-		"COMMENT_ARGS":       strings.Join(ctx.EscapedCommentArgs, ","),
-		"DIR":                path,
-		"HEAD_BRANCH_NAME":   ctx.Pull.HeadBranch,
-		"HEAD_COMMIT":        ctx.Pull.HeadCommit,
-		"HEAD_REPO_NAME":     ctx.HeadRepo.Name,
-		"HEAD_REPO_OWNER":    ctx.HeadRepo.Owner,
-		"PULL_AUTHOR":        ctx.Pull.Author,
-		"PULL_NUM":           fmt.Sprintf("%d", ctx.Pull.Num),
-		"PULL_URL":           ctx.Pull.URL,
-		"USER_NAME":          ctx.User.Username,
-		"OUTPUT_STATUS_FILE": outputFilePath,
-		"COMMAND_NAME":       ctx.CommandName,
+		"BASE_BRANCH_NAME":           ctx.Pull.BaseBranch,
+		"BASE_REPO_NAME":             ctx.BaseRepo.Name,
+		"BASE_REPO_OWNER":            ctx.BaseRepo.Owner,
+		"COMMENT_ARGS":               strings.Join(ctx.EscapedCommentArgs, ","),
+		"DIR":                        path,
+		"HEAD_BRANCH_NAME":           ctx.Pull.HeadBranch,
+		"HEAD_COMMIT":                ctx.Pull.HeadCommit,
+		"HEAD_REPO_NAME":             ctx.HeadRepo.Name,
+		"HEAD_REPO_OWNER":            ctx.HeadRepo.Owner,
+		"PULL_AUTHOR":                ctx.Pull.Author,
+		"PULL_NUM":                   fmt.Sprintf("%d", ctx.Pull.Num),
+		"PULL_URL":                   ctx.Pull.URL,
+		"USER_NAME":                  ctx.User.Username,
+		"OUTPUT_STATUS_FILE":         outputFilePath,
+		"OUTPUT_MODIFIED_FILES_FILE": outputModifiedFilesFilePath,
+		"COMMAND_NAME":               ctx.CommandName,
 	}
 
 	finalEnvVars := baseEnvVars

server/events/command/context.go

@@ -55,4 +55,11 @@ type Context struct {
 
 	// Set true if there were any errors during the command execution
 	CommandHasErrors bool
+
+	// ExtraModifiedFiles is a list of additional file paths (relative to the
+	// repo root) that should be treated as modified when determining which
+	// projects to plan/apply. This can be populated by pre-workflow hooks
+	// using the OUTPUT_MODIFIED_FILES_FILE environment variable to dynamically
+	// trigger plans for projects affected by uncommitted changes.
+	ExtraModifiedFiles []string
 }

server/events/pre_workflow_hooks_command_runner.go

@@ -5,6 +5,8 @@ package events
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/google/uuid"
@@ -91,6 +93,27 @@ func (w *DefaultPreWorkflowHooksCommandRunner) RunPreHooks(ctx *command.Context,
 		return err
 	}
 
+	// Read the extra modified files written by the hook to OUTPUT_MODIFIED_FILES_FILE.
+	// Each non-empty line is treated as a repo-relative file path that should be
+	// considered modified when determining which projects to plan.
+	outputModifiedFilesFilePath := filepath.Join(repoDir, "OUTPUT_MODIFIED_FILES_FILE")
+	if _, statErr := os.Stat(outputModifiedFilesFilePath); statErr == nil {
+		content, readErr := os.ReadFile(outputModifiedFilesFilePath)
+		if readErr != nil {
+			ctx.Log.Warn("unable to read OUTPUT_MODIFIED_FILES_FILE: %s", readErr)
+		} else {
+			for _, line := range strings.Split(string(content), "\n") {
+				line = strings.TrimSpace(line)
+				if line != "" {
+					ctx.ExtraModifiedFiles = append(ctx.ExtraModifiedFiles, line)
+				}
+			}
+			if len(ctx.ExtraModifiedFiles) > 0 {
+				ctx.Log.Info("pre-workflow hook set %d extra modified files: %v", len(ctx.ExtraModifiedFiles), ctx.ExtraModifiedFiles)
+			}
+		}
+	}
+
 	ctx.Log.Info("Pre-workflow hooks completed successfully")
 
 	return nil

server/events/pre_workflow_hooks_command_runner_test.go

@@ -5,6 +5,8 @@ package events_test
 
 import (
 	"errors"
+	"os"
+	"path/filepath"
 	"testing"
 
 	. "github.com/petergtz/pegomock/v4"
@@ -550,3 +552,141 @@ func TestRunPreHooks_Clone(t *testing.T) {
 		Assert(t, *unlockCalled == true, "unlock function called")
 	})
 }
+
+func TestRunPreHooks_OutputModifiedFilesFile(t *testing.T) {
+	log := logging.NewNoopLogger(t)
+
+	var newPull = testdata.Pull
+	newPull.BaseRepo = testdata.GithubRepo
+
+	testHook := valid.WorkflowHook{
+		StepName:   "test",
+		RunCommand: "some command",
+	}
+
+	globalCfg := valid.GlobalCfg{
+		Repos: []valid.Repo{
+			{
+				ID: testdata.GithubRepo.ID(),
+				PreWorkflowHooks: []*valid.WorkflowHook{
+					&testHook,
+				},
+			},
+		},
+	}
+
+	planCmd := &events.CommentCommand{
+		Name: command.Plan,
+	}
+
+	t.Run("extra modified files populated from OUTPUT_MODIFIED_FILES_FILE", func(t *testing.T) {
+		preWorkflowHooksSetup(t)
+
+		// Use a real temp directory so the file can be written and read.
+		repoDir := t.TempDir()
+
+		ctx := &command.Context{
+			Pull:     newPull,
+			HeadRepo: testdata.GithubRepo,
+			User:     testdata.User,
+			Log:      log,
+		}
+
+		var unlockCalled = newBool(false)
+		unlockFn := func() {
+			unlockCalled = newBool(true)
+		}
+
+		preWh.GlobalCfg = globalCfg
+
+		When(preWhWorkingDirLocker.TryLock(testdata.GithubRepo.FullName, newPull.Num, events.DefaultWorkspace,
+			events.DefaultRepoRelDir, "", command.Plan)).ThenReturn(unlockFn, nil)
+		When(preWhWorkingDir.Clone(Any[logging.SimpleLogging](), Eq(testdata.GithubRepo), Eq(newPull),
+			Eq(events.DefaultWorkspace))).ThenReturn(repoDir, nil)
+
+		// Simulate the hook writing to OUTPUT_MODIFIED_FILES_FILE
+		outputModifiedFilesFilePath := filepath.Join(repoDir, "OUTPUT_MODIFIED_FILES_FILE")
+		err := os.WriteFile(outputModifiedFilesFilePath, []byte("components/s3/ue1-dev\ncomponents/s3/ue1-prod\n"), 0600)
+		Ok(t, err)
+
+		When(whPreWorkflowHookRunner.Run(Any[models.WorkflowHookCommandContext](), Eq(testHook.RunCommand),
+			Any[string](), Any[string](), Eq(repoDir))).ThenReturn("", "", nil)
+
+		err = preWh.RunPreHooks(ctx, planCmd)
+
+		Ok(t, err)
+		Equals(t, []string{"components/s3/ue1-dev", "components/s3/ue1-prod"}, ctx.ExtraModifiedFiles)
+		Assert(t, *unlockCalled == true, "unlock function called")
+	})
+
+	t.Run("no extra modified files when OUTPUT_MODIFIED_FILES_FILE absent", func(t *testing.T) {
+		preWorkflowHooksSetup(t)
+
+		repoDir := t.TempDir()
+
+		ctx := &command.Context{
+			Pull:     newPull,
+			HeadRepo: testdata.GithubRepo,
+			User:     testdata.User,
+			Log:      log,
+		}
+
+		var unlockCalled = newBool(false)
+		unlockFn := func() {
+			unlockCalled = newBool(true)
+		}
+
+		preWh.GlobalCfg = globalCfg
+
+		When(preWhWorkingDirLocker.TryLock(testdata.GithubRepo.FullName, newPull.Num, events.DefaultWorkspace,
+			events.DefaultRepoRelDir, "", command.Plan)).ThenReturn(unlockFn, nil)
+		When(preWhWorkingDir.Clone(Any[logging.SimpleLogging](), Eq(testdata.GithubRepo), Eq(newPull),
+			Eq(events.DefaultWorkspace))).ThenReturn(repoDir, nil)
+		When(whPreWorkflowHookRunner.Run(Any[models.WorkflowHookCommandContext](), Eq(testHook.RunCommand),
+			Any[string](), Any[string](), Eq(repoDir))).ThenReturn("", "", nil)
+
+		err := preWh.RunPreHooks(ctx, planCmd)
+
+		Ok(t, err)
+		Assert(t, len(ctx.ExtraModifiedFiles) == 0, "extra modified files should be empty")
+		Assert(t, *unlockCalled == true, "unlock function called")
+	})
+
+	t.Run("blank lines in OUTPUT_MODIFIED_FILES_FILE are ignored", func(t *testing.T) {
+		preWorkflowHooksSetup(t)
+
+		repoDir := t.TempDir()
+
+		ctx := &command.Context{
+			Pull:     newPull,
+			HeadRepo: testdata.GithubRepo,
+			User:     testdata.User,
+			Log:      log,
+		}
+
+		var unlockCalled = newBool(false)
+		unlockFn := func() {
+			unlockCalled = newBool(true)
+		}
+
+		preWh.GlobalCfg = globalCfg
+
+		When(preWhWorkingDirLocker.TryLock(testdata.GithubRepo.FullName, newPull.Num, events.DefaultWorkspace,
+			events.DefaultRepoRelDir, "", command.Plan)).ThenReturn(unlockFn, nil)
+		When(preWhWorkingDir.Clone(Any[logging.SimpleLogging](), Eq(testdata.GithubRepo), Eq(newPull),
+			Eq(events.DefaultWorkspace))).ThenReturn(repoDir, nil)
+
+		outputModifiedFilesFilePath := filepath.Join(repoDir, "OUTPUT_MODIFIED_FILES_FILE")
+		err := os.WriteFile(outputModifiedFilesFilePath, []byte("\ncomponents/s3/ue1-dev\n\n"), 0600)
+		Ok(t, err)
+
+		When(whPreWorkflowHookRunner.Run(Any[models.WorkflowHookCommandContext](), Eq(testHook.RunCommand),
+			Any[string](), Any[string](), Eq(repoDir))).ThenReturn("", "", nil)
+
+		err = preWh.RunPreHooks(ctx, planCmd)
+
+		Ok(t, err)
+		Equals(t, []string{"components/s3/ue1-dev"}, ctx.ExtraModifiedFiles)
+		Assert(t, *unlockCalled == true, "unlock function called")
+	})
+}

server/events/project_command_builder.go

@@ -474,7 +474,8 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex
 	ctx.Log.Debug("%d files were modified in this pull request. Modified files: %v", len(modifiedFiles), modifiedFiles)
 
 	// If we're not including git untracked files, we can skip the clone if there are no modified files.
-	if !p.IncludeGitUntrackedFiles {
+	// We also cannot skip the clone if a pre-workflow hook specified extra modified files.
+	if !p.IncludeGitUntrackedFiles && len(ctx.ExtraModifiedFiles) == 0 {
 		shouldSkipClone, err := p.shouldSkipClone(ctx, modifiedFiles)
 		if err != nil {
 			return nil, err
@@ -509,6 +510,13 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex
 		modifiedFiles = append(modifiedFiles, untrackedFiles...)
 	}
 
+	// Append any extra modified files specified by pre-workflow hooks.
+	// These are treated as if they were modified in the pull request.
+	if len(ctx.ExtraModifiedFiles) > 0 {
+		ctx.Log.Debug("appending %d extra modified files from pre-workflow hooks: %v", len(ctx.ExtraModifiedFiles), ctx.ExtraModifiedFiles)
+		modifiedFiles = append(modifiedFiles, ctx.ExtraModifiedFiles...)
+	}
+
 	// Parse config file if it exists.
 	repoCfgFile := p.GlobalCfg.RepoConfigFile(ctx.Pull.BaseRepo.ID())
 	hasRepoCfg, err := p.ParserValidator.HasRepoCfg(repoDir, repoCfgFile)