Skip to content

Commit 6baf84c

Browse files
s1ntaxe770rs1ntaxe770r
authored
Feat/terragrunt parallelism (#2667)
1 parent 53efc5d commit 6baf84c

20 files changed

Lines changed: 871 additions & 607 deletions

File tree

‎cli/pkg/digger/digger.go‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,16 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
248248
var iacUtils iac_utils.IacUtils
249249
projectPath := path.Join(workingDir, job.ProjectDir)
250250
if job.Terragrunt {
251-
terraformExecutor = execution.Terragrunt{WorkingDir: projectPath}
251+
var driftParallelism *int
252+
if command == "digger drift-detect" {
253+
driftParallelism = job.DriftTerragruntParallelism
254+
}
255+
terraformExecutor = execution.Terragrunt{WorkingDir: projectPath, Parallelism: driftParallelism}
256+
if driftParallelism != nil {
257+
slog.Info("drift_terragrunt_parallelism enabled",
258+
"projectName", job.ProjectName,
259+
"parallelism", *driftParallelism)
260+
}
252261
iacUtils = iac_utils.TerraformUtils{}
253262
} else if job.OpenTofu {
254263
terraformExecutor = execution.OpenTofu{WorkingDir: projectPath, Workspace: job.ProjectWorkspace}
@@ -680,7 +689,16 @@ func RunJob(
680689
var iacUtils iac_utils.IacUtils
681690
projectPath := path.Join(workingDir, job.ProjectDir)
682691
if job.Terragrunt {
683-
terraformExecutor = execution.Terragrunt{WorkingDir: projectPath}
692+
var driftParallelism *int
693+
if command == "digger drift-detect" {
694+
driftParallelism = job.DriftTerragruntParallelism
695+
}
696+
terraformExecutor = execution.Terragrunt{WorkingDir: projectPath, Parallelism: driftParallelism}
697+
if driftParallelism != nil {
698+
slog.Info("drift_terragrunt_parallelism enabled",
699+
"projectName", job.ProjectName,
700+
"parallelism", *driftParallelism)
701+
}
684702
iacUtils = iac_utils.TerraformUtils{}
685703
} else if job.OpenTofu {
686704
terraformExecutor = execution.OpenTofu{WorkingDir: projectPath, Workspace: job.ProjectWorkspace}

‎cli/pkg/github/github.go‎

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -174,21 +174,22 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
174174
}
175175

176176
jobs := scheduler.Job{
177-
ProjectName: project,
178-
ProjectDir: projectConfig.Dir,
179-
ProjectWorkspace: projectConfig.Workspace,
180-
Terragrunt: projectConfig.Terragrunt,
181-
OpenTofu: projectConfig.OpenTofu,
182-
Pulumi: projectConfig.Pulumi,
183-
Commands: []string{command},
184-
ApplyStage: scheduler.ToConfigStage(workflow.Apply),
185-
PlanStage: scheduler.ToConfigStage(workflow.Plan),
186-
PullRequestNumber: nil,
187-
EventName: "manual_invocation",
188-
RequestedBy: githubActor,
189-
Namespace: ghRepository,
190-
StateEnvVars: stateEnvVars,
191-
CommandEnvVars: commandEnvVars,
177+
ProjectName: project,
178+
ProjectDir: projectConfig.Dir,
179+
ProjectWorkspace: projectConfig.Workspace,
180+
Terragrunt: projectConfig.Terragrunt,
181+
DriftTerragruntParallelism: projectConfig.DriftTerragruntParallelism,
182+
OpenTofu: projectConfig.OpenTofu,
183+
Pulumi: projectConfig.Pulumi,
184+
Commands: []string{command},
185+
ApplyStage: scheduler.ToConfigStage(workflow.Apply),
186+
PlanStage: scheduler.ToConfigStage(workflow.Plan),
187+
PullRequestNumber: nil,
188+
EventName: "manual_invocation",
189+
RequestedBy: githubActor,
190+
Namespace: ghRepository,
191+
StateEnvVars: stateEnvVars,
192+
CommandEnvVars: commandEnvVars,
192193
}
193194
err = digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir)
194195
if err != nil {
@@ -245,24 +246,25 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
245246
}
246247

247248
job := scheduler.Job{
248-
ProjectName: projectConfig.Name,
249-
ProjectDir: projectConfig.Dir,
250-
ProjectWorkspace: projectConfig.Workspace,
251-
Terragrunt: projectConfig.Terragrunt,
252-
OpenTofu: projectConfig.OpenTofu,
253-
Pulumi: projectConfig.Pulumi,
254-
Commands: []string{"digger drift-detect"},
255-
ApplyStage: scheduler.ToConfigStage(workflow.Apply),
256-
PlanStage: scheduler.ToConfigStage(workflow.Plan),
257-
CommandEnvVars: commandEnvVars,
258-
StateEnvVars: stateEnvVars,
259-
RequestedBy: githubActor,
260-
Namespace: ghRepository,
261-
EventName: "drift-detect",
262-
StateEnvProvider: StateEnvProvider,
263-
CommandEnvProvider: CommandEnvProvider,
264-
StateRoleArn: stateArn,
265-
CommandRoleArn: cmdArn,
249+
ProjectName: projectConfig.Name,
250+
ProjectDir: projectConfig.Dir,
251+
ProjectWorkspace: projectConfig.Workspace,
252+
Terragrunt: projectConfig.Terragrunt,
253+
DriftTerragruntParallelism: projectConfig.DriftTerragruntParallelism,
254+
OpenTofu: projectConfig.OpenTofu,
255+
Pulumi: projectConfig.Pulumi,
256+
Commands: []string{"digger drift-detect"},
257+
ApplyStage: scheduler.ToConfigStage(workflow.Apply),
258+
PlanStage: scheduler.ToConfigStage(workflow.Plan),
259+
CommandEnvVars: commandEnvVars,
260+
StateEnvVars: stateEnvVars,
261+
RequestedBy: githubActor,
262+
Namespace: ghRepository,
263+
EventName: "drift-detect",
264+
StateEnvProvider: StateEnvProvider,
265+
CommandEnvProvider: CommandEnvProvider,
266+
StateRoleArn: stateArn,
267+
CommandRoleArn: cmdArn,
266268
}
267269

268270
err = digger.RunJob(job, ghRepository, githubActor, &githubPrService, policyChecker, nil, backendApi, &notification, currentDir)

‎docs/ce/drift/backendless-scoping-projects.mdx‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,32 @@ Here, every project generated from the `crdb` block whose directory matches `pro
103103

104104
Per-block patterns and top-level patterns compose: a project must pass both checks to run drift detection.
105105

106+
## Limiting Terragrunt concurrency
107+
108+
For large Terragrunt estates, an unbounded run can overwhelm the runner with memory and process churn. Use `drift_terragrunt_parallelism` to cap how many modules Terragrunt loads at once during each drift run — this is the `digger.yml` equivalent of `terragrunt run-all plan -parallelism=N`.
109+
110+
It can be set at three scopes; the most specific wins:
111+
112+
- **Project** (under a `projects:` entry): overrides everything for that one project.
113+
- **Block** (inside a `generate_projects.blocks[]` entry): applies to every project generated by that block.
114+
- **Top-level** (directly under `generate_projects:`): applies to every project that does not set its own value.
115+
116+
```yaml
117+
generate_projects:
118+
drift_terragrunt_parallelism: 10 # default for every project below
119+
blocks:
120+
- block_name: prod
121+
include: "live/prod/**"
122+
terragrunt: true
123+
drift_terragrunt_parallelism: 20 # override for this block
124+
- block_name: staging
125+
include: "live/staging/**"
126+
terragrunt: true
127+
# inherits the top-level 10
128+
```
129+
130+
When set, Digger appends `-parallelism=N` to every Terragrunt invocation (init, plan, apply, destroy) **during drift detection only** — `digger plan` and `digger apply` runs are unaffected. Valid values are `1` through `20`. Leave it unset to use Terragrunt's default.
131+
106132
## Notes
107133

108134
- `drift_include_patterns` / `drift_exclude_patterns` use [doublestar](https://github.com/bmatcuk/doublestar) glob matching against the project directory path.

‎docs/ce/reference/digger.yml.mdx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,10 @@ Automatically generate projects from directory structure using the `generate_pro
385385
Glob patterns matched against each project's `dir`. Matching projects are skipped during drift detection. Evaluated after `drift_include_patterns`.
386386
</ParamField>
387387

388+
<ParamField path="drift_terragrunt_parallelism" type="integer">
389+
Default cap on the worker pool Terragrunt uses for dependency loading on each unit during drift detection — equivalent to passing `-parallelism=N`. Must be between `1` and `20`. Applied to every Terragrunt project that does not set its own `drift_terragrunt_parallelism` (project-level overrides block-level overrides this top-level default). Only takes effect for drift runs; plan/apply are unaffected.
390+
</ParamField>
391+
388392
<ParamField path="aws_role_to_assume" type="object">
389393
Default AWS role configuration for all generated projects. See [AWS Role Configuration](#aws-role-configuration).
390394
</ParamField>
@@ -425,6 +429,10 @@ Automatically generate projects from directory structure using the `generate_pro
425429
Glob patterns matched against the `dir` of each project generated by this block. Matching projects are skipped during drift detection. Evaluated after this block's `drift_include_patterns`.
426430
</ParamField>
427431

432+
<ParamField path="drift_terragrunt_parallelism" type="integer">
433+
Caps the worker pool Terragrunt uses for dependency loading on each unit during drift detection — equivalent to passing `-parallelism=N` to Terragrunt. Useful for large estates where unbounded concurrency overwhelms the runner. Must be between `1` and `20`. Only takes effect for drift runs (plan/apply are unaffected) and only when the generated projects are Terragrunt. Overrides the top-level `drift_terragrunt_parallelism` default for this block.
434+
</ParamField>
435+
428436
<ParamField path="workflow" type="string" default="default">
429437
Workflow to use for projects in this block.
430438
</ParamField>

‎ee/cli/pkg/gitlab/gitlab.go‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -89,24 +89,25 @@ func GitLabCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
8989
}
9090

9191
job := scheduler.Job{
92-
ProjectName: projectConfig.Name,
93-
ProjectDir: projectConfig.Dir,
94-
ProjectWorkspace: projectConfig.Workspace,
95-
Terragrunt: projectConfig.Terragrunt,
96-
OpenTofu: projectConfig.OpenTofu,
97-
Pulumi: projectConfig.Pulumi,
98-
Commands: []string{"digger drift-detect"},
99-
ApplyStage: scheduler.ToConfigStage(workflow.Apply),
100-
PlanStage: scheduler.ToConfigStage(workflow.Plan),
101-
CommandEnvVars: commandEnvVars,
102-
StateEnvVars: stateEnvVars,
103-
RequestedBy: actor,
104-
Namespace: repoFullName,
105-
EventName: "drift-detect",
106-
StateEnvProvider: StateEnvProvider,
107-
CommandEnvProvider: CommandEnvProvider,
108-
StateRoleArn: stateArn,
109-
CommandRoleArn: cmdArn,
92+
ProjectName: projectConfig.Name,
93+
ProjectDir: projectConfig.Dir,
94+
ProjectWorkspace: projectConfig.Workspace,
95+
Terragrunt: projectConfig.Terragrunt,
96+
DriftTerragruntParallelism: projectConfig.DriftTerragruntParallelism,
97+
OpenTofu: projectConfig.OpenTofu,
98+
Pulumi: projectConfig.Pulumi,
99+
Commands: []string{"digger drift-detect"},
100+
ApplyStage: scheduler.ToConfigStage(workflow.Apply),
101+
PlanStage: scheduler.ToConfigStage(workflow.Plan),
102+
CommandEnvVars: commandEnvVars,
103+
StateEnvVars: stateEnvVars,
104+
RequestedBy: actor,
105+
Namespace: repoFullName,
106+
EventName: "drift-detect",
107+
StateEnvProvider: StateEnvProvider,
108+
CommandEnvProvider: CommandEnvProvider,
109+
StateRoleArn: stateArn,
110+
CommandRoleArn: cmdArn,
110111
}
111112

112113
notification, err := driftNotificationProvider.Get(prService)

0 commit comments

Comments
 (0)