Introduce CI redundancy groups

Author: AndriySvyryd

azure-pipelines-internal-tests.yml

@@ -94,6 +94,7 @@ extends:
     stages:
     - stage: build
       displayName: Build
+      continueOnError: true
       jobs:
       - template: /eng/common/templates-official/jobs/jobs.yml@self
         parameters:
@@ -470,3 +471,78 @@ extends:
                   HelixAccessToken: $(_HelixAccessToken)
                   SYSTEM_ACCESSTOKEN: $(System.AccessToken)
                   DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token)
+    - stage: validate
+      displayName: Validate
+      dependsOn: build
+      condition: always()
+      jobs:
+      - job: Validate_Job_Results
+        displayName: Validate Job Results
+        dependsOn:
+          - Windows
+          - macOS
+          - Linux
+          - Linux_Cosmos
+          - Helix_Windows
+          - Helix_macOS_x64
+          - Helix_macOS_ARM64
+          - Helix_Ubuntu_SqlServer
+          - Helix_Ubuntu_Cosmos
+          - Helix_Ubuntu
+        condition: always()
+        pool:
+          name: $(DncEngInternalBuildPool)
+          demands: ImageOverride -equals 1es-windows-2022
+          os: windows
+        variables:
+          WindowsResult: $[ dependencies.Windows.result ]
+          HelixWindowsResult: $[ dependencies.Helix_Windows.result ]
+          LinuxResult: $[ dependencies.Linux.result ]
+          HelixUbuntuResult: $[ dependencies.Helix_Ubuntu.result ]
+          HelixUbuntuSqlServerResult: $[ dependencies.Helix_Ubuntu_SqlServer.result ]
+          MacOsResult: $[ dependencies.macOS.result ]
+          HelixMacOsX64Result: $[ dependencies.Helix_macOS_x64.result ]
+          HelixMacOsArm64Result: $[ dependencies.Helix_macOS_ARM64.result ]
+          LinuxCosmosResult: $[ dependencies.Linux_Cosmos.result ]
+          HelixUbuntuCosmosResult: $[ dependencies.Helix_Ubuntu_Cosmos.result ]
+        steps:
+          - pwsh: |
+              $groupResults = @{
+                  Windows = @("$(WindowsResult)", "$(HelixWindowsResult)")
+                  Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)", "$(HelixUbuntuSqlServerResult)")
+                  MacOS = @("$(MacOsResult)", "$(HelixMacOsX64Result)", "$(HelixMacOsArm64Result)")
+                  Cosmos = @("$(LinuxCosmosResult)", "$(HelixUbuntuCosmosResult)")
+              }
+
+              $failedGroups = @()
+
+              foreach ($groupName in $groupResults.Keys)
+              {
+                  $results = $groupResults[$groupName]
+                  Write-Host "$groupName results: $($results -join ', ')"
+
+                  # Ignore jobs that were skipped (e.g. condition-gated)
+                  $ran = $results | Where-Object { $_ -ne 'Skipped' }
+                  if (-not $ran)
+                  {
+                      Write-Host "  -> all jobs skipped, treating group as successful"
+                      continue
+                  }
+
+                  $hasSuccess = $ran | Where-Object { $_ -eq 'Succeeded' }
+                  if ($hasSuccess)
+                  {
+                      continue
+                  }
+
+                  $failedGroups += $groupName
+              }
+
+              if ($failedGroups.Count -gt 0)
+              {
+                  Write-Error "All jobs failed for group(s): $($failedGroups -join ', ')."
+                  exit 1
+              }
+
+              Write-Host 'Group validation passed.'
+            displayName: Evaluate grouped job outcomes

azure-pipelines-public.yml

@@ -37,6 +37,7 @@ pr: ['*']
 stages:
 - stage: build
   displayName: Build
+  continueOnError: true
   jobs:
     - template: eng/common/templates/jobs/jobs.yml
       parameters:
@@ -320,3 +321,75 @@ stages:
                 env:
                   HelixAccessToken: $(_HelixAccessToken)
                   SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+- stage: validate
+  displayName: Validate
+  dependsOn: build
+  condition: always()
+  jobs:
+  - job: Validate_Job_Results
+    displayName: Validate Job Results
+    dependsOn:
+      - Windows
+      - macOS
+      - Linux
+      - Helix_Windows
+      - Helix_macOS_x64
+      - Helix_macOS_ARM64
+      - Helix_Ubuntu_SqlServer
+      - Helix_Ubuntu_Cosmos
+      - Helix_Ubuntu
+    condition: always()
+    pool:
+      name: $(DncEngPublicBuildPool)
+      demands: ImageOverride -equals 1es-windows-2022-open
+    variables:
+      WindowsResult: $[ dependencies.Windows.result ]
+      HelixWindowsResult: $[ dependencies.Helix_Windows.result ]
+      LinuxResult: $[ dependencies.Linux.result ]
+      HelixUbuntuResult: $[ dependencies.Helix_Ubuntu.result ]
+      HelixUbuntuSqlServerResult: $[ dependencies.Helix_Ubuntu_SqlServer.result ]
+      MacOsResult: $[ dependencies.macOS.result ]
+      HelixMacOsX64Result: $[ dependencies.Helix_macOS_x64.result ]
+      HelixMacOsArm64Result: $[ dependencies.Helix_macOS_ARM64.result ]
+      HelixUbuntuCosmosResult: $[ dependencies.Helix_Ubuntu_Cosmos.result ]
+    steps:
+      - pwsh: |
+          $groupResults = @{
+              Windows = @("$(WindowsResult)", "$(HelixWindowsResult)")
+              Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)", "$(HelixUbuntuSqlServerResult)")
+              MacOS = @("$(MacOsResult)", "$(HelixMacOsX64Result)", "$(HelixMacOsArm64Result)")
+              Cosmos = @("$(HelixUbuntuCosmosResult)")
+          }
+
+          $failedGroups = @()
+
+          foreach ($groupName in $groupResults.Keys)
+          {
+              $results = $groupResults[$groupName]
+              Write-Host "$groupName results: $($results -join ', ')"
+
+              # Ignore jobs that were skipped (e.g. condition-gated)
+              $ran = $results | Where-Object { $_ -ne 'Skipped' }
+              if (-not $ran)
+              {
+                  Write-Host "  -> all jobs skipped, treating group as successful"
+                  continue
+              }
+
+              $hasSuccess = $ran | Where-Object { $_ -eq 'Succeeded' }
+              if ($hasSuccess)
+              {
+                  continue
+              }
+
+              $failedGroups += $groupName
+          }
+
+          if ($failedGroups.Count -gt 0)
+          {
+              Write-Error "All jobs failed for group(s): $($failedGroups -join ', ')."
+              exit 1
+          }
+
+          Write-Host 'Group validation passed.'
+        displayName: Evaluate grouped job outcomes