Migrate to ESRP publishing

eng/tsp-core/pipelines/jobs/build-for-publish.yml

@@ -20,9 +20,10 @@ jobs:
         displayName: Test
 
       - template: /eng/tsp-core/pipelines/templates/upload-coverage.yml
-
-      - script: pnpm chronus pack --exclude standalone --pack-destination $(Build.ArtifactStagingDirectory)/npm-packages-stable
-        displayName: Pack packages
+      - template: /eng/tsp-core/pipelines/templates/pack.yml
+        parameters:
+          artifactName: npm-packages-stable
+          name: npm_packages_stable
 
       - task: CopyFiles@2
         displayName: "Copy VSCode extension .vsix to artifact directory"
@@ -45,6 +46,16 @@ jobs:
           scriptLocation: inlineScript
           inlineScript: node ./eng/tsp-core/scripts/upload-bundler-packages.js
 
+      - task: AzureCLI@2
+        displayName: Upload scenario manifest
+        condition: eq(variables['PUBLISH_PKG_TYPESPEC_HTTP_SPECS'], 'true')
+        inputs:
+          workingDirectory: packages/http-specs
+          azureSubscription: "TypeSpec Storage"
+          scriptType: "bash"
+          scriptLocation: "inlineScript"
+          inlineScript: "pnpm upload-manifest"
+
       # Update version for next version publish
       - script: node ./packages/internal-build-utils/cmd/cli.js bump-version-preview .
         displayName: Bump version to prerelease targets
@@ -53,8 +64,10 @@ jobs:
         displayName: Regen manifest for compiler
         workingDirectory: ./packages/compiler
 
-      - script: pnpm chronus pack --exclude standalone --pack-destination $(Build.ArtifactStagingDirectory)/npm-packages-next
-        displayName: Pack next packages
+      - template: /eng/tsp-core/pipelines/templates/pack.yml
+        parameters:
+          artifactName: npm-packages-next
+          name: npm_packages_next
 
       # Publish Next playground
       - task: AzureCLI@1
@@ -72,6 +85,9 @@ jobs:
               --overwrite
 
     templateContext:
+      # Optimize so each output doesn't create its own scan
+      # https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs
+      outputParentDirectory: $(Build.ArtifactStagingDirectory)
       outputs:
         - output: pipelineArtifact
           path: $(Build.ArtifactStagingDirectory)/npm-packages-stable

eng/tsp-core/pipelines/jobs/create-github-release.yml

@@ -0,0 +1,32 @@
+parameters:
+  - name: artifactName
+    type: string
+  - name: dependsOn
+    type: string
+    default: ""
+
+jobs:
+  - job: build
+    dependsOn: ${{ parameters.dependsOn }}
+    displayName: Create github release(s)
+    pool:
+      name: azsdk-pool
+      image: ubuntu-24.04
+      os: linux
+
+    steps:
+      - download: current
+        artifact: ${{ parameters.artifactName }}
+        displayName: Download package manifest
+
+      - template: /eng/tsp-core/pipelines/templates/install.yml
+
+      - script: |
+          echo "Publish summary:"
+          cat "$(Pipeline.Workspace)/${{ parameters.artifactName }}/publish-summary.json"
+        displayName: Log publish summary
+
+      - script: pnpm chronus-github create-releases --repo microsoft/typespec --publish-summary "$(Pipeline.Workspace)/${{ parameters.artifactName }}/publish-summary.json"
+        displayName: Create github releases
+        env:
+          GITHUB_TOKEN: $(azuresdk-github-pat)

eng/tsp-core/pipelines/jobs/publish-npm.yml

@@ -3,73 +3,44 @@ parameters:
     type: string
   - name: tag
     type: string
-  - name: publishGithubRelease
-    type: boolean
-    default: false
+  - name: condition
+    default: succeeded()
   - name: dependsOn
     type: string
     default: ""
 
 jobs:
-  - job: publish_npm_${{ parameters.tag }}
+  - deployment: publish_npm_${{ parameters.tag }}
     displayName: Publish Npm (${{ parameters.tag }})
     dependsOn: ${{ parameters.dependsOn }}
-    pool:
-      name: $(LINUXPOOL)
-      image: $(LINUXVMIMAGE)
+    condition: ${{ parameters.condition }}
+    environment: typespec-cli
+    pool: # hardcoding the pool and image name because deployment jobs do not support variable expansion in pool names
+      name: azsdk-pool
+      image: ubuntu-24.04
       os: linux
-
-    steps:
-      - download: current
-        artifact: ${{ parameters.artifactName }}
-        displayName: Download Npm Packages
-
-      - template: ../templates/install.yml
-
-      - script: ls $(Pipeline.Workspace)/${{ parameters.artifactName }}
-        displayName: List files in artifact
-
-      - script: |
-          echo "Updating config"
-          pnpm config set '//registry.npmjs.org/:_authToken' '${NPM_AUTH_TOKEN}'
-          echo "Publishing at tag ${{ parameters.tag }}"
-          pnpm chronus publish "$(Pipeline.Workspace)/${{ parameters.artifactName }}/*.tgz" --tag ${{ parameters.tag }}  --access public  --engine npm --report-summary ./publish-summary.json
-        displayName: Publish
-        env:
-          NPM_AUTH_TOKEN: $(azure-sdk-npm-token)
-
-      - pwsh: |
-          $publishSummary = Get-Content "./publish-summary.json" | ConvertFrom-Json
-          if ($publishSummary.packages.'@typespec/http-specs'.published -eq $true) {
-            Write-Output "Setting PublishHttpSpecs variable to true."
-            Write-Host "##vso[task.setvariable variable=PublishHttpSpecs]true"
-          } else {
-            Write-Output "Setting PublishHttpSpecs variable to false."
-            Write-Host "##vso[task.setvariable variable=PublishHttpSpecs]false"
-          }
-        displayName: "Check if @typespec/http-specs was published"
-
-      - script: pnpm -r --filter '@typespec/http-specs...' build
-        condition: eq(variables['PublishHttpSpecs'], 'true')
-        displayName: Build spector
-
-      - task: AzureCLI@2
-        displayName: Upload scenario manifest
-        condition: eq(variables['PublishHttpSpecs'], 'true')
-        inputs:
-          workingDirectory: packages/http-specs
-          azureSubscription: "TypeSpec Storage"
-          scriptType: "bash"
-          scriptLocation: "inlineScript"
-          inlineScript: "pnpm upload-manifest"
-
-      - ${{ if eq(parameters.publishGithubRelease, true) }}:
-          - script: |
-              echo "Publish summary:"
-              cat ./publish-summary.json
-            displayName: Log publish summary
-
-          - script: pnpm chronus-github create-releases --repo microsoft/typespec --publish-summary ./publish-summary.json
-            displayName: Create github releases
-            env:
-              GITHUB_TOKEN: $(azuresdk-github-pat)
+    templateContext:
+      type: releaseJob
+      isProduction: true
+      inputs:
+        - input: pipelineArtifact
+          artifactName: ${{ parameters.artifactName }}
+          itemPattern: "**/*.{tgz}"
+          targetPath: $(Pipeline.Workspace)/${{ parameters.artifactName }}/
+
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+            - script: ls $(Pipeline.Workspace)/${{ parameters.artifactName }}
+              displayName: List files in artifact
+
+            # - template: /eng/tsp-core/pipelines/templates/release/esrp-release.yml
+            #   parameters:
+            #     tag: ${{ parameters.tag }}
+            #     path: "$(Pipeline.Workspace)/${{ parameters.artifactName }}"
+
+            - script: |
+                echo "ESRP_RELEASE_DETAILS:"
+                echo "$ESRP_RELEASE_DETAILS"
+              displayName: "Log ESRP release details"

eng/tsp-core/pipelines/publish.yml

@@ -6,8 +6,12 @@ trigger:
       - main
       # For patch releases
       - release/*
+      - esrp-publish # TODO: for test remove
+  paths:
     exclude:
       - packages/http-client-csharp
+      - packages/http-client-java
+      - packages/http-client-python
 
 pr: none
 
@@ -46,17 +50,30 @@ extends:
           os: linux
 
         jobs:
+          - job: debug
+            variables:
+              varFromA: $[convertToJson(stageDependencies.build.outputs)]
+              deps: $[convertToJson(stageDependencies)] # create a variable with the job dependencies
+            steps:
+              - pwsh: Write-Host "$(deps)"
+
           - template: /eng/tsp-core/pipelines/jobs/publish-npm.yml
             parameters:
               artifactName: npm-packages-stable
               tag: latest
-              publishGithubRelease: true
+              condition: eq('true', stageDependencies.build.build.outputs['pack_npm_packages_stable.PUBLISH_PKG_ANY'])
 
           - template: /eng/tsp-core/pipelines/jobs/publish-npm.yml
             parameters:
               dependsOn: publish_npm_latest
               artifactName: npm-packages-next
               tag: next
+              condition: eq('true', stageDependencies.build.build.outputs['pack_npm_packages_next.PUBLISH_PKG_ANY'])
+
+          - template: /eng/tsp-core/pipelines/jobs/create-github-release.yml
+            parameters:
+              dependsOn: publish_npm_latest
+              artifactName: npm-packages-stable
 
       - stage: publish_vscode
         displayName: Publish VSCode extension
@@ -99,25 +116,3 @@ extends:
                 displayName: "Build"
               - script: docker push $(imageName) --all-tags
                 displayName: "Push"
-
-      - stage: validate_manifest
-        displayName: Validate Manifest
-        dependsOn: build
-        jobs:
-          - job: validate_manifest
-            displayName: Validate Manifest
-            pool:
-              name: $(WINDOWSPOOL)
-              image: $(WINDOWSVMIMAGE)
-              os: windows
-            variables:
-              TYPESPEC_SKIP_DOCUSAURUS_BUILD: true # Disable docusaurus build
-            steps:
-              - template: /eng/tsp-core/pipelines/templates/install.yml
-              - template: /eng/tsp-core/pipelines/templates/build.yml
-
-              - script: pnpm run validate-scenarios --debug
-                displayName: Validate Scenarios
-
-              - script: pnpm run validate-mock-apis --debug
-                displayName: Validate mock apis

eng/tsp-core/pipelines/templates/pack.yml

@@ -0,0 +1,19 @@
+# Template installing all dependencies.
+parameters:
+  - name: artifactName
+    type: string
+  - name: name
+    type: string
+
+steps:
+  - pwsh: |
+      $outDir = "$(Build.ArtifactStagingDirectory)/${{ parameters.artifactName }}"
+      echo "Packing npm packages into $outDir"
+      pnpm chronus pack --exclude standalone --pack-destination $outDir
+
+      $summaryFilePath = "$outDir/publish-summary.json"
+      echo "Create manifest of unpublished packages in $summaryFilePath"
+
+      node eng/tsp-core/scripts/filter-unpublished-packages.ts "$outDir" --manifest $summaryFilePath
+    name: pack_${{ parameters.name }}
+    displayName: Pack packages ${{ parameters.artifactName }}

eng/tsp-core/pipelines/templates/release/esrp-release.yml

@@ -0,0 +1,26 @@
+parameters:
+  - name: tag
+    type: string
+    default: stable
+  - name: path
+    type: string
+
+steps:
+  - task: EsrpRelease@9
+    displayName: "Publish via ESRP"
+    condition: and(succeeded(), ne(variables['SkipPublishing'], 'true'))
+    inputs:
+      ConnectedServiceName: "Azure SDK PME Managed Identity"
+      ClientId: "5f81938c-2544-4f1f-9251-dd9de5b8a81b"
+      DomainTenantId: "975f013f-7f24-47e8-a7d3-abc4752bf346"
+      Usemanagedidentity: true
+      KeyVaultName: "kv-azuresdk-codesign"
+      SignCertName: "azure-sdk-esrp-release-certificate"
+      Intent: "PackageDistribution"
+      ContentType: "npm"
+      FolderLocation: ${{ parameters.path }}
+      Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }}
+      Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }}
+      ServiceEndpointUrl: "https://api.esrp.microsoft.com"
+      MainPublisher: "ESRPRELPACMANTEST"
+      productstate: ${{ parameters.tag }}