Add NuGet publishing (#40)

Introduces the ability to publish NuGet package releases from
Project Mu repos.

Also includes changes to publish artifacts by type to more cleanly
control how they are produced & consumed - binaries, logs, and other.

**Integration Notes**

This commit updates the major version of mu_devops. This means it
might require integration work that will result in a repo build
breaking unless performed.

- Artifacts are now published under three separate categories of
"binaries",
- "logs", and "other"
- Previously all artifacts were published under `"Build Logs
$(System.JobName)"`
  - Any flows dependent on artifact names will need to be updated
- Three new templates are provided for reusable publishing of content in
these categories:
    1. `Steps/BinaryCopyAndPublish.yml`
    2. `Steps/CommonLogCopyAndPublish.yml`
    3. `Steps/OtherCopyAndPublish.yml`
- `Steps/PrGate.yml` has a new template parameter -
`artifacts_identifier`
- This can be used to adjust the name assigned to artifacts so it makes
the
    most sense for a given platform
    - The default value is an empty string
- For most platforms, it is recommended to pass the package name and
build target
- `Steps/PrGate.yml` has new template parameters to control the binary
and other
  content published.
  - The default value for both is an empty string

**Non-Breaking Change Notes**

- There is a new step template to easily publish content from the three
artifact
  categories of a given pipeline to NuGet (`Steps/NuGet.yml`).
- It is a step template so it can easily access file content already on
the job
    build agent.
- It provides the ability to select which categories of artifacts are
published.
- There is a new job template to generate a build matrix. This is
tailored toward
firmware build scenarios that involve groups of packages that are built
together on a single agent and others that are individually built on a
dedicated agent.

Signed-off-by: Michael Kubacki <michael.kubacki@microsoft.com>

Author: makubacki

Jobs/CreateBuildMatrix.yml

@@ -0,0 +1,67 @@
+## @file
+# Mu DevOps template file to produce a build matrix according to the given
+# package and build target parameters.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+parameters:
+# An optional job dependency for this job to start.
+- name: dependency
+  displayName: Job Dependency
+  type: string
+  default: ''
+# A group package list is not split to a separate package per job in the matrix.
+- name: group_package_list
+  displayName: Group Package List (Optional - Will be Built Together)
+  type: string
+  default: ''
+# Each package in an individual package list is split to a separate job per package in the matrix.
+- name: individual_package_list
+  displayName: Individual Package List (Required - Will be Built Individually)
+  type: string
+  default: ''
+# The targets that need be supported. These are kept as a list in the output of the matrix.
+- name: target_list
+  displayName: Targets (e.g. DEBUG, RELEASE)
+  type: string
+  default: ''
+
+jobs:
+
+  - job: CreateBuildMatrix
+    displayName: Create Build Matrix
+    dependsOn: ${{ parameters.dependency }}
+
+    steps:
+    - checkout: none
+
+    - task: PowerShell@2
+      name: CalculateMatrix
+      displayName: Calculate Matrix
+      inputs:
+          targetType: 'inline'
+          script: |
+            $configs = @{}
+            '${{ parameters.target_list }}'.split(',').Trim() | % {
+              $t = $_
+              if (![string]::IsNullOrEmpty('${{ parameters.individual_package_list }}')) {
+                '${{ parameters.individual_package_list }}'.split(',').Trim() | % {
+                  $p = $_
+                  $configs["${p} ${t}"] = @{
+                    'package' = $p
+                    'target' = $t
+                  }
+                }
+              }
+              if (![string]::IsNullOrEmpty('${{ parameters.group_package_list }}')) {
+                $configs["Non-Platform Package(s) ${t}"] = @{
+                  'package' = '${{ parameters.group_package_list }}'.Trim()
+                  'target' = $t
+                }
+              }
+            }
+            $c = $configs | ConvertTo-Json -Depth 10 -Compress
+            Write-Host "##vso[task.setvariable variable=Matrix;isOutput=true;]$c"
+

Jobs/PrGate.yml

@@ -67,14 +67,19 @@ parameters:
 # Build step
 jobs:
 
+  - template: ../Jobs/CreateBuildMatrix.yml
+    parameters:
+      group_package_list: ${{ parameters.packages }}
+      target_list: ${{ parameters.target_list }}
+
   - job: Build
+    dependsOn: CreateBuildMatrix
+    condition: Succeeded('CreateBuildMatrix')
 
     # Use matrix to speed up the build process
     strategy:
-      matrix:
-        TARGET_BUILD:
-          Build.Pkgs: ${{ parameters.packages }}
-          Build.Targets: ${{ parameters.target_list }}
+      matrix: $[ dependencies.CreateBuildMatrix.outputs['CalculateMatrix.Matrix'] ]
+
     workspace:
       clean: all
 
@@ -93,9 +98,10 @@ jobs:
     - ${{ parameters.extra_steps }}
     - template: ../Steps/PrGate.yml
       parameters:
+        artifacts_identifier: '$(package) $(target)'
         build_file: ${{ parameters.build_file }}
-        build_pkgs: $(Build.Pkgs)
-        build_targets: $(Build.Targets)
+        build_pkgs: $(package)
+        build_targets: $(target)
         build_archs: ${{ parameters.arch_list }}
         do_ci_build: ${{ parameters.do_ci_build }}
         do_ci_setup: ${{ parameters.do_ci_setup }}

Steps/BinaryCopyAndPublish.yml

@@ -0,0 +1,38 @@
+## @file
+# Azure Pipelines step template to publish binary files specified in the template
+# parameters as build artifacts.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+parameters:
+- name: artifacts_binary
+  displayName: Binary Artifacts to Publish
+  type: string
+  default: ''
+- name: artifacts_identifier
+  displayName: Artifacts Identifier
+  type: string
+  default: 'Artifacts'
+
+steps:
+# Copy binaries to the artifact staging directory
+- task: CopyFiles@2
+  displayName: Copy Build Binaries
+  inputs:
+    targetFolder: "$(Build.ArtifactStagingDirectory)/Binaries"
+    SourceFolder: "Build"
+    contents: |
+      ${{ parameters.artifacts_binary }}
+    flattenFolders: true
+  condition: and(succeededOrFailed(), ne('${{ parameters.artifacts_binary }}', ''))
+
+# Publish build artifacts to Azure Artifacts/TFS or a file share
+- task: PublishBuildArtifacts@1
+  continueOnError: true
+  displayName: Publish Build Binaries
+  inputs:
+    pathtoPublish: "$(Build.ArtifactStagingDirectory)/Binaries"
+    artifactName: "Binaries ${{ parameters.artifacts_identifier }}"
+  condition: and(succeededOrFailed(), ne('${{ parameters.artifacts_binary }}', ''))

Steps/BuildBaseTools.yml

@@ -35,7 +35,7 @@ steps:
 - task: CopyFiles@2
   displayName: "Copy base tools build log"
   inputs:
-    targetFolder: '$(Build.ArtifactStagingDirectory)'
+    targetFolder: '$(Build.ArtifactStagingDirectory)/Logs'
     SourceFolder: 'BaseTools/BaseToolsBuild'
     contents: |
       BASETOOLS_BUILD*.*

Steps/BuildPlatform.yml

@@ -6,6 +6,18 @@
 ##
 
 parameters:
+- name: artifacts_binary
+  displayName: Binary Artifacts to Publish
+  type: string
+  default: ''
+- name: artifacts_identifier
+  displayName: Artifacts Identifier
+  type: string
+  default: 'Artifacts'
+- name: artifacts_other
+  displayName: Other Artifacts to Publish
+  type: string
+  default: ''
 - name: build_arch
   displayName: Architectures (e.g. IA32, X64)
   type: string
@@ -50,10 +62,6 @@ parameters:
   displayName: Tool Chain (e.g. VS2022)
   type: string
   default: ''
-- name: extra_artifacts
-  displayName: Additional Artifacts to Publish
-  type: string
-  default: ''
 
 steps:
 
@@ -121,35 +129,19 @@ steps:
   condition: and(and(gt(variables.pkg_count, 0), succeeded()), eq(variables['Run'], true))
   timeoutInMinutes: ${{ parameters.run_timeout }}
 
-# Copy the build logs to the artifact staging directory
-- task: CopyFiles@2
-  displayName: Copy Build Logs
-  inputs:
-    targetFolder: "$(Build.ArtifactStagingDirectory)"
-    SourceFolder: "Build"
-    contents: |
-      BUILDLOG_*.txt
-      BUILDLOG_*.md
-      CI_*.txt
-      CI_*.md
-      CISETUP.txt
-      SETUPLOG.txt
-      UPDATE_LOG.txt
-      PREVALLOG.txt
-      TestSuites.xml
-      **/BUILD_TOOLS_REPORT.html
-      **/OVERRIDELOG.TXT
-      BASETOOLS_BUILD*.*
-      **/FD_REPORT.HTML
-      ${{ parameters.extra_artifacts }}
-    flattenFolders: true
-  condition: succeededOrFailed()
+# Copy build logs to the artifact staging directory
+- template: CommonLogCopyAndPublish.yml
+  parameters:
+    artifacts_identifier: ${{ parameters.artifacts_identifier }}
 
-# Publish build artifacts to Azure Artifacts/TFS or a file share
-- task: PublishBuildArtifacts@1
-  continueOnError: true
-  displayName: Publish Build Logs
-  inputs:
-    pathtoPublish: "$(Build.ArtifactStagingDirectory)"
-    artifactName: "Build Logs $(System.JobName)"
-  condition: succeededOrFailed()
+# Copy build binaries to the artifact staging directory
+- template: BinaryCopyAndPublish.yml
+  parameters:
+    artifacts_binary: ${{ parameters.artifacts_binary }}
+    artifacts_identifier: ${{ parameters.artifacts_identifier }}
+
+# Copy other files to the artifact staging directory
+- template: OtherCopyAndPublish.yml
+  parameters:
+    artifacts_other: ${{ parameters.artifacts_other }}
+    artifacts_identifier: ${{ parameters.artifacts_identifier }}

Steps/CommonLogCopyAndPublish.yml

@@ -0,0 +1,48 @@
+## @file
+# Azure Pipelines step template to copy the common log files produced
+# by an edk2 firmware build.
+#
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+parameters:
+- name: artifacts_identifier
+  displayName: Artifacts Identifier
+  type: string
+  default: 'Artifacts'
+
+steps:
+- task: CopyFiles@2
+  displayName: Copy Build Logs
+  inputs:
+    targetFolder: "$(Build.ArtifactStagingDirectory)/Logs"
+    SourceFolder: "Build"
+    contents: |
+      **/BUILD_REPORT.TXT
+      **/BUILD_TOOLS_REPORT.html
+      **/BUILD_TOOLS_REPORT.json
+      **/FD_REPORT.HTML
+      **/OVERRIDELOG.TXT
+      BASETOOLS_BUILD*.*
+      BUILDLOG_*.md
+      BUILDLOG_*.txt
+      CI_*.md
+      CI_*.txt
+      CISETUP.txt
+      coverage.html
+      coverage.xml
+      PREVALLOG.txt
+      SETUPLOG.txt
+      TestSuites.xml
+      UPDATE_LOG.txt
+    flattenFolders: true
+  condition: succeededOrFailed()
+
+- task: PublishBuildArtifacts@1
+  continueOnError: true
+  displayName: Publish Build Logs
+  inputs:
+    pathtoPublish: '$(Build.ArtifactStagingDirectory)/Logs'
+    artifactName: 'Logs ${{ parameters.artifacts_identifier }}'
+  condition: succeededOrFailed()