Merge branch 'main' into issue_485

Author: Andrew

.cargo/config.toml

@@ -3,7 +3,7 @@
 registry-auth = true
 
 [registries]
-powershell = { index = "sparse+https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/Cargo/index/" }
+POWERSHELL = { index = "sparse+https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/Cargo/index/" }
 
 [registry]
 global-credential-providers = ["cargo:token"]
@@ -17,4 +17,10 @@ rustflags = ["-Ccontrol-flow-guard", "-Ctarget-feature=+crt-static", "-Clink-arg
 
 # The following is only needed for release builds
 [source.crates-io]
-replace-with = "powershell"
+replace-with = "POWERSHELL"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"

.github/workflows/rust.yml

@@ -28,6 +28,22 @@ jobs:
       shell: pwsh
       run: ./build.ps1 -test
 
+  build-musl:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - run: rustup target add x86_64-unknown-linux-musl
+    - name: Install musl build tools
+      run: sudo apt update && sudo apt install musl-tools -y
+    - name: Build
+      shell: pwsh
+      run: ./build.ps1 -clippy -target x86_64-unknown-linux-musl
+    - name: Run tests
+      shell: pwsh
+      run: ./build.ps1 -test -target x86_64-unknown-linux-musl
+
   build-windows:
 
     runs-on: windows-latest

.pipelines/DSC-Official.yml

@@ -1,23 +1,26 @@
 name: DSC-Release-$(Build.BuildId)
 trigger: none
 
-parameters:
-- name: 'debugConsole'
-  displayName: 'Enable debug console'
-  type: boolean
-  default: false
-
 pr:
   branches:
     include:
     - onebranch
     - release*
 
+schedules:
+- cron: '0 3 * * 1'
+  displayName: Weekly Build
+  branches:
+    include:
+    - main
+  always: true
+
 variables:
   BuildConfiguration: 'release'
   PackageRoot: '$(System.ArtifactsDirectory)/Packages'
-  LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0'
-  WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2019/vse2022:latest
+#  LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0'
+  LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest'
+  WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
 
 resources:
   repositories:
@@ -30,7 +33,12 @@ extends:
   template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
   parameters:
     featureFlags:
-      debugConsole: ${{ parameters.debugConsole }}
+      WindowsHostVersion: 
+        Disk: Large
+        Version: 2022
+        Network: KS1 # note that this property is sticky so commenting out will use the previous set one
+# Currently can't be used as some NPM pkgs like tree-sitter-cli reach out to GitHub to get the actual zip pkg
+#        Network: NetLock
     customTags: 'ES365AIMigrationTooling'
     globalSdl:
       disableLegacyManifest: true
@@ -51,7 +59,7 @@ extends:
         apiscan:
           enabled: false
 
-    stages:
+    stages:     
     - stage: BuildAndSign
       displayName: Build Native Binaries
       dependsOn: []
@@ -60,6 +68,7 @@ extends:
         displayName: Set PackageVersion
         pool:
           type: windows
+          vmImage: windows-latest
         variables:
           repoRoot: $(Build.SourcesDirectory)\DSC
           ob_sdl_tsa_configFile: $(Build.SourcesDirectory)\DSC\.config\tsaoptions.json
@@ -124,24 +133,15 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
-          inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
-
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
-          env:
-            ob_restore_phase: true
         - pwsh: |
             Set-Location "$(Build.SourcesDirectory)/DSC"
-            ./build.ps1 -Release -Architecture $(buildName) -SkipLinkCheck -UseCFS
+            $LLVMBIN = "$($env:PROGRAMFILES)\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm\bin"
+            if (!(Test-Path $LLVMBIN)) {
+              throw "LLVM path '$LLVMBIN' does not exist"
+            }
+            $env:PATH += ";$LLVMBIN"
+            write-verbose -verbose (gcm clang.exe | out-string)
+            ./build.ps1 -Release -Architecture $(buildName) -SkipLinkCheck
           displayName: 'Build $(buildName)'
           env:
             ob_restore_phase: true
@@ -201,16 +201,20 @@ extends:
         dependsOn: BuildWin
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+          ob_sdl_tsa_configFile: '$(Build.SourcesDirectory)\DSC\.config\tsaoptions.json'
+          ob_sdl_sbom_enabled: true
+          ob_signing_setup_enabled: true
+          ob_sdl_codeql_compiled_enabled: true
         pool:
           type: windows
         steps:
+        - checkout: self
         - download: current
           artifact: drop_BuildAndSign_BuildWinx64
           patterns: '*.msix'
         - download: current
           artifact: drop_BuildAndSign_BuildWinarm64
           patterns: '*.msix'
-        - checkout: self
         - pwsh: |
             Set-Location "$(Build.SourcesDirectory)/DSC"
             $null = New-Item -ItemType Directory -Path "./bin/msix" -Force -ErrorAction Ignore
@@ -221,25 +225,6 @@ extends:
           displayName: 'Create msixbundle'
           condition: succeeded()
 
-      - job: PublishSigned
-        dependsOn: BuildWin
-        variables:
-          signOutPath: $[ dependencies.BuildWin.outputs['signOutPath.signOutPath'] ]
-          ob_sdl_tsa_configFile: $(Build.SourcesDirectory)\DSC\.config\tsaoptions.json
-          ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
-          ob_sdl_sbom_enabled: false
-          ob_signing_setup_enabled: false
-          ob_sdl_codeql_compiled_enabled: false
-        pool:
-          type: windows
-        steps:
-        - task: CopyFiles@2
-          displayName: "Copy Files for 'PublishPipelineArtifact@1' publish task"
-          inputs:
-            SourceFolder: $(signOutPath)
-            Contents: '**'
-            TargetFolder: $(Build.ArtifactStagingDirectory)/signed
-
       - job: BuildLinux
         dependsOn: SetPackageVersion
         variables:
@@ -257,23 +242,8 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
-          inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
-
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
-          env:
-            ob_restore_phase: true
         - pwsh: |
-            ./build.ps1 -Release -Architecture x86_64-unknown-linux-gnu -UseCFS
+            ./build.ps1 -Release -Architecture x86_64-unknown-linux-gnu
             ./build.ps1 -PackageType tgz -Architecture x86_64-unknown-linux-gnu -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
           displayName: 'Build x86_64-unknown-linux-gnu'
@@ -287,7 +257,6 @@ extends:
         displayName: Linux-ARM64-gnu
         pool:
           type: linux
-          hostArchitecture: arm64
         steps:
         - task: RustInstaller@1
           inputs:
@@ -297,26 +266,72 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
+        - pwsh: |
+            apt update
+            apt -y install gcc-aarch64-linux-gnu
+            ./build.ps1 -Release -Architecture aarch64-unknown-linux-gnu
+            ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-gnu -Release
+            Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
+          displayName: 'Build aarch64-unknown-linux-gnu'
+          condition: succeeded()
+
+      - job: BuildLinuxMusl
+        dependsOn: SetPackageVersion
+        variables:
+          PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ]
+          ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+        displayName: Linux-x64-musl
+        pool:
+          type: linux
+        steps:
+        - task: RustInstaller@1
           inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
+            rustVersion: ms-stable
+            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            additionalTargets: x86_64-unknown-linux-musl
+          displayName: Install Rust
+          env:
+            ob_restore_phase: true
+        - pwsh: |
+            apt update
+            apt -y install musl-tools
+            ./build.ps1 -Release -Architecture x86_64-unknown-linux-musl
+            ./build.ps1 -PackageType tgz -Architecture x86_64-unknown-linux-musl -Release
+            Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
+          displayName: 'Build x86_64-unknown-linux-musl'
+          condition: succeeded()
 
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
+      - job: BuildLinuxArm64Musl
+        dependsOn: SetPackageVersion
+        variables:
+          PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ]
+          ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+        displayName: Linux-ARM64-musl
+        pool:
+          type: linux
+        steps:
+        - task: RustInstaller@1
+          inputs:
+            rustVersion: ms-stable
+            toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json
+            additionalTargets: aarch64-unknown-linux-musl
+          displayName: Install Rust
           env:
             ob_restore_phase: true
         - pwsh: |
-            ./build.ps1 -Release -Architecture aarch64-unknown-linux-gnu -UseCFS
-            ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-gnu -Release
+            $env:CC_aarch64_unknown_linux_musl='clang'
+            $env:AR_aarch64_unknown_linux_musl='llvm-ar'
+            $env:CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"
+            apt update
+            apt -y install clang
+            apt -y install llvm
+            apt -y install musl-tools
+            apt -y install gcc-multilib
+            rustup default stable-aarch64-unknown-linux-musl
+            ./build.ps1 -Release -Architecture aarch64-unknown-linux-musl
+            ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-musl -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
-          displayName: 'Build aarch64-unknown-linux-gnu'
+          displayName: 'Build aarch64-unknown-linux-musl'
           condition: succeeded()
 
       - job: BuildMac
@@ -345,23 +360,8 @@ extends:
           displayName: Install Rust
           env:
             ob_restore_phase: true
-        - task: AzureCLI@2
-          inputs:
-            azureSubscription: az-PowerShell-feed-ingestion
-            scriptType: 'pscore'
-            scriptLocation: 'inlineScript'
-            inlineScript: |
-              $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv
-
-              # Set the access token as a secret, so it doesn't get leaked in the logs
-              Write-Host "##vso[task.setsecret]$accessToken"
-              $header = "Bearer $accessToken"
-              Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_POWERSHELL_TOKEN]$header"
-          displayName: 'Get Azure DevOps Token'
-          env:
-            ob_restore_phase: true
         - pwsh: |
-            ./build.ps1 -Release -Architecture $(buildName) -UseCFS
+            ./build.ps1 -Release -Architecture $(buildName)
             ./build.ps1 -PackageType tgz -Architecture $(buildName) -Release
             Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)"
             Write-Host "##vso[artifact.upload containerfolder=release;artifactname=release]$(ob_outputDirectory)/DSC-$(PackageVersion)-$(buildName).tar.gz"
@@ -370,9 +370,9 @@ extends:
 
     - stage: Release
       dependsOn: BuildAndSign
+      condition: ne(variables['Build.Reason'], 'Schedule')
       variables:
         PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ]
-        drop: $(Pipeline.Workspace)/drop_build_main
       jobs:
       - job: Validation
         displayName: Manual validation
@@ -392,10 +392,17 @@ extends:
         pool:
           type: windows
         variables:
-          ob_outputDirectory: '$(Build.SourcesDirectory)'
+          ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
+          ob_sdl_sbom_enabled: false
+          ob_signing_setup_enabled: false
+          ob_sdl_codeql_compiled_enabled: false
+          drop: $(Pipeline.Workspace)/drop_build_main
         steps:
         - download: current
           displayName: Download artifacts
+          patterns: |
+            '**/*.zip'
+            '**/*.tar.gz'
         - task: GitHubRelease@1
           displayName: Create GitHub release
           inputs:

.vscode/settings.json

@@ -2,6 +2,7 @@
     "rust-analyzer.linkedProjects": [
         "./dsc/Cargo.toml",
         "./dsc_lib/Cargo.toml",
+        "./echo/Cargo.toml",
         "./osinfo/Cargo.toml",
         "./registry/Cargo.toml",
         "./runcommandonset/Cargo.toml",
@@ -21,5 +22,8 @@
         "schemas/2024/04/bundled/config/document.vscode.json": "**.dsc.{yaml,yml,config.yaml,config.yml}",
         "schemas/2024/04/bundled/resource/manifest.vscode.json": "**.dsc.resource.{yaml,yml}"
     },
-    "sarif-viewer.connectToGithubCodeScanning": "off"
+    "sarif-viewer.connectToGithubCodeScanning": "off",
+    "vscode-nmake-tools.workspaceBuildDirectories": [
+        "."
+    ]
 }