Binlog forward compatibility and deserialisation-less reading support

.editorconfig

@@ -408,3 +408,4 @@ dotnet_diagnostic.IDE0290.severity = suggestion
 # Collection initialization can be simplified
 dotnet_diagnostic.IDE0300.severity = suggestion
 dotnet_diagnostic.IDE0301.severity = suggestion
+dotnet_diagnostic.IDE0305.severity = suggestion

.exp-insertions.yml

@@ -1,20 +1,26 @@
-# Pipeline creates a dotnet with experimental msbuild dlls.
+# Pipeline creates experimental msbuild insertions.
 
 trigger: none # Prevents this pipeline from triggering on check-ins
 pr: none # don't run this on PR as well
 
 parameters:
-  # Dotnet installer channel from where to take the latest dotnet bits.
+  # Dotnet installer channel from which to take the latest dotnet bits.
   - name: DotnetInstallerChannel
     displayName: Dotnet installer channel
     type: string
+    default: 'none'
+  # VS version for which to take the latest Retail MSBuild bits.
+  - name: VSVersionName
+    displayName: VS Version
+    type: string
+    default: 'none'
   # Branch from the MSBuild Build CI pipeline. Default: main
-  # Top run for the branch would be used to create an experimental insertion. 
+  # Top run for the branch would be used to create an experimental insertion.
   - name: MSBuildBranch
     displayName: MSBuild Branch
     type: string
     default: 'refs/heads/main'
-  # BuildID from the MSBuild Build CI pipeline. Overrides the choice of MSBuildBranch parameter 
+  # BuildID from the MSBuild Build CI pipeline. Overrides the choice of MSBuildBranch parameter
   - name: MSBuildBuildID
     displayName: MSBuild CI Run Override
     type: string
@@ -23,105 +29,192 @@ parameters:
 variables:
   - name: _MsBuildCiPipelineId
     value: 9434
+  - name: _MSBuildConfigFilePathRequestURL
+    value: 'https://dev.azure.com/cloudbuild/CloudBuild/_apis/git/repositories/CloudBuildConfig/items?versionDescriptor.version=main&path=config/batmon/Q-Prod-Co3/Coordinator/ToolsReleaseConfig-GeneralPublic.json&api-version=5.0'
+  - name: VSVersion
+    value: ${{parameters.VSVersionName}}
 
 pool:
   vmImage: windows-latest
 
-steps:
-- powershell: |
-    mkdir '$(System.ArtifactsDirectory)/installer'
-
-    $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
-    $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
-
-    foreach ($sdk in $sdks)
-    {
-      Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
-      Invoke-WebRequest `
-        -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
-        -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
-    }
-    mkdir '$(Pipeline.Workspace)/artifacts'
-
-  displayName: Download latest dotnet sdks
-
-# Download latest build artifacts for a branch from MSBuild Build CI
-- ${{ if eq(parameters.MSBuildBuildID, 'default') }}:
+jobs:
+- job: CreateExpDotnet
+  displayName: Create Experimental Dotnet
+  condition: ne('${{ parameters.DotnetInstallerChannel }}', 'none')
+  steps:
+  - powershell: |
+      mkdir '$(System.ArtifactsDirectory)/installer'
+
+      $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
+      $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
+
+      foreach ($sdk in $sdks)
+      {
+        Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
+        Invoke-WebRequest `
+          -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
+          -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
+      }
+      mkdir '$(Pipeline.Workspace)/artifacts'
+    displayName: Download latest dotnet sdks
+
   - task: DownloadBuildArtifacts@1
     inputs:
       buildType: specific
       project: DevDiv
-      pipeline: $(_MsBuildCiPipelineId) 
-      buildVersionToDownload: latestFromBranch
-      branchName: '${{parameters.MSBuildBranch}}'  
+      pipeline: $(_MsBuildCiPipelineId)
+      ${{ if eq(parameters.MSBuildBuildID, 'default') }}:
+        buildVersionToDownload: latestFromBranch
+        branchName: '${{parameters.MSBuildBranch}}'
+      ${{ else }}:
+        buildVersionToDownload: specific
+        buildId: ${{parameters.MSBuildBuildID}}
       artifactName: bin
+      itemPattern: 'MSBuild.Bootstrap/**'
       downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
-      itemPattern: "MSBuild.Bootstrap/**"  
-    displayName: Download latest msbuild from branch
+    displayName: Download msbuild artifacts
+
+  - powershell: |
+      $sdk = "dotnet-sdk-win-x64"
+
+      Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
+      Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+      $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
+      $dotnetVersion = $dotnetDirectory.Name
+      Write-Host "Detected dotnet version: $dotnetVersion"
+
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
+
+      Write-Host "Compressing dotnet sdk files"
+      Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
+
+    displayName: Dogfood msbuild dlls to dotnet sdk win-x64
+
+  - powershell: |
+      $sdk = "dotnet-sdk-linux-x64"
+
+      mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+      Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
+      tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+
+      $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
+      $dotnetVersion = $dotnetDirectory.Name
+      Write-Host "Detected dotnet version: $dotnetVersion"
+
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
+
+      Write-Host "Compressing dotnet sdk files"
+      tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
+    displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Pipeline.Workspace)/artifacts'
+      artifactName: ExperimentalDotnet
+      parallel: true
+    condition: always()
+    displayName: Publish crank assests artifacts
+
+
+- job: CreateExpMSBuild
+  displayName: "Create Experimental MSBuild"
+  condition: ne('${{ parameters.VSVersionName }}', 'none')
+  steps:
+  - powershell: |
+      $token = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes("PAT:$env:ACCESSTOKEN"))
+      $headers = @{ Authorization = "Basic $token" };
+      $response = Invoke-RestMethod -Uri "$(_MSBuildConfigFilePathRequestURL)" -Headers $headers -Method Get
+      $MSBuildDropPath = $response.Tools.MSBuild.Locations
+      Write-Host "##vso[task.setvariable variable=MSBuildDropPath]$MSBuildDropPath"
+      Write-Host "MSBuild Drop Path directory: $MSBuildDropPath"
+    displayName: Get Retail MSBuild Drop Path
+    env:
+      ACCESSTOKEN: $(cloudbuild-token)
+
+  - task: NuGetToolInstaller@1
+    displayName: 'Install NuGet.exe'
+
+  - task: NuGetCommand@2
+    displayName: Restore internal tools
+    inputs:
+      command: restore
+      feedsToUse: config
+      restoreSolution: '$(Build.SourcesDirectory)\eng\common\internal\Tools.csproj'
+      nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
+      restoreDirectory: '$(Build.SourcesDirectory)\.packages'
+
+  - powershell: |
+      mkdir "$(Pipeline.Workspace)/artifacts"
+
+      $dropAppDirectory = Get-ChildItem -Directory -Path "$(Build.SourcesDirectory)/.packages/drop.app"
+      $dropAppVersion = $dropAppDirectory.Name
+      Write-Host "Detected drop.exe version: $dropAppVersion"
+
+      $dropExePath = "$(Build.SourcesDirectory)/.packages/drop.app/$dropAppVersion/lib/net45/drop.exe"
+      Write-Host "Detected drop.exe path: $dropExePath"
+
+      Write-Host "Downloading VS msbuild"
+      & "$dropExePath" get --patAuthEnvVar 'cloudbuild-token' -u "$(MSBuildDropPath)\$(VSVersion)" -d "$(System.ArtifactsDirectory)/VSMSBuildDrop"
+      Write-Host "Download of VS msbuild finished"
+
+      Write-Host "Copying VS msbuild to $(Pipeline.Workspace)/VSMSBuild"
+      Copy-Item -Path "$(System.ArtifactsDirectory)/VSMSBuildDrop/*" -Destination "$(Pipeline.Workspace)/VSMSBuild" -Recurse
+      Write-Host "Copy of VS msbuild finished"
+    displayName: Download msbuild vs drop
+    env:
+      cloudbuild-token: $(cloudbuild-token)
 
-# Download build artifacts for MSBuild Build CI specific build
-- ${{ if ne(parameters.MSBuildBuildID, 'default') }}:
   - task: DownloadBuildArtifacts@1
     inputs:
       buildType: specific
       project: DevDiv
-      pipeline: $(_MsBuildCiPipelineId) 
-      buildVersionToDownload: specific
-      buildId: ${{parameters.MSBuildBuildID}} 
+      pipeline: $(_MsBuildCiPipelineId)
+      ${{ if eq(parameters.MSBuildBuildID, 'default') }}:
+        buildVersionToDownload: latestFromBranch
+        branchName: '${{parameters.MSBuildBranch}}'
+      ${{ else }}:
+        buildVersionToDownload: specific
+        buildId: ${{parameters.MSBuildBuildID}}
       artifactName: bin
+      itemPattern: |
+        MSBuild.Bootstrap/*/net472/**
+        Microsoft.Build.Conversion/*/net472/Microsoft.Build.Conversion.Core.dll
+        Microsoft.Build.Engine/*/net472/Microsoft.Build.Engine.dll
+        MSBuildTaskHost/**/MSBuildTaskHost.exe
+        MSBuildTaskHost/**/MSBuildTaskHost.pdb
+        MSBuild/*/*/net472/MSBuild.exe*
       downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
-      itemPattern: "MSBuild.Bootstrap/**"
-    displayName: Download specified msbuild build
-
-- powershell: |
-    $sdk = "dotnet-sdk-win-x64"
-
-    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
-    Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
-
-    $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
-    $dotnetVersion = $dotnetDirectory.Name
-    Write-Host "Detected dotnet version: $dotnetVersion"
-
-    Write-Host "Updating MSBuild dlls."
-    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
-      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
-      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
-      -configuration Release `
-      -makeBackup $false
-
-    Write-Host "Compressing dotnet sdk files"
-    Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
-
-  displayName: Dogfood msbuild dlls to dotnet sdk win-x64
-
-- powershell: |
-    $sdk = "dotnet-sdk-linux-x64"
-
-    mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
-
-    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
-    tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
-
-    $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
-    $dotnetVersion = $dotnetDirectory.Name
-    Write-Host "Detected dotnet version: $dotnetVersion"
-
-    Write-Host "Updating MSBuild dlls."
-    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
-      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
-      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
-      -configuration Release `
-      -makeBackup $false
-
-    Write-Host "Compressing dotnet sdk files"
-    tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
-  displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
-
-- task: PublishPipelineArtifact@1
-  inputs:
-    targetPath: '$(Pipeline.Workspace)/artifacts'
-    artifactName: ExperimentalDotnet
-    parallel: true
-  condition: always()
-  displayName: Publish crank assests artifacts
+    displayName: Download msbuild artifacts
+
+  - powershell: |
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)/MSBuild/Current/Bin" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
+
+      ls "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)"
+      Write-Host "Compressing msbuild files"
+      Get-ChildItem -Path "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/MSBuild.zip"
+    displayName: Dogfood msbuild dlls
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Pipeline.Workspace)/artifacts'
+      artifactName: ExperimentalMSBuild
+      parallel: true
+    condition: always()
+    displayName: Publish crank assests artifacts

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# .git-blame-ignore-revs
+# Removed trailing whitespace from *.cs,*.props,*.targets
+9a0cef6f75bf13ffbbde956b8f7d7ad7d6e0d996
+# Remove trailing whitespace from many *.cs files
+629447b151dc49cf4d9a57719beff12983cf1907

.github/CODEOWNERS

@@ -0,0 +1,14 @@
+# Users referenced in this file will automatically be requested as reviewers for PRs that modify the given paths.
+# See https://help.github.com/articles/about-code-owners/
+
+/eng/SourceBuild*       @dotnet/source-build-internal
+/eng/                   @dotnet/kitten
+/.github/               @dotnet/kitten
+/global.json            @dotnet/kitten
+/.exp-insertions.yml    @dotnet/kitten
+/.opt-prof.yml          @dotnet/kitten
+/.vsts-dotnet-ci.yml    @dotnet/kitten
+/.vsts-dotnet.yml       @dotnet/kitten
+/NuGet.config           @dotnet/kitten
+/Directory.Build*       @dotnet/kitten
+/.git*                  @dotnet/kitten

.vsconfig

@@ -1,12 +1,7 @@
 {
   "version": "1.0",
   "components": [
-    "Microsoft.Net.Component.4.8.SDK",
-    "Microsoft.Net.Component.4.7.2.SDK",
-    "Microsoft.Net.Component.4.TargetingPack",
-    "Microsoft.Net.Component.4.7.2.TargetingPack",
     "Microsoft.VisualStudio.Workload.ManagedDesktop",
-    "Microsoft.Net.Component.3.5.DeveloperTools",
-    "Microsoft.VisualStudio.Workload.NetCoreTools"
+    "Microsoft.Net.Component.3.5.DeveloperTools"
   ]
 }

.vsts-dotnet.yml

@@ -2,6 +2,7 @@ trigger:
 - main
 - vs*
 - exp/*
+- perf/*
 
 # If defined here, these values are not overrideable
 # Once they exist, we should define these as "runtime parameters"
@@ -95,9 +96,8 @@ stages:
       value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber)
 
     steps:
-    - task: NuGetToolInstaller@0
-      inputs:
-        versionSpec: '4.9.2'
+    - task: NuGetToolInstaller@1
+      displayName: 'Install NuGet.exe'
 
     - task: NuGetCommand@2
       displayName: Restore internal tools
@@ -217,6 +217,7 @@ stages:
       inputs:
         DropName: $(VisualStudio.DropName)
         DropFolder: 'artifacts\VSSetup\$(BuildConfiguration)\Insertion'
+        DropRetentionDays: '30' # extended by insertion + VS release
         AccessToken: '$(System.AccessToken)'
         DropServiceUri: 'https://devdiv.artifacts.visualstudio.com'
         VSDropServiceUri: 'https://vsdrop.corp.microsoft.com/file/v1'
@@ -309,4 +310,5 @@ stages:
         -TsaRepositoryName "dotnet-msbuild"
         -TsaCodebaseName "dotnet-msbuild"
         -TsaPublish $True
+        -CrScanAdditionalRunConfigParams @("SuppressionsPath < $(Build.SourcesDirectory)\eng\CredScanSuppressions.json")
         -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)\eng\policheck_exclusions.xml")'