diff --git a/.azure-pipelines/common-templates/install-tools.yml b/.azure-pipelines/common-templates/install-tools.yml index a4c9c40cc8..732696fa77 100644 --- a/.azure-pipelines/common-templates/install-tools.yml +++ b/.azure-pipelines/common-templates/install-tools.yml @@ -48,23 +48,70 @@ steps: inputs: workingFile: $(Build.SourcesDirectory)/autorest.powershell/common/config/rush/.npmrc + - task: PowerShell@2 + displayName: Apply npm auth config to user profile + inputs: + targetType: inline + pwsh: true + script: | + # Copy authenticated .npmrc to the user home dir so ALL npm subprocesses + # (including autorest's internal npm calls) use the private feed + auth tokens. + $src = "$(Build.SourcesDirectory)/.npmrc" + $dst = Join-Path $env:USERPROFILE ".npmrc" + Copy-Item -Path $src -Destination $dst -Force + Write-Host "Copied npm config to $dst" + - task: Npm@1 displayName: Install AutoRest inputs: command: custom - customCommand: install -g autorest@3.7.2 - + workingDir: $(Build.SourcesDirectory) + customCommand: install -g autorest@3.7.2 --registry https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/PowerShell_V2_Build/npm/registry/ + - task: Npm@1 displayName: Install AutorestCore inputs: command: custom - customCommand: install -g @autorest/core@3.10.4 + workingDir: $(Build.SourcesDirectory) + customCommand: install -g @autorest/core@3.10.4 --registry https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/PowerShell_V2_Build/npm/registry/ + + - task: PowerShell@2 + displayName: Pre-populate autorest extension cache + inputs: + targetType: inline + pwsh: true + script: | + # Autorest resolves extensions from ~/.autorest///node_modules/. + # Pre-install every extension referenced by autorest-configuration.md so autorest + # makes zero npm network calls during module generation. + $registry = "https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/PowerShell_V2_Build/npm/registry/" + $extensions = @( + "@autorest/core@3.10.4", + "@autorest/modelerfour@4.24.3" + ) + foreach ($ext in $extensions) { + $parts = $ext -split '@(?=[^@]+$)' # split on last @ + $pkg = $parts[0] + $ver = $parts[1] + $cacheDir = Join-Path $env:USERPROFILE ".autorest\$($pkg.Replace('/','\'))\$ver" + $nodeModules = Join-Path $cacheDir "node_modules\$($pkg.Replace('/','\'))" + if (Test-Path $nodeModules) { + Write-Host "Cache already present: $ext" + continue + } + New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null + Write-Host "Pre-installing $ext into $cacheDir" + npm install $ext --prefix $cacheDir --registry $registry + if ($LASTEXITCODE -ne 0) { throw "Failed to pre-install $ext (exit $LASTEXITCODE)" } + Write-Host "Done: $ext" + } - task: Npm@1 displayName: Install Rush inputs: command: custom - customCommand: install -g @microsoft/rush + workingDir: $(Build.SourcesDirectory) + customCommand: install -g @microsoft/rush --registry https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/PowerShell_V2_Build/npm/registry/ - task: PowerShell@2 displayName: Rush Build @@ -74,5 +121,8 @@ steps: workingDirectory: "autorest.powershell" script: | rush install + if ($LASTEXITCODE -ne 0) { throw "rush install failed with exit code $LASTEXITCODE" } rush link - rush rebuild \ No newline at end of file + if ($LASTEXITCODE -ne 0) { throw "rush link failed with exit code $LASTEXITCODE" } + rush rebuild + if ($LASTEXITCODE -ne 0) { throw "rush rebuild failed with exit code $LASTEXITCODE" } \ No newline at end of file diff --git a/tools/Configure-PrivateNpmFeed.ps1 b/tools/Configure-PrivateNpmFeed.ps1 index e8d6f75ada..5689165590 100644 --- a/tools/Configure-PrivateNpmFeed.ps1 +++ b/tools/Configure-PrivateNpmFeed.ps1 @@ -35,3 +35,18 @@ Write-Host "Created $rootNpmrc" $rushNpmrc = Join-Path $SourcesDirectory "autorest.powershell/common/config/rush/.npmrc" Set-Content -Path $rushNpmrc -Value $npmrcContent -NoNewline Write-Host "Updated $rushNpmrc" + +# Create NuGet.config to redirect dotnet restore to the private feed +$nugetFeed = $Registry -replace "/npm/registry/$", "/nuget/v3/index.json" +$nugetConfig = @" + + + + + + + +"@ +$nugetConfigPath = Join-Path $SourcesDirectory "NuGet.config" +Set-Content -Path $nugetConfigPath -Value $nugetConfig -NoNewline +Write-Host "Created $nugetConfigPath" diff --git a/tools/GenerateModules.ps1 b/tools/GenerateModules.ps1 index ce7f44b411..f169452cac 100644 --- a/tools/GenerateModules.ps1 +++ b/tools/GenerateModules.ps1 @@ -32,10 +32,6 @@ if (-not $Isolated) { # Module import. Import-Module PowerShellGet -# Install Powershell-yaml -if (!(Get-Module -Name powershell-yaml -ListAvailable)) { - Install-Module powershell-yaml -Repository PSGallery -Scope CurrentUser -Force -} $ScriptRoot = $PSScriptRoot $ModulesSrc = Join-Path $ScriptRoot "..\src\" @@ -52,9 +48,39 @@ if (-not (Test-Path $ModuleMappingPath)) { # Build AutoREST.PowerShell submodule. Set-Location (Join-Path $ScriptRoot "../autorest.powershell") -rush update --purge +rush install rush build +# Diagnostic: show autorest cache state and npm registry config before generation. +Write-Host "--- Autorest/npm diagnostics ---" +$autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" } +Write-Host "AUTOREST_HOME: $autorestHome" +$coreCacheDir = Join-Path $autorestHome "@autorest\core" +if (Test-Path $coreCacheDir) { + Write-Host "Autorest core cache versions: $(Get-ChildItem $coreCacheDir -Directory | Select-Object -ExpandProperty Name)" + $nodeModulesDir = Join-Path $coreCacheDir "3.10.4\node_modules\@autorest\core" + Write-Host "Cache 3.10.4 node_modules present: $(Test-Path $nodeModulesDir)" +} else { + Write-Host "WARNING: Autorest core cache directory not found at $coreCacheDir" +} +$modelerfourCacheDir = Join-Path $autorestHome "@autorest\modelerfour" +if (Test-Path $modelerfourCacheDir) { + Write-Host "Autorest modelerfour cache versions: $(Get-ChildItem $modelerfourCacheDir -Directory | Select-Object -ExpandProperty Name)" + $modelerfourNodeModules = Join-Path $modelerfourCacheDir "4.24.3\node_modules\@autorest\modelerfour" + Write-Host "Cache modelerfour 4.24.3 node_modules present: $(Test-Path $modelerfourNodeModules)" +} else { + Write-Host "WARNING: Autorest modelerfour cache directory not found at $modelerfourCacheDir" +} +Write-Host "npm registry: $(npm config get registry 2>&1)" +Write-Host "npm_config_registry env: $env:npm_config_registry" +Write-Host "NPM_CONFIG_USERCONFIG: $env:NPM_CONFIG_USERCONFIG" +$userNpmrc = Join-Path $env:USERPROFILE ".npmrc" +Write-Host "~/.npmrc exists: $(Test-Path $userNpmrc)" +if (Test-Path $userNpmrc) { + Write-Host "~/.npmrc registry line: $(Select-String -Path $userNpmrc -Pattern '^registry=' | Select-Object -First 1 -ExpandProperty Line)" +} +Write-Host "--- End diagnostics ---" + $RequiredGraphModules = @() $AuthModuleManifest = Join-Path $ModulesSrc "Authentication" "Authentication" "artifacts" "Microsoft.Graph.Authentication.psd1" $LoadedAuthModule = Import-Module $AuthModuleManifest -PassThru -ErrorAction SilentlyContinue diff --git a/tools/GenerateServiceModule.ps1 b/tools/GenerateServiceModule.ps1 index d5385173ac..b5fb25b05d 100644 --- a/tools/GenerateServiceModule.ps1 +++ b/tools/GenerateServiceModule.ps1 @@ -69,10 +69,33 @@ $ApiVersion | ForEach-Object { else { $FullModuleVersion = $ModuleMetadata.versions[$CurrentApiVersion].version } - npx autorest --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2 - if ($LastExitCode -ne 0) { - Write-Host -ForegroundColor Red "AutoREST failed to generate '$ModuleFullName' module." - return $LastExitCode + # Pass @autorest/modelerfour as a local --use: argument so autorest loads it + # directly from the pre-populated cache without calling fetchPackageMetadata. + # fetchPackageMetadata is called unconditionally before the cache check inside + # ExtensionManager.findPackage, and it hits the npm registry which is DNS-blocked + # by 1ES supply-chain security on release pipelines. A local --use path bypasses + # findPackage entirely: the extension goes straight into localExtensions, and when + # use-extension in autorest-configuration.md is processed, resolveExtension finds + # it there without any registry call. + $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" } + $modelerFourPath = Join-Path $autorestHome "@autorest" "modelerfour" "4.24.3" "node_modules" "@autorest" "modelerfour" + # @(if ...) always produces [object[]], preventing PowerShell from unwrapping the + # single-element array to a [string] scalar. A scalar string splatted with @ + # enumerates IEnumerable, passing each character as a separate argument. + $modelerFourUseFlag = @(if (Test-Path $modelerFourPath) { "--use:$modelerFourPath" }) + if ($modelerFourUseFlag.Count -eq 0) { + Write-Host -ForegroundColor Yellow "WARNING: @autorest/modelerfour local cache not found at $modelerFourPath — autorest will attempt npm registry lookup (may fail in network-isolated environment)" + } + + $autorestLog = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "autorest-$($ModuleFullName -replace '[^a-zA-Z0-9-]', '-').log") + npx autorest @modelerFourUseFlag --verbose --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2 2>&1 | Out-File -FilePath $autorestLog -Encoding utf8 + $autorestExitCode = $LASTEXITCODE + if ($autorestExitCode -ne 0) { + Write-Host -ForegroundColor Red "AutoREST failed (exit $autorestExitCode) generating '$ModuleFullName'." + Write-Host -ForegroundColor Yellow "=== AutoREST log: $autorestLog ===" + if (Test-Path $autorestLog) { Get-Content $autorestLog | ForEach-Object { Write-Host $_ } } + Write-Host -ForegroundColor Yellow "=== End AutoREST log ===" + return $autorestExitCode } Write-Debug "AutoRest generated '$ModuleFullName' successfully." diff --git a/tools/ManageGeneratedModule.ps1 b/tools/ManageGeneratedModule.ps1 index 5d0e9ff48a..9ea2761727 100644 --- a/tools/ManageGeneratedModule.ps1 +++ b/tools/ManageGeneratedModule.ps1 @@ -47,7 +47,7 @@ foreach ($Package in $NugetPackagesToRemove) { # Add nuget packages from generate modules. foreach ($Package in $NugetPackagesToAdd) { Write-Debug "Executing: dotnet add $ModuleCsProj package $Package" - dotnet add $ModuleCsProj package $Package -s https://api.nuget.org/v3/index.json | Out-Null + dotnet add $ModuleCsProj package $Package | Out-Null if ($LastExitCode) { Write-Error "Failed to execute: dotnet add $ModuleCsProj package $Package" } diff --git a/tools/ReadModuleReadMe.ps1 b/tools/ReadModuleReadMe.ps1 index a6f8c5c196..93b07bfa48 100644 --- a/tools/ReadModuleReadMe.ps1 +++ b/tools/ReadModuleReadMe.ps1 @@ -6,12 +6,24 @@ param( [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $FieldToRead ) $ErrorActionPreference = "Stop" + +function ConvertFrom-SimpleYaml { + param([string]$Yaml) + $result = @{} + $Yaml -split "`n" | ForEach-Object { + if ($_.Trim() -match '^([^:]+):\s*(.*)$') { + $result[$Matches[1].Trim()] = $Matches[2].Trim() + } + } + return $result +} + $FieldValue = $null # Read readme.md. $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning" if ($ReadMeContent.Length -eq 2) { # Convert versioning section to yaml. - $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml + $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml $FieldValue = $VersioningSection[$FieldToRead] } return $FieldValue diff --git a/tools/WriteToModuleReadMe.ps1 b/tools/WriteToModuleReadMe.ps1 index 0a921551ee..5ac0234e47 100644 --- a/tools/WriteToModuleReadMe.ps1 +++ b/tools/WriteToModuleReadMe.ps1 @@ -7,12 +7,24 @@ param( [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $NewFieldValue ) $ErrorActionPreference = "Stop" + +function ConvertFrom-SimpleYaml { + param([string]$Yaml) + $result = @{} + $Yaml -split "`n" | ForEach-Object { + if ($_.Trim() -match '^([^:]+):\s*(.*)$') { + $result[$Matches[1].Trim()] = $Matches[2].Trim() + } + } + return $result +} + # Read readme.md. $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning" if ($ReadMeContent.Length -eq 2) { # Convert versioning section to yaml. $UpdatedVersionSection = "### Versioning" + $ReadMeContent[1] - $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml + $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml $FieldValue = $VersioningSection[$FieldName] $RegexPattern = "$FieldName`:\s*$FieldValue" $UpdatedVersionSection = $UpdatedVersionSection -replace $RegexPattern, "$FieldName`: $NewFieldValue"