diff --git a/src/GraphWrapperRuntime/Runtime/GraphClientCmdlet.cs b/src/GraphWrapperRuntime/Runtime/GraphClientCmdlet.cs index 1ca58994035..a09e873faf3 100644 --- a/src/GraphWrapperRuntime/Runtime/GraphClientCmdlet.cs +++ b/src/GraphWrapperRuntime/Runtime/GraphClientCmdlet.cs @@ -13,7 +13,7 @@ namespace Microsoft.Graph.Wrapper.Runtime // The shared skeleton of every generated wrapper cmdlet: the -AccessToken/-Headers surface, // transport acquisition, and Graph error translation. Derived cmdlets own only what is // unique to their operation - path parameters, body binding, and the request itself. - public abstract class GraphClientCmdlet : PSCmdlet + public abstract class GraphClientCmdlet : PSCmdlet, IDisposable { // One shared HttpClient for every -AccessToken invocation in the process. The token // rides per-request (see StaticBearerTokenAuthenticationProvider), so different tokens @@ -79,6 +79,31 @@ protected void AddRequestHeaders(RequestHeaders requestHeaders) } } + private readonly System.Threading.CancellationTokenSource _stopping = new System.Threading.CancellationTokenSource(); + + // Ctrl+C while a single request is in flight. Checking Stopping between requests cannot + // interrupt one that is already running, so the token is handed to the kiota call instead. + // + // Cmdlet.PipelineStopToken would be the direct route and IS declared by + // PowerShellStandard.Library, but it does not exist in the Windows PowerShell 5.1 runtime + // these netstandard2.0 modules also target - it would compile, pass on PowerShell 7, and + // throw on 5.1. StopProcessing is virtual on every supported edition, so the token is + // raised from there. + protected System.Threading.CancellationToken StoppingToken => _stopping.Token; + + protected override void StopProcessing() + { + _stopping.Cancel(); + base.StopProcessing(); + } + + // PowerShell disposes a cmdlet that implements IDisposable once the pipeline ends. + public void Dispose() + { + _stopping.Dispose(); + GC.SuppressFinalize(this); + } + // The single error surface for a failed Graph call, identical across every cmdlet. protected void ThrowGraphRequestFailed(Exception exception, object? targetObject) { diff --git a/tools/Build-WrapperModule.ps1 b/tools/Build-WrapperModule.ps1 index fb18ab69fbc..436f28fa6de 100644 --- a/tools/Build-WrapperModule.ps1 +++ b/tools/Build-WrapperModule.ps1 @@ -55,17 +55,6 @@ dotnet build configuration. Default: Debug. .PARAMETER SkipKiota Reuse the previously generated client (fast inner loop when only the wrappers changed). -.PARAMETER ModuleVersion -Three-part version for the wrapper packages. Default 3.0.0 - the wrapper modules are the v3 -line, not a build of the v2 service-module release train whose version lives in -config/ModuleMetadata.json. - -.PARAMETER Prerelease -Prerelease label carried by every package (alphanumeric, never empty). Defaults to -alpha in CI and alpha locally, so no two builds ever publish the same -id and version with different contents. Wrapper packages stay prereleases until the quality -bar for public distribution is agreed. - .PARAMETER Pack Also produce a package per module under //. @@ -90,23 +79,6 @@ param( [string]$Configuration = 'Debug', [string]$ModuleMappingConfigPath, [string]$ArtifactsLocation, - # The wrapper modules are the v3 line, not a build of the v2 service-module release train, - # so their version is their own rather than ModuleMetadata.json's (which belongs to v2 and - # is still read here for authors, tags and the rest of the package identity). - [ValidatePattern('^\d+\.\d+\.\d+$')] - [string]$ModuleVersion = '3.0.0', - # Wrapper packages always carry a prerelease label, and every build gets a DISTINCT one: - # two packages that share an id and version but not their contents are indistinguishable to - # a feed and to anyone who already installed one. The build id supplies that distinctness in - # CI; a UTC timestamp does locally, where no build id exists. - # The pattern is DELIBERATELY narrower than the PowerShellGet prerelease grammar rather than a - # restatement of it: PowerShellGet also accepts a hyphen, which this rejects. A label is joined - # to the version by a hyphen already, so one that itself begins with a hyphen reads as - # '3.0.0--label', and one containing a hyphen splits the label when a version string is parsed - # by eye. Alphanumeric-only keeps every generated label unambiguous, and both defaults above - # ('alpha' plus a build id or a UTC timestamp) satisfy it by construction. - [ValidatePattern('^[A-Za-z0-9]+$')] - [string]$Prerelease = "alpha$(if ($env:BUILD_BUILDID) { $env:BUILD_BUILDID } else { (Get-Date).ToUniversalTime().ToString('yyyyMMddHHmm') })", [switch]$SkipKiota, [switch]$Pack ) @@ -119,7 +91,7 @@ $generatorProject = Join-Path $repoRoot 'tools\WrapperGenerator' $authCsproj = Join-Path $repoRoot 'src\Authentication\Authentication\Microsoft.Graph.Authentication.csproj' $runtimeCsproj = Join-Path $repoRoot 'src\GraphWrapperRuntime\Runtime\Microsoft.Graph.Wrapper.Runtime.csproj' # The Authentication version the wrappers compile against, for the manifest's RequiredModules -# minimum and the nuspec dependency floor. Read from the project, never written here. +# minimum. Read from the project, never written here. $authVersion = ([xml](Get-Content $authCsproj -Raw)).Project.PropertyGroup.Version | Where-Object { $_ } | Select-Object -First 1 if (-not $authVersion) { throw "no Version in $authCsproj" } @@ -138,17 +110,16 @@ $targetFramework = "$targetFramework".Trim() if (-not $ModuleMappingConfigPath) { $ModuleMappingConfigPath = Join-Path $repoRoot 'config\ModulesMapping.jsonc' } if (-not $ArtifactsLocation) { $ArtifactsLocation = Join-Path $repoRoot 'artifacts' } -# Package identity (authors, owners, licence, tags) comes from the same single source the -# AutoRest service modules use, so a wrapper package and the module it will eventually replace -# cannot disagree about ownership. The VERSION is deliberately not taken from there: that -# entry tracks the v2 release train, and a wrapper package stamped with it would claim a -# version the real SDK is about to publish. +# Package metadata comes from the same single source the AutoRest service modules use, so a +# wrapper package and the module it will eventually replace cannot disagree about version or +# ownership. $moduleMetadataPath = Join-Path $repoRoot 'config\ModuleMetadata.json' [hashtable]$moduleMetadata = Get-Content $moduleMetadataPath -Raw | ConvertFrom-Json -AsHashTable -# The metadata prerelease field belongs to the service-module release train and is deliberately -# not read; the wrapper label (validated non-empty) is appended unconditionally, because a -# wrapper package without one cannot exist. -$fullVersion = "$ModuleVersion-$Prerelease" +$versionEntry = $moduleMetadata.versions[$ApiVersion] +if (-not $versionEntry -or -not $versionEntry.version) { throw "No version configured for '$ApiVersion' in $moduleMetadataPath." } +$moduleVersion = $versionEntry.version +$modulePrerelease = $versionEntry.prerelease +$fullVersion = if ($modulePrerelease) { "$moduleVersion-$modulePrerelease" } else { $moduleVersion } # The population is the specs this generator can actually read, intersected with the modules the # repository is configured to ship - the same two inputs tools/GenerateServiceModule.ps1 uses. @@ -191,32 +162,6 @@ function New-ProjectFromTemplate { Set-Content -Path $DestinationPath -Value $content -Encoding utf8 } -# The manifest GUID is a module's identity for ModuleSpecification matching (RequiredModules, -# Import-Module -FullyQualifiedName), so it must be the SAME across builds. The shipped SDK -# locks GUIDs to the published gallery entry (tools/BuildModule.ps1, autorest.powershell#981); -# a never-published wrapper has no gallery entry to lock to, so its identity is DERIVED -# instead: an RFC 4122 name-based (v5) UUID - SHA-1 over a fixed namespace GUID plus the module -# name - identical on every build with no lookup table to maintain. The namespace constant and -# the algorithm ARE the identity contract: changing either orphans every previously installed -# wrapper module. -function Get-WrapperModuleGuid { - param([Parameter(Mandatory)][string]$ModuleName) - - $namespaceBytes = ([guid]'8a11e1b5-95b3-4dbf-b0ba-6d58b0f6f6a4').ToByteArray() - # Guid.ToByteArray() emits the first three fields little-endian; RFC 4122 hashes network order. - [Array]::Reverse($namespaceBytes, 0, 4); [Array]::Reverse($namespaceBytes, 4, 2); [Array]::Reverse($namespaceBytes, 6, 2) - $sha1 = [System.Security.Cryptography.SHA1]::Create() - try { - $hash = $sha1.ComputeHash([byte[]]($namespaceBytes + [System.Text.Encoding]::UTF8.GetBytes($ModuleName))) - } - finally { $sha1.Dispose() } - $guidBytes = $hash[0..15] - $guidBytes[6] = ($guidBytes[6] -band 0x0F) -bor 0x50 # version 5 - $guidBytes[8] = ($guidBytes[8] -band 0x3F) -bor 0x80 # RFC 4122 variant - [Array]::Reverse($guidBytes, 0, 4); [Array]::Reverse($guidBytes, 4, 2); [Array]::Reverse($guidBytes, 6, 2) - [guid][byte[]]$guidBytes -} - function Get-CompiledCmdletNames { param([Parameter(Mandatory)][string]$AssemblyPath) @@ -395,9 +340,7 @@ function Build-Module { $manifestArgs = @{ Path = $psd1Path RootModule = "$moduleName.dll" - Guid = Get-WrapperModuleGuid -ModuleName $moduleName - ModuleVersion = $ModuleVersion - Prerelease = $Prerelease + ModuleVersion = $moduleVersion RequiredModules = @(@{ ModuleName = 'Microsoft.Graph.Authentication'; ModuleVersion = $authVersion }) Author = 'Microsoft Graph' CompanyName = 'Microsoft' @@ -407,15 +350,14 @@ function Build-Module { AliasesToExport = @() VariablesToExport = @() } - # Std.UriTemplate is requested by Microsoft.Kiota.Abstractions (measured: the AssemblyRef - # lives there, not in the HTTP library). That requester sits in Authentication's isolated - # load context, so its directory probing can never reach this module folder - and - # Authentication cannot serve the request either, because its Dependencies folder does - # not ship the assembly. Preloading it here puts it in the default context, where the - # isolated context's fallback resolution unifies with it. The Multipart serializer needs - # no entry: its requester is the client assembly in this folder, so normal - # module-directory probing finds it. Proven live by tools/Test-WrapperLive.ps1 on the - # session path. + if ($modulePrerelease) { $manifestArgs.Prerelease = $modulePrerelease } + # Std.UriTemplate is requested by the kiota HTTP library, which lives in Authentication's + # isolated load context - a requester whose directory probing can never reach this module + # folder, and whose request Authentication cannot serve because its Dependencies folder + # does not ship the assembly. Preloading it here puts it in the default context, where + # the isolated context's fallback resolution unifies with it. The Multipart serializer + # needs no entry: its requester is the client assembly in this folder, so normal + # module-directory probing finds it. Proven live in Test-LiveSmoke on the session path. $manifestArgs.RequiredAssemblies = @('Std.UriTemplate.dll') New-ModuleManifest @manifestArgs @@ -426,12 +368,6 @@ function Build-Module { # and the shared Authentication/kiota closure via the PruneModuleBin target - both at # the project level, the only place that intent can be expressed. See # tools/Templates/WrapperModule.csproj.template for why the closure must not ship. - # The nuspec must declare the Authentication dependency even though the psd1 already - # does: Install-Module resolves dependencies from NUGET metadata, not the manifest, - # so without this element a clean machine gets the wrapper with no Authentication and - # import fails. Declared as an open floor, not the shipped SDK's exact bracket pin - - # matching the manifest's RequiredModules minimum and the use-latest ruling (Ramses, - # 2026-08-19: pins existed only for AutoRest limitations). $nuspecPath = Join-Path $binDir "$moduleName.nuspec" $tags = ($moduleMetadata['tags']) -join ' ' # Native runtime payloads only exist when a dependency ships them; dotnet pack fails @@ -456,9 +392,6 @@ function Build-Module { $($moduleMetadata['releaseNotes']) $($moduleMetadata['copyright']) $tags - - - @@ -475,7 +408,7 @@ function Build-Module { # a PowerShell module package. NuspecBasePath resolves the globs against the build # output so the nuspec need not know how deep bin// is. $packArgs = @($csprojPath, '-c', $Configuration, '--no-build', '--nologo', '-v', 'minimal', - "-p:NuspecFile=$nuspecPath", "-p:NuspecBasePath=$binDir", "-p:Version=$ModuleVersion", + "-p:NuspecFile=$nuspecPath", "-p:NuspecBasePath=$binDir", "-p:Version=$moduleVersion", '-p:NoPackageAnalysis=true', '-o', $moduleArtifacts) $packOut = & dotnet pack @packArgs 2>&1 if ($LASTEXITCODE -ne 0) { diff --git a/tools/Test-WrapperPaging.ps1 b/tools/Test-WrapperPaging.ps1 new file mode 100644 index 00000000000..c7e6fc17e8b --- /dev/null +++ b/tools/Test-WrapperPaging.ps1 @@ -0,0 +1,156 @@ +<# +.SYNOPSIS +Deterministic pagination gate (#3706). Drives the REAL compiled Get-MgUser_List through a stub +transport that fabricates a two-page collection - no tenant data is read beyond one /me call. + +.DESCRIPTION +The runtime's session adapter cache is pre-seeded with an adapter whose HttpClient returns two +fabricated pages - the same internal cache tools/Test-WrapperLive.ps1 reflects on, but where +that gate only READS the _adapter slot to observe reuse, this one WRITES both _key and _adapter +to inject the stub. The active Connect-MgGraph session is used only as the cache KEY - the +exact reference-equality contract the cache was built on; every list request hits the stub. + +Proves, deterministically: + 1. -All follows the @odata.nextLink and streams both pages (2 requests; request 2 hits the + literal nextLink URL) + 2. without -All: first page only, exactly ONE truncation warning, no extra request + 3. -All -Top N: the total cap stops the loop at whole-page granularity + +Requires an active or cached Graph session (any scopes - only /me is actually requested). + +.EXAMPLE +pwsh -NoProfile -File .\tools\Test-WrapperPaging.ps1 +#> +param([string]$Configuration = 'Release') +$ErrorActionPreference = 'Stop' +$repoRoot = Split-Path $PSScriptRoot -Parent +$wrapperPsd1 = Join-Path $repoRoot "src\Users\wrapper\v1.0\bin\$Configuration\netstandard2.0\Microsoft.Graph.Wrapper.Users.psd1" +if (-not (Test-Path $wrapperPsd1)) { + throw "no built module manifest at $wrapperPsd1 - run tools\Build-WrapperModule.ps1 -Module Users -Configuration $Configuration first" +} + +$fail = [System.Collections.Generic.List[string]]::new() +function Assert([bool]$ok, [string]$what) { + $tag = if ($ok) { 'PASS' } else { $script:fail.Add($what); 'FAIL' } + Write-Host "$tag $what" +} + +$probe = @" +`$ErrorActionPreference = 'Stop' +Import-Module Microsoft.Graph.Authentication +Import-Module '$wrapperPsd1' +Connect-MgGraph -NoWelcome +# Authentication.Core loads lazily on the first real request; one /me GET forces it so the +# reflection below can find HttpHelpers. The only network call this gate makes. +`$null = Invoke-MgGraphRequest -Method GET -Uri 'https://graph.microsoft.com/v1.0/me' + +Add-Type -TypeDefinition @' +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +public class PagingStubHandler : HttpMessageHandler +{ + public System.Collections.Generic.List Urls = new System.Collections.Generic.List(); + protected override Task SendAsync(HttpRequestMessage request, CancellationToken token) + { + Urls.Add(request.RequestUri.ToString()); + string body = Urls.Count == 1 + ? "{\"value\":[{\"id\":\"u1\"},{\"id\":\"u2\"}],\"@odata.nextLink\":\"https://graph.microsoft.com/v1.0/users?`$skiptoken=page2\"}" + : "{\"value\":[{\"id\":\"u3\"}]}"; + var resp = new HttpResponseMessage(HttpStatusCode.OK); + resp.Content = new StringContent(body, Encoding.UTF8, "application/json"); + return Task.FromResult(resp); + } +} +'@ + +function Get-LoadedAssembly([string]`$name) { + `$found = [System.AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { `$_.GetName().Name -eq `$name } | Select-Object -First 1 + if (-not `$found) { throw "assembly not loaded: `$name" } + `$found +} +`$runtimeAsm = Get-LoadedAssembly 'Microsoft.Graph.Wrapper.Runtime' +`$kiotaHttp = Get-LoadedAssembly 'Microsoft.Kiota.Http.HttpClientLibrary' +`$kiotaAbs = Get-LoadedAssembly 'Microsoft.Kiota.Abstractions' + +# The real session client is only the cache key; requests go to the stub. Auth.Core sits in the +# default context after the /me call, so the type literal resolves directly. +`$real = [Microsoft.Graph.PowerShell.Authentication.Helpers.HttpHelpers]::GetGraphHttpClient() +`$handler = [PagingStubHandler]::new() +`$stubClient = [System.Net.Http.HttpClient]::new(`$handler) +`$anon = `$kiotaAbs.GetType('Microsoft.Kiota.Abstractions.Authentication.AnonymousAuthenticationProvider'). + GetConstructor([type[]]@()).Invoke(@()) +`$adapterType = `$kiotaHttp.GetType('Microsoft.Kiota.Http.HttpClientLibrary.HttpClientRequestAdapter') +`$adapter = `$adapterType.GetConstructors() | Where-Object { `$_.GetParameters().Count -eq 5 } | + ForEach-Object { `$_.Invoke(@(`$anon, `$null, `$null, `$stubClient, `$null)) } | Select-Object -First 1 +# Without this guard a kiota constructor change would seed a null adapter, the cache would +# rebuild a REAL one over the session client, and the gate would silently read live tenant data. +if (-not `$adapter) { throw 'no 5-parameter HttpClientRequestAdapter constructor - kiota surface changed' } + +`$cache = `$runtimeAsm.GetType('Microsoft.Graph.Wrapper.Runtime.SessionAdapterCache', `$true) +`$flags = [System.Reflection.BindingFlags]'NonPublic,Static' +`$cache.GetField('_key', `$flags).SetValue(`$null, `$real) +`$cache.GetField('_adapter', `$flags).SetValue(`$null, `$adapter) + +# Warnings are CAPTURED on every case, never suppressed: under -All the warning branch must be +# unreachable, and asserting zero converts what would be a blind redirect into coverage. +# --- 1. -All follows the nextLink --- +`$handler.Urls.Clear() +`$w1 = @() +`$all = @(Get-MgUser_List -All -WarningVariable w1 -WarningAction SilentlyContinue) +`$r1 = [pscustomobject]@{ Test = 'all'; Items = `$all.Count; Requests = `$handler.Urls.Count; Warnings = `$w1.Count; SecondUrl = if (`$handler.Urls.Count -gt 1) { `$handler.Urls[1] } else { '' } } + +# --- 2. worker without -All: one page + one warning --- +`$handler.Urls.Clear() +`$w2 = @() +`$page1 = @(Get-MgUser_List -WarningVariable w2 -WarningAction SilentlyContinue) +`$r2 = [pscustomobject]@{ Test = 'warn'; Items = `$page1.Count; Requests = `$handler.Urls.Count; Warnings = `$w2.Count; Text = if (`$w2) { "`$(`$w2[0])" } else { '' } } + +# --- 3. -All -Top 2: total cap stops after page 1 --- +`$handler.Urls.Clear() +`$w3 = @() +`$capped = @(Get-MgUser_List -All -Top 2 -WarningVariable w3 -WarningAction SilentlyContinue) +`$r3 = [pscustomobject]@{ Test = 'cap'; Items = `$capped.Count; Requests = `$handler.Urls.Count; Warnings = `$w3.Count } + +# --- 4. the PUBLIC dispatcher forwards -All (declaration alone is not proof) --- +`$handler.Urls.Clear() +`$w4 = @() +`$viaDispatcher = @(Get-MgUser -All -WarningVariable w4 -WarningAction SilentlyContinue) +`$r4 = [pscustomobject]@{ Test = 'dispatcher'; Items = `$viaDispatcher.Count; Requests = `$handler.Urls.Count; Warnings = `$w4.Count } + +# --- 5. the PUBLIC dispatcher without -All: warning must cross InvokeScript to the caller --- +`$handler.Urls.Clear() +`$w5 = @() +`$viaDispatcherPage1 = @(Get-MgUser -WarningVariable w5 -WarningAction SilentlyContinue) +`$r5 = [pscustomobject]@{ Test = 'dispwarn'; Items = `$viaDispatcherPage1.Count; Requests = `$handler.Urls.Count; Warnings = `$w5.Count } + +@(`$r1, `$r2, `$r3, `$r4, `$r5) | ConvertTo-Json -Compress +"@ +$enc = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($probe)) +$out = & pwsh -NoProfile -NonInteractive -EncodedCommand $enc 2>&1 +$json = $out | Where-Object { $_ -match '^\[' } | Select-Object -Last 1 +if (-not $json) { Assert $false "probe produced no result: $(($out | Select-Object -Last 3) -join ' | ')" } +else { + $r = $json | ConvertFrom-Json + $all = $r | Where-Object Test -eq 'all' + Assert ($all.Items -eq 3) "-All returned all 3 items across pages (got $($all.Items))" + Assert ($all.Requests -eq 2) "-All made exactly 2 requests (got $($all.Requests))" + Assert ($all.SecondUrl -like '*skiptoken=page2*') "request 2 hit the literal nextLink ($($all.SecondUrl))" + Assert ($all.Warnings -eq 0) "-All emits no warning (got $($all.Warnings))" + $warn = $r | Where-Object Test -eq 'warn' + Assert ($warn.Items -eq 2 -and $warn.Requests -eq 1) "worker without -All: first page only, 1 request (items=$($warn.Items) req=$($warn.Requests))" + Assert ($warn.Warnings -eq 1 -and $warn.Text -like '*Use -All*') "exactly one truncation warning with -All guidance (n=$($warn.Warnings))" + $cap = $r | Where-Object Test -eq 'cap' + Assert ($cap.Items -eq 2 -and $cap.Requests -eq 1 -and $cap.Warnings -eq 0) "-All -Top 2: cap stops after page 1, no warning (items=$($cap.Items) req=$($cap.Requests) warn=$($cap.Warnings))" + $disp = $r | Where-Object Test -eq 'dispatcher' + Assert ($disp.Items -eq 3 -and $disp.Requests -eq 2 -and $disp.Warnings -eq 0) "dispatcher forwards -All end to end (items=$($disp.Items) req=$($disp.Requests) warn=$($disp.Warnings))" + $dw = $r | Where-Object Test -eq 'dispwarn' + Assert ($dw.Items -eq 2 -and $dw.Requests -eq 1) "dispatcher without -All: first page only, 1 request (items=$($dw.Items) req=$($dw.Requests))" + Assert ($dw.Warnings -eq 1) "the truncation warning crosses InvokeScript to the dispatcher caller (n=$($dw.Warnings))" +} + +'' +if ($fail.Count -eq 0) { 'RESULT: ALL PASS' } else { "RESULT: $($fail.Count) FAILURE(S)" } +exit $fail.Count diff --git a/tools/WrapperGenerator.Tests/EmitterTests.cs b/tools/WrapperGenerator.Tests/EmitterTests.cs index fa906fe73d0..3779dc82243 100644 --- a/tools/WrapperGenerator.Tests/EmitterTests.cs +++ b/tools/WrapperGenerator.Tests/EmitterTests.cs @@ -22,10 +22,116 @@ public void DispatcherRethrowsTheWorkersOriginalErrorRecord() new EmitContext("Test.Client"), "Message", "MessageCollectionResponse", new HashSet(), new HashSet()); - Assert.Contains("catch (RuntimeException rex) when (rex.ErrorRecord is not null)", source); + Assert.Contains("catch (RuntimeException rex) when (rex is not PipelineStoppedException && rex.ErrorRecord is not null)", source); Assert.Contains("ThrowTerminatingError(rex.ErrorRecord);", source); } + // Pagination pins (#3706). EmitListGet is the single template behind every list-shaped + // cmdlet, so these pins govern all of them. The contract, with its evidence, lives in + // tools/WrapperGenerator/docs/pagination.md: -All follows every non-empty @odata.nextLink; + // -Top is a TOTAL cap under -All at whole-page granularity; one short truncation warning + // fires when a nextLink survives without -All, costing no extra request; a pipeline stop + // is never re-branded as a Graph failure. + [Fact] + public void ListEmitsAllSwitchNextLinkLoopAndTruncationWarning() + { + var naming = Naming.WithSuffix(Naming.Resolve(new OperationInfo(HttpMethod.Get, "/users")), "_List"); + + var source = CmdletEmitter.EmitListGet(naming, new EmitContext("Test.Client"), "User", + "UserCollectionResponse", new HashSet { "$top", "$filter" }); + + // -All declared on the worker; the loop runs only under it and Ctrl+C ends it. + Assert.Contains("public SwitchParameter All { get; set; }", source); + Assert.Contains("if (All.IsPresent)", source); + Assert.Contains(".WithUrl(nextLink).GetAsync(", source); + Assert.Contains("!string.IsNullOrEmpty(nextLink) && !Stopping", source); + + // -Top caps the total; unbound -Top leaves the loop uncapped. + Assert.Contains("!this.IsParameterBound(nameof(Top)) || fetched < Top", source); + + // Warning: only on a surviving non-empty nextLink, no extra call, short text. + Assert.Contains("WriteWarning(\"More results are available. Use -All to return all pages.\");", source); + Assert.Contains("else if (!string.IsNullOrEmpty(result?.OdataNextLink))", source); + + // Pipeline stop passes through the shared catch untouched. + Assert.Contains("catch (Exception ex) when (ex is not PipelineStoppedException && ex is not OperationCanceledException)", source); + + // The FIRST request keeps the direct builder call the operation-inventory regex keys on. + Assert.Contains(".GetAsync(requestConfiguration =>", source); + } + + // A list operation whose spec declares no $top has no -Top parameter, so the loop must be + // uncapped and emit no counter - a fetched variable without a cap would be dead code. + [Fact] + public void ListWithoutTopEmitsUncappedLoopAndNoCounter() + { + var naming = Naming.WithSuffix(Naming.Resolve(new OperationInfo(HttpMethod.Get, "/users")), "_List"); + + var source = CmdletEmitter.EmitListGet(naming, new EmitContext("Test.Client"), "User", + "UserCollectionResponse", new HashSet()); + + Assert.Contains("if (All.IsPresent)", source); + Assert.DoesNotContain("public int Top", source); + Assert.DoesNotContain("fetched", source); + } + + // The continuation request re-applies headers (ConsistencyLevel on page 2+ of a $search + // query, caller -Headers) but never query bindings: the nextLink already carries the + // original query state, and a raw-URL builder ignores templated query parameters, so + // re-binding them would be dead code. + [Fact] + public void ContinuationReappliesHeadersButNeverQueryBindings() + { + var naming = new CmdletNaming( + VerbsClass: "VerbsCommon", + VerbName: "Get", + Noun: "MgUser_List", + ClassName: "GetMgUser_ListCommand", + PathParamNames: [], + BuilderExpression: "Users", + HeaderParams: new[] { new HeaderParam("ConsistencyLevel", "ConsistencyLevel") }); + + var source = CmdletEmitter.EmitListGet(naming, new EmitContext("Test.Client"), "User", + "UserCollectionResponse", new HashSet { "$filter", "$search" }); + + var continuation = source[source.IndexOf(".WithUrl(")..]; + Assert.Contains("requestConfiguration.Headers.Add(\"ConsistencyLevel\", ConsistencyLevel!);", continuation); + Assert.Contains("AddRequestHeaders(requestConfiguration.Headers);", continuation); + Assert.DoesNotContain("QueryParameters", continuation); + } + + // The public dispatcher must declare -All on its List set (the binder rejects an undeclared + // parameter before ProcessRecord, so an -All the dispatcher lacks never reaches forwarding), + // and both of its catch layers must pass a pipeline stop through rather than re-branding it. + [Fact] + public void DispatcherDeclaresAllOnListSetAndPassesPipelineStopThrough() + { + var list = Naming.Resolve(new OperationInfo(HttpMethod.Get, "/users")); + var item = Naming.Resolve(new OperationInfo(HttpMethod.Get, "/users/{user-id}")); + + var source = CmdletEmitter.EmitGetDispatcher( + list, item, Naming.WithSuffix(list, "_List"), Naming.WithSuffix(item, "_Get"), + new EmitContext("Test.Client"), "User", "UserCollectionResponse", + new HashSet { "$top" }, new HashSet()); + + Assert.Contains("public SwitchParameter All { get; set; }", source); + Assert.Contains("catch (RuntimeException rex) when (rex is not PipelineStoppedException && rex.ErrorRecord is not null)", source); + Assert.Contains("catch (Exception ex) when (ex is not PipelineStoppedException && ex is not OperationCanceledException)", source); + } + + // Control: non-list shapes must not grow paging machinery. + [Fact] + public void NonListShapesEmitNoPagingMachinery() + { + var naming = Naming.Resolve(new OperationInfo(HttpMethod.Delete, "/users/{user-id}")); + + var source = CmdletEmitter.EmitRemove(naming, new EmitContext("Test.Client")); + + Assert.DoesNotContain("WithUrl", source); + Assert.DoesNotContain("SwitchParameter All", source); + Assert.DoesNotContain("WriteWarning", source); + } + // A collision-renamed property must emit the suffixed PARAMETER but assign the model's // real property: -DeviceId1 binds, body.DeviceId receives (Update-MgDevice pattern). [Fact] diff --git a/tools/WrapperGenerator/CmdletEmitter.cs b/tools/WrapperGenerator/CmdletEmitter.cs index 5a19b001b95..c9a697ef6ba 100644 --- a/tools/WrapperGenerator/CmdletEmitter.cs +++ b/tools/WrapperGenerator/CmdletEmitter.cs @@ -82,9 +82,13 @@ private static (IReadOnlyList Shared, IReadOnlyList Li // The shared try/catch tail around every Graph call. Only the ErrorRecord's target object // varies, and sometimes the nesting depth (EmitUpdate's re-fetch sits one block deeper). - // The error surface itself (id, category) lives on GraphClientCmdlet. + // The error surface itself (id, category) lives on GraphClientCmdlet. A pipeline stop + // (downstream Select-Object -First, Ctrl+C) passes through untouched: it is the engine's + // stop signal, not a Graph failure. The filter is load-bearing only where WriteObject runs + // inside the try - list workers and dispatchers - and is emitted uniformly so every catch + // tail in the corpus stays identical. private static string CatchBlock(string targetIdExpr, string extraIndent = "") => $$""" - {{extraIndent}}catch (Exception ex) + {{extraIndent}}catch (Exception ex) when (ex is not PipelineStoppedException && ex is not OperationCanceledException) {{extraIndent}}{ {{extraIndent}}ThrowGraphRequestFailed(ex, {{targetIdExpr}}); {{extraIndent}}return; @@ -674,6 +678,18 @@ public static string EmitListGet(CmdletNaming naming, EmitContext ctx, string en var paramDecls = string.Join("\n\n", applicable.Select(o => o.ParamDecl(null))); var bindings = string.Join("\n\n", applicable.Select(o => o.Binding)); + // -Top is a TOTAL cap under -All, at whole-page granularity - the published ListCmdlet's + // semantics: limit = Top, iterate while fetched < limit, and whole final pages ship + // because the overflow trimmer has zero call sites in current generated output (its + // injection directive anchors on a callback name autorest no longer emits; evidence in + // docs/pagination.md). The counter exists only when the operation declares $top; without + // it the loop is uncapped and only nextLink exhaustion or a pipeline stop ends it. + var hasTop = queryParamNames.Contains("$top"); + var fetchedInit = hasTop ? "\n var fetched = result?.Value?.Count ?? 0;" : ""; + var capCondition = hasTop ? " && (!this.IsParameterBound(nameof(Top)) || fetched < Top)" : ""; + var fetchedAdd = hasTop ? "\n fetched += page.Count;" : ""; + var continuationHeaders = HeaderBindingsFor(naming.HeaderParams, extraIndent: " ") + GenericHeadersBinding(" "); + return $$""" #nullable enable @@ -699,6 +715,12 @@ public class {{naming.ClassName}} : GraphClientCmdlet {{AccessTokenParamDecl()}} {{paramDecls}} + + // Follows every @odata.nextLink until the collection is exhausted (a bound -Top caps + // the total). Without it only the first page returns, plus a truncation warning when + // more pages existed. + [Parameter(Mandatory = false)] + public SwitchParameter All { get; set; } {{HeaderParamDecls(naming)}} {{GenericHeadersParamDecl()}} @@ -715,13 +737,39 @@ protected override void ProcessRecord() {{HeaderBindings(naming)}} {{GenericHeadersBinding()}} }).GetAwaiter().GetResult(); + + // A collection response and its Value are both nullable on the kiota client; an + // empty page writes nothing rather than dereferencing null. Each page streams to + // the pipeline before the next request is issued, matching the published SDK. + if (result?.Value is { } items) + WriteObject(items, enumerateCollection: true); + + if (All.IsPresent) + {{{fetchedInit}} + var nextLink = result?.OdataNextLink; + while (!string.IsNullOrEmpty(nextLink) && !Stopping{{capCondition}}) + { + // The nextLink already carries the original query state, and a raw-URL + // builder ignores templated query parameters anyway - so the continuation + // re-applies headers only; query bindings here would be dead code. + result = client.{{naming.BuilderExpression}}.WithUrl(nextLink).GetAsync(requestConfiguration => + {{{continuationHeaders}} + }, StoppingToken).GetAwaiter().GetResult(); + if (result?.Value is { } page) + { + WriteObject(page, enumerateCollection: true);{{fetchedAdd}} + } + nextLink = result?.OdataNextLink; + } + } + else if (!string.IsNullOrEmpty(result?.OdataNextLink)) + { + // Deliberately stronger than the published SDK, which truncates silently; + // approved in the design spec. One line, no extra request. + WriteWarning("More results are available. Use -All to return all pages."); + } } {{CatchBlock(TargetId(naming))}} - - // A collection response and its Value are both nullable on the kiota client; an - // empty page writes nothing rather than dereferencing null. - if (result?.Value is { } items) - WriteObject(items, enumerateCollection: true); } } } @@ -829,6 +877,12 @@ public class {{listNaming.ClassName}} : GraphClientCmdlet {{selectExpandDecls}} {{listOnlyParamDecls}} + + // Declared here because the binder rejects a parameter the dispatcher does not accept + // before ProcessRecord ever runs; once declared, the wholesale BoundParameters splat + // forwards it to the list worker with no further plumbing. + [Parameter(Mandatory = false, ParameterSetName = "List")] + public SwitchParameter All { get; set; } {{HeaderParamDeclsFor(sharedHeaders, parameterSetName: null)}} {{HeaderParamDeclsFor(listOnlyHeaders, parameterSetName: "List")}} {{HeaderParamDeclsFor(getOnlyHeaders, parameterSetName: "Get")}} @@ -852,8 +906,9 @@ protected override void ProcessRecord() // as a RuntimeException carrying the worker's ErrorRecord. Rethrow that record // unchanged so the caller sees the worker's error identity (NoGraphSession, // GraphRequestFailed, ...) instead of every failure collapsing into a generic - // dispatcher error. - catch (RuntimeException rex) when (rex.ErrorRecord is not null) + // dispatcher error. A pipeline stop is a RuntimeException too and must NOT be + // rethrown as a terminating error - both filters here let it pass to the engine. + catch (RuntimeException rex) when (rex is not PipelineStoppedException && rex.ErrorRecord is not null) { ThrowTerminatingError(rex.ErrorRecord); return; diff --git a/tools/WrapperGenerator/README.md b/tools/WrapperGenerator/README.md index aa93b7a8604..8948e1ea8ee 100644 --- a/tools/WrapperGenerator/README.md +++ b/tools/WrapperGenerator/README.md @@ -127,94 +127,7 @@ Filtered OpenAPI (Graph) └─► [2] WrapperGenerator ─► the cmdlet wrappers (this tool) ``` -The wrappers compile and run only alongside step 1's output. `tools/Build-WrapperModule.ps1` runs both steps and wires the result into one buildable module; the next section is that path end to end. - -## Build a module end to end - -`tools/Build-WrapperModule.ps1` runs both steps above and everything after them, so one command -takes a module from an OpenAPI document to something `Import-Module` accepts. - -**Prerequisites** — the .NET SDK, PowerShell 7+, and the kiota CLI on `PATH`: - -```powershell -dotnet tool install --global Microsoft.OpenApi.Kiota -``` - -**Build one module:** - -```powershell -.\tools\Build-WrapperModule.ps1 -Module Mail -``` - -Everything lands in `src/Mail/wrapper/v1.0/`. Per module the script runs: - -| Step | Produces | -|---|---| -| 1. `kiota generate` | `Client/` — the `ApiClient` and its models | -| 2. WrapperGenerator | `Cmdlets/` — one `*.g.cs` per cmdlet, plus `Shared.g.cs` | -| 3–4. project files from `tools/Templates/` | `Client/Client.csproj` and `Microsoft.Graph.Wrapper.Mail.csproj` | -| 5. `dotnet build` | `bin/{Configuration}/netstandard2.0/Microsoft.Graph.Wrapper.Mail.dll` | -| 6. `New-ModuleManifest` | `Microsoft.Graph.Wrapper.Mail.psd1`, next to the dll | -| 7. `dotnet pack` (only with `-Pack`) | `artifacts/Mail/Microsoft.Graph.Wrapper.Mail.{version}.nupkg` | - -Steps 1 and 2 read the **same** OpenAPI document, so the wrappers always match the client they -compile against — that is the reason one script owns both rather than two run in sequence. - -**Import what you just built:** - -```powershell -Import-Module .\src\Mail\wrapper\v1.0\bin\Debug\netstandard2.0\Microsoft.Graph.Wrapper.Mail.psd1 -``` - -The module is named `Microsoft.Graph.Wrapper.{Module}`, so it imports side by side with an -installed `Microsoft.Graph.{Module}` without colliding. - -**The switches you will actually reach for:** - -| To | Use | -|---|---| -| build every module configured for the API version | omit `-Module` | -| re-run only the wrappers, reusing the client on disk | `-SkipKiota` | -| build what the gates build | `-Configuration Release` | -| read a different spec root | `-SpecRoot` (default `openApiDocs_KiotaCompat`) | - -`Get-Help .\tools\Build-WrapperModule.ps1 -Full` documents each parameter and why its default is -what it is. - -**Package it** — `-Pack` writes one nupkg per module under `artifacts/{Module}/`: - -```powershell -.\tools\Build-WrapperModule.ps1 -ApiVersion v1.0 -Pack -``` - -Packages are `3.0.0` (`-ModuleVersion`) and always carry a prerelease label (`-Prerelease`, -defaulting to `alpha{build id}` in CI and `alpha{UTC timestamp}` locally). Neither default is -cosmetic. The wrappers are the v3 line, not a build of the v2 service-module release train whose -version lives in `config/ModuleMetadata.json`, so a stable-versioned wrapper package would -collide number-for-number with a real SDK release; and two packages sharing an id and version but -not their contents are indistinguishable to a feed and to anyone who already installed one. - -Three properties are what make a package installable rather than merely produced, and each is -there because its absence was observed: the nuspec declares `Microsoft.Graph.Authentication` as a -dependency, because `Install-Module` resolves from NuGet metadata and not from the manifest, so -without it a clean machine gets a wrapper that cannot import; the prerelease label is mandatory -rather than optional; and the manifest GUID is derived from the module name — a name-based RFC -4122 UUID, identical on every build — so `Update-Module` keeps its identity across handouts -instead of seeing a new module each time. - -**Check it** — the smoke test reads the **package**, not `bin/`, in a fresh pwsh process per -module. Importing build output proves the compiler ran; it does not prove the artifact a user -installs carries the assembly, its dependencies and a manifest that agrees with them. So pack -first: - -```powershell -.\tools\Build-WrapperModule.ps1 -Module Mail -Pack -.\tools\Test-WrapperModule.ps1 -Module Mail -``` - -It refuses a package packed before any of its compile inputs under `src` — a green run cannot be -a stale one — and reports `n/a` rather than a pass for a shape the module never binds. The full -gate set, in order and with a population reported per gate, is `.\tools\Invoke-WrapperGates.ps1`. +The wrappers compile and run only alongside step 1's output. Wiring the two into one buildable module is later work (see Gaps). ## The source files @@ -231,10 +144,7 @@ gate set, in order and with a population reported per gate, is `.\tools\Invoke-W | `OperationInfo.cs`, `EmitContext.cs`, `GeneratorConfig.cs` | Small data/config carriers | | `GeneratorExtensions.cs` | String + schema helper methods | -## Run the generator on its own - -Step 2 by itself — for changing the generator, or emitting a handful of operations without -building a whole module. +## Build, run, test **Build:** @@ -297,7 +207,8 @@ diff shows exactly what the change did to the output: .\tools\New-WrapperOutputManifest.ps1 # refresh docs/WrapperCmdlets-V1.0*.csv ``` -The switches are the same ones described under [Build a module end to end](#build-a-module-end-to-end). +Omit `-Module` to build every module configured for the API version; add `-Pack` to produce a +package per module under `artifacts/{Module}/`. `docs/WrapperCmdlets-V1.0.csv` is the reviewable inventory of that output — one row per emitted cmdlet with its module, verb, noun and request path — with per-module totals in @@ -322,12 +233,12 @@ dotnet test tools/WrapperGenerator.Tests .\tools\Test-BodyBindingCoverage.ps1 # 5. Runtime gate: the module imports and each bound shape accepts what a person would type -.\tools\Test-WrapperModule.ps1 -Module # needs -Pack output; see above +.\tools\Test-WrapperModule.ps1 -Module -Configuration Release ``` The unit tests guard the naming and classification rules (their expected values are real published names from `src/Authentication/Authentication/custom/common/MgCommandMetadata.json`). The parity gate checks actual generated output against that same inventory; names on the deliberate-corrections list ([docs/edge-cases/naming-edge-cases.md](docs/edge-cases/naming-edge-cases.md)) are reported as `[CORRECTED]` instead of failing. -The generated cmdlets **are** compiled: `Build-WrapperModule.ps1` builds each module against the kiota client it was generated with, the only authority on whether an emitted parameter's CLR type matches the member it assigns. Compilation cannot see an *omitted* member, so the omission oracle exists separately; neither can see whether PowerShell converts a value at runtime, so the runtime gate exists separately again. Naming parity is enforced independently by `Compare-WrapperCmdletNames.ps1`. The runtime gate reads the packed artifact rather than `bin/`, and refuses one packed before any of its compile inputs under `src`, so it cannot validate a stale build and report green. +The generated cmdlets **are** compiled: `Build-WrapperModule.ps1` builds each module against the kiota client it was generated with, the only authority on whether an emitted parameter's CLR type matches the member it assigns. Compilation cannot see an *omitted* member, so the omission oracle exists separately; neither can see whether PowerShell converts a value at runtime, so the runtime gate exists separately again. Naming parity is enforced independently by `Compare-WrapperCmdletNames.ps1`. The runtime gate refuses a binary older than any of its compiled inputs — `Build-` and `Test-` both default to `Debug`, so a Release-only build once left it validating a three-day-old assembly and reporting green. ## Gaps / not done yet diff --git a/tools/WrapperGenerator/docs/pagination.md b/tools/WrapperGenerator/docs/pagination.md new file mode 100644 index 00000000000..68810b1f5d7 --- /dev/null +++ b/tools/WrapperGenerator/docs/pagination.md @@ -0,0 +1,73 @@ +# Pagination (#3706): decisions and evidence + +Every list-shaped cmdlet follows `@odata.nextLink` under an opt-in `-All` switch; without it, +a surviving nextLink produces one short warning and no extra request. This file records the +decisions behind that behavior and the evidence each one rests on, so the emitter and test +comments can stay lean and cite here. + +## The contract + +| Invocation | Behavior | +|---|---| +| (no switches) | First page only. If the response carries a non-empty `@odata.nextLink`, one warning: `More results are available. Use -All to return all pages.` | +| `-All` | Follows every non-empty nextLink until exhaustion. Each page streams to the pipeline before the next request is issued. | +| `-All -Top N` | `-Top` is a TOTAL cap at whole-page granularity: iteration stops once fetched >= N; the final page is written whole, so more than N items can return. | +| `-Top N` alone | Single request with `$top=N` (unchanged from pre-pagination behavior). Known divergence: the published SDK auto-paginates when N exceeds the service page cap (999); the wrapper passes the raw `$top` through and the service rejects it. Pre-existing behavior, recorded here as deliberate scope. | + +Continuation requests go through kiota's `WithUrl(nextLink)` and re-apply **headers only** +(`ConsistencyLevel`, `-Headers`): the nextLink already carries the original query state, and a +raw-URL builder ignores templated query parameters (see the `WithUrl` doc comment in any +generated request builder), so re-binding query options would be dead code. + +A pipeline stop (`Select-Object -First N` downstream, Ctrl+C) passes through both catch layers +(worker and dispatcher) via `when (ex is not PipelineStoppedException)` filters instead of +being re-branded a Graph failure. `-All` also checks `Stopping` between pages. In-flight HTTP +cancellation on Ctrl+C is a known limitation: the loop stops between pages, not mid-request. + +## Decisions and their evidence + +**The truncation warning is a deliberate deviation from the published SDK.** The published +AutoRest `ListCmdlet` truncates silently: `src/*/v1.0/custom/ListCmdlet.cs` contains no +`WriteWarning`, and the only warning site in a generated list cmdlet is the generic event pump. +The warning was approved in the design spec ("warning (no extra call) when nextLink present +without -All"; spec section 7 building block 4, and section 9 resolved questions: "Pagination +warning - Approved"). + +**Whole-page `-Top` granularity is the published SDK's shipped behavior.** Its `ListCmdlet` +sets `limit = Top` under `-All` and iterates while `totalFetchedItems < limit` +(`ListCmdlet.cs`, `InitializeCmdlet`/`ShouldIteratePages`). The final-page trimmer +(`GetOverflowItemsNextLinkUri`) exists but has **zero call sites in current generated +output**: its injection directive (`src/readme.graph.md`, `odataNextLinkCallRegex`) anchors on +a callback named `onOk`, and current autorest emits `on2Xx` - verified by grep over the +generated cmdlet trees (definition present in `custom/ListCmdlet.cs`, zero call sites in +`generated/cmdlets/`). Whether older gallery builds (from the `onOk` era) trimmed is not +established and does not bear on parity with what generates today. + +**`-PageSize` and `-CountVariable` are out of scope.** #3706 promises `-All` plus the warning. +The published `-PageSize`/`-CountVariable` surface (and its private `-Count` demotion) is a +separate parity decision - note that a naive `-CountVariable` port would silently fail: the +published implementation reads `@odata.count` from `AdditionalData`, while kiota deserializes +it into the typed `OdataCount` property. + +**Delta operations are out of scope for this loop because of their SHAPE, not a guard.** +They generate - 72 cmdlets - but the spec classes them as functions, so they route to the +function emitter and never reach `EmitListGet`. They therefore follow no nextLink, and they +write the response envelope rather than enumerating items - which is why they already expose +`@odata.deltaLink`, unlike the published SDK's delta cmdlets. Tracked in #3742. + +**Scope is v1.0.** The generated corpus contains only v1.0 modules. + +## Verification + +- Emitter pins: `WrapperGenerator.Tests/EmitterTests.cs` (loop, cap, warning, headers-only + continuation, dispatcher declaration, catch filters, no-`$top` uncapped emission, non-list + shapes paging-free). +- Deterministic transport proof: `tools/Test-WrapperPaging.ps1` drives the real compiled + worker AND public dispatcher across a fabricated two-page collection - request counts, + literal nextLink continuation, warning exactly-once (including across `InvokeScript`), + zero warnings under `-All`, cap behavior. +- Live: the truncation warning fires exactly once against real Graph (`-Top 5` on `/users`); + `tools/Test-WrapperLive.ps1` covers the session pipeline end to end. +- Behavioral gates: operation inventory, name parity and the body-binding oracle are + parameter-blind and unchanged by this feature; the inventory additionally pins that the + first emitted request keeps the direct `client.X.GetAsync(` form its regex keys on.