diff --git a/QueryVault/Jobs/CreateArchiveJob_AllDatabases.sql b/QueryVault/Jobs/CreateArchiveJob_AllDatabases.sql index d1c61e6..86c9ac0 100644 --- a/QueryVault/Jobs/CreateArchiveJob_AllDatabases.sql +++ b/QueryVault/Jobs/CreateArchiveJob_AllDatabases.sql @@ -65,10 +65,24 @@ BEGIN BEGIN TRY PRINT ''Processing database: '' + @DatabaseName; - -- Set date range for archiving + -- Resume from the last successfully archived endpoint. This closes any + -- gap left by a failed scheduled run. Use the configured lookback only + -- when this source has no completed archive yet. SET @EndDateTime = SYSUTCDATETIME(); - SET @StartDateTime = DATEADD(DAY, -@DaysToArchive, @EndDateTime); - SET @RunName = ''Scheduled Archive - '' + CONVERT(VARCHAR(30), @EndDateTime, 120); + SELECT @StartDateTime = MAX(EndDateTime) + FROM dbo.RunMetadata + WHERE SourceDatabaseName = @DatabaseName + AND SourceServerName = @@SERVERNAME + AND RunStatus = N''Completed''; + + SET @StartDateTime = ISNULL( + @StartDateTime, + DATEADD(DAY, -@DaysToArchive, @EndDateTime) + ); + SET @RunName = ''Scheduled Archive - '' + + CONVERT(VARCHAR(30), @StartDateTime, 126) + + '' through '' + + CONVERT(VARCHAR(30), @EndDateTime, 126); -- Execute archive procedure EXEC dbo.usp_ArchiveQueryStore diff --git a/QueryVault/Jobs/CreateArchiveJob_Template.sql b/QueryVault/Jobs/CreateArchiveJob_Template.sql index 24d139a..5ee9453 100644 --- a/QueryVault/Jobs/CreateArchiveJob_Template.sql +++ b/QueryVault/Jobs/CreateArchiveJob_Template.sql @@ -65,10 +65,24 @@ BEGIN RETURN; END --- Set date range for archiving +-- Resume from the last successfully archived endpoint. This closes any gap +-- left by a failed scheduled run. Use the configured lookback only when this +-- source has no completed archive yet. SET @EndDateTime = SYSUTCDATETIME(); -SET @StartDateTime = DATEADD(DAY, -@DaysToArchive, @EndDateTime); -SET @RunName = ''Scheduled Archive - '' + CONVERT(VARCHAR(30), @EndDateTime, 120); +SELECT @StartDateTime = MAX(EndDateTime) +FROM dbo.RunMetadata +WHERE SourceDatabaseName = ''@DatabaseName'' + AND SourceServerName = @@SERVERNAME + AND RunStatus = N''Completed''; + +SET @StartDateTime = ISNULL( + @StartDateTime, + DATEADD(DAY, -@DaysToArchive, @EndDateTime) +); +SET @RunName = ''Scheduled Archive - '' + + CONVERT(VARCHAR(30), @StartDateTime, 126) + + '' through '' + + CONVERT(VARCHAR(30), @EndDateTime, 126); -- Execute archive procedure EXEC dbo.usp_ArchiveQueryStore diff --git a/QueryVault/README.md b/QueryVault/README.md index 0414c9a..1dea857 100644 --- a/QueryVault/README.md +++ b/QueryVault/README.md @@ -125,6 +125,21 @@ EXEC QueryVaultDB.dbo.usp_ArchiveQueryStore @RetentionDays = 90; ``` +### Archive the Last Safely Completed Hour + +```sql +EXEC QueryVaultDB.dbo.usp_ArchiveQueryStore + @SourceDatabaseName = 'MyDatabase', + @RunName = 'Last completed hour', + @LookbackMinutes = 60, + @DoNotDelete = 0; +``` + +Use `@LookbackMinutes = 1440` for one day. For an exact historical period, +continue to pass `@StartDateTime` and `@EndDateTime`. The SQL Agent jobs use +the latest completed endpoint as their next start time and therefore catch up +automatically after a failed scheduled run. + ### Query Archived Data ```sql diff --git a/QueryVault/Scripts/Voyager2_CreateDisabledArchiveJob.sql b/QueryVault/Scripts/Voyager2_CreateDisabledArchiveJob.sql index 0b7d72b..a7d5e0c 100644 --- a/QueryVault/Scripts/Voyager2_CreateDisabledArchiveJob.sql +++ b/QueryVault/Scripts/Voyager2_CreateDisabledArchiveJob.sql @@ -66,6 +66,10 @@ GRANT INSERT, ALTER ON dbo.query_store_plan TO [queryvault_executor]; GRANT INSERT, ALTER ON dbo.query_store_runtime_stats TO [queryvault_executor]; GRANT INSERT, ALTER ON dbo.query_store_runtime_stats_interval TO [queryvault_executor]; GRANT INSERT, ALTER ON dbo.query_store_wait_stats TO [queryvault_executor]; +GRANT INSERT, ALTER ON dbo.query_store_runtime_stats_contributor TO [queryvault_executor]; +GRANT INSERT, ALTER ON dbo.query_store_runtime_stats_canonical TO [queryvault_executor]; +GRANT INSERT, ALTER ON dbo.query_store_wait_stats_contributor TO [queryvault_executor]; +GRANT INSERT, ALTER ON dbo.query_store_wait_stats_canonical TO [queryvault_executor]; GRANT ALTER ON dbo.query_store_query_PartitionMaintenance TO [queryvault_executor]; GRANT ALTER ON dbo.query_store_query_text_PartitionMaintenance TO [queryvault_executor]; @@ -73,6 +77,10 @@ GRANT ALTER ON dbo.query_store_plan_PartitionMaintenance TO [queryvault_executor GRANT ALTER ON dbo.query_store_runtime_stats_PartitionMaintenance TO [queryvault_executor]; GRANT ALTER ON dbo.query_store_runtime_stats_interval_PartitionMaintenance TO [queryvault_executor]; GRANT ALTER ON dbo.query_store_wait_stats_PartitionMaintenance TO [queryvault_executor]; +GRANT ALTER ON dbo.query_store_runtime_stats_contributor_PartitionMaintenance TO [queryvault_executor]; +GRANT ALTER ON dbo.query_store_runtime_stats_canonical_PartitionMaintenance TO [queryvault_executor]; +GRANT ALTER ON dbo.query_store_wait_stats_contributor_PartitionMaintenance TO [queryvault_executor]; +GRANT ALTER ON dbo.query_store_wait_stats_canonical_PartitionMaintenance TO [queryvault_executor]; -- Required only when usp_ArchiveQueryStore extends PF_RunID/PS_RunID. GRANT ALTER ANY DATASPACE TO [queryvault_executor]; @@ -148,26 +156,46 @@ SET NOCOUNT ON; DECLARE @DatabaseName NVARCHAR(128); DECLARE @RunName NVARCHAR(255); +DECLARE @StartDateTime DATETIME2(7); +DECLARE @EndDateTime DATETIME2(7); +DECLARE @DefaultDaysToArchive INT; DECLARE @FailureCount INT = 0; DECLARE database_cursor CURSOR LOCAL FAST_FORWARD FOR -SELECT DatabaseName +SELECT DatabaseName, DefaultDaysToArchive FROM dbo.DatabaseConfig WHERE IsEnabled = 1 AND ServerName = @@SERVERNAME ORDER BY DatabaseName; OPEN database_cursor; -FETCH NEXT FROM database_cursor INTO @DatabaseName; +FETCH NEXT FROM database_cursor INTO @DatabaseName, @DefaultDaysToArchive; WHILE @@FETCH_STATUS = 0 BEGIN BEGIN TRY - SET @RunName = N''SQL Agent daily archive - '' + CONVERT(NVARCHAR(30), SYSUTCDATETIME(), 126); + SET @EndDateTime = SYSUTCDATETIME(); + + SELECT @StartDateTime = MAX(EndDateTime) + FROM dbo.RunMetadata + WHERE SourceDatabaseName = @DatabaseName + AND SourceServerName = @@SERVERNAME + AND RunStatus = N''Completed''; + + SET @StartDateTime = ISNULL( + @StartDateTime, + DATEADD(DAY, -@DefaultDaysToArchive, @EndDateTime) + ); + SET @RunName = N''SQL Agent archive - '' + + CONVERT(NVARCHAR(30), @StartDateTime, 126) + + N'' through '' + + CONVERT(NVARCHAR(30), @EndDateTime, 126); EXEC dbo.usp_ArchiveQueryStore @SourceDatabaseName = @DatabaseName, @RunName = @RunName, + @StartDateTime = @StartDateTime, + @EndDateTime = @EndDateTime, @DoNotDelete = 0; END TRY BEGIN CATCH @@ -175,7 +203,7 @@ BEGIN PRINT N''QueryVault archive failed for '' + QUOTENAME(@DatabaseName) + N'': '' + ERROR_MESSAGE(); END CATCH; - FETCH NEXT FROM database_cursor INTO @DatabaseName; + FETCH NEXT FROM database_cursor INTO @DatabaseName, @DefaultDaysToArchive; END; CLOSE database_cursor; diff --git a/QueryVault/StoredProcedures/DatabaseProject/usp_ArchiveQueryStore.sql b/QueryVault/StoredProcedures/DatabaseProject/usp_ArchiveQueryStore.sql index f09bbbf..09a7f1b 100644 --- a/QueryVault/StoredProcedures/DatabaseProject/usp_ArchiveQueryStore.sql +++ b/QueryVault/StoredProcedures/DatabaseProject/usp_ArchiveQueryStore.sql @@ -5,7 +5,8 @@ CREATE PROCEDURE dbo.usp_ArchiveQueryStore @EndDateTime DATETIME2(7) = NULL, @DoNotDelete BIT = 0, @RetentionDays INT = NULL, - @BatchSize INT = NULL + @BatchSize INT = NULL, + @LookbackMinutes INT = NULL AS BEGIN SET NOCOUNT ON; @@ -28,6 +29,12 @@ BEGIN DECLARE @WaitReplicaGroupProjection NVARCHAR(128); BEGIN TRY + IF @LookbackMinutes IS NOT NULL AND @LookbackMinutes <= 0 + THROW 51004, '@LookbackMinutes must be greater than zero.', 1; + + IF @LookbackMinutes IS NOT NULL AND @StartDateTime IS NOT NULL + THROW 51005, 'Specify either @LookbackMinutes or @StartDateTime, not both.', 1; + -- Get configuration for the database DECLARE @ConfigID INT; DECLARE @DefaultDaysToArchive INT; @@ -66,10 +73,14 @@ BEGIN WHEN @RequestedEndDateTime < @SafeQueryStoreCutoff THEN @RequestedEndDateTime ELSE @SafeQueryStoreCutoff END; - SET @ActualStartDateTime = ISNULL( - @StartDateTime, - DATEADD(DAY, -@DefaultDaysToArchive, @ActualEndDateTime) - ); + SET @ActualStartDateTime = CASE + WHEN @LookbackMinutes IS NOT NULL + THEN DATEADD(MINUTE, -@LookbackMinutes, @ActualEndDateTime) + ELSE ISNULL( + @StartDateTime, + DATEADD(DAY, -@DefaultDaysToArchive, @ActualEndDateTime) + ) + END; IF @ActualEndDateTime < @ActualStartDateTime THROW 51003, 'The requested archive range does not contain a safely flushed Query Store interval.', 1; diff --git a/QueryVault/StoredProcedures/usp_ArchiveQueryStore.sql b/QueryVault/StoredProcedures/usp_ArchiveQueryStore.sql index 3fc948c..6a851b9 100644 --- a/QueryVault/StoredProcedures/usp_ArchiveQueryStore.sql +++ b/QueryVault/StoredProcedures/usp_ArchiveQueryStore.sql @@ -14,7 +14,8 @@ CREATE OR ALTER PROCEDURE dbo.usp_ArchiveQueryStore @EndDateTime DATETIME2(7) = NULL, @DoNotDelete BIT = 0, @RetentionDays INT = NULL, - @BatchSize INT = NULL + @BatchSize INT = NULL, + @LookbackMinutes INT = NULL AS BEGIN SET NOCOUNT ON; @@ -37,6 +38,12 @@ BEGIN DECLARE @WaitReplicaGroupProjection NVARCHAR(128); BEGIN TRY + IF @LookbackMinutes IS NOT NULL AND @LookbackMinutes <= 0 + THROW 51004, '@LookbackMinutes must be greater than zero.', 1; + + IF @LookbackMinutes IS NOT NULL AND @StartDateTime IS NOT NULL + THROW 51005, 'Specify either @LookbackMinutes or @StartDateTime, not both.', 1; + -- Get configuration for the database DECLARE @ConfigID INT; DECLARE @DefaultDaysToArchive INT; @@ -75,10 +82,14 @@ BEGIN WHEN @RequestedEndDateTime < @SafeQueryStoreCutoff THEN @RequestedEndDateTime ELSE @SafeQueryStoreCutoff END; - SET @ActualStartDateTime = ISNULL( - @StartDateTime, - DATEADD(DAY, -@DefaultDaysToArchive, @ActualEndDateTime) - ); + SET @ActualStartDateTime = CASE + WHEN @LookbackMinutes IS NOT NULL + THEN DATEADD(MINUTE, -@LookbackMinutes, @ActualEndDateTime) + ELSE ISNULL( + @StartDateTime, + DATEADD(DAY, -@DefaultDaysToArchive, @ActualEndDateTime) + ) + END; IF @ActualEndDateTime < @ActualStartDateTime THROW 51003, 'The requested archive range does not contain a safely flushed Query Store interval.', 1; diff --git a/QueryVault/Tests/TestQueryStoreAggregationGrain.sql b/QueryVault/Tests/TestQueryStoreAggregationGrain.sql index 65e1757..0633996 100644 --- a/QueryVault/Tests/TestQueryStoreAggregationGrain.sql +++ b/QueryVault/Tests/TestQueryStoreAggregationGrain.sql @@ -33,6 +33,10 @@ BEGIN TRY IF OBJECT_DEFINITION(OBJECT_ID(N'dbo.usp_ArchiveQueryStore', N'P')) NOT LIKE N'%i.end_time <= @EndDateTime%' THROW 51101, 'Capture regression: completed interval end-time filter is missing.', 1; + IF OBJECT_DEFINITION(OBJECT_ID(N'dbo.usp_ArchiveQueryStore', N'P')) NOT LIKE N'%@LookbackMinutes%' + OR OBJECT_DEFINITION(OBJECT_ID(N'dbo.usp_ArchiveQueryStore', N'P')) NOT LIKE N'%DATEADD(MINUTE, -@LookbackMinutes, @ActualEndDateTime)%' + THROW 51109, 'Capture regression: relative lookback period support is missing.', 1; + IF OBJECT_DEFINITION(OBJECT_ID(N'dbo.usp_ArchiveQueryStore', N'P')) NOT LIKE N'%query_store_runtime_stats_contributor%' OR OBJECT_DEFINITION(OBJECT_ID(N'dbo.usp_ArchiveQueryStore', N'P')) NOT LIKE N'%query_store_wait_stats_contributor%' OR OBJECT_DEFINITION(OBJECT_ID(N'dbo.usp_ArchiveQueryStore', N'P')) NOT LIKE N'%usp_MaterializeCanonicalQueryStoreStats%' diff --git a/QueryVault/Tests/TestRepositoryContracts.ps1 b/QueryVault/Tests/TestRepositoryContracts.ps1 index f709c15..1dcd996 100644 --- a/QueryVault/Tests/TestRepositoryContracts.ps1 +++ b/QueryVault/Tests/TestRepositoryContracts.ps1 @@ -128,6 +128,9 @@ Assert-Condition (-not $archiveProcedure.Contains("@RunID - @MaxRunIDInFunction" Assert-Condition ($archiveProcedure.Contains("query_store_runtime_stats_contributor")) "Runtime contributor capture is missing." Assert-Condition ($archiveProcedure.Contains("query_store_wait_stats_contributor")) "Wait contributor capture is missing." Assert-Condition ($archiveProcedure.Contains("usp_MaterializeCanonicalQueryStoreStats")) "Canonical materialization call is missing." +Assert-Condition ($archiveProcedure.Contains("@LookbackMinutes INT = NULL")) "Relative capture lookback parameter is missing." +Assert-Condition ($archiveProcedure.Contains("DATEADD(MINUTE, -@LookbackMinutes, @ActualEndDateTime)")) "Relative capture lookback is not based on the safely flushed endpoint." +Assert-Condition ($archiveProcedure.Contains("Specify either @LookbackMinutes or @StartDateTime, not both.")) "Ambiguous relative/explicit capture input guard is missing." Assert-Condition (-not $archiveProcedure.Contains("Multiple Query Store runtime-stat rows exist at the documented aggregation grain")) "Obsolete runtime duplicate rejection remains." Assert-Condition (-not $archiveProcedure.Contains("Multiple Query Store wait-stat rows exist at the documented aggregation grain")) "Obsolete wait duplicate rejection remains." @@ -146,6 +149,22 @@ foreach ($tableName in @( Assert-Condition (Test-Path -LiteralPath (Join-Path $projectRoot "Tables\QueryStore\$tableName") -PathType Leaf) "Missing additive aggregation table: $tableName" } +$voyagerJob = Get-Content -LiteralPath (Join-Path $projectRoot "Scripts\Voyager2_CreateDisabledArchiveJob.sql") -Raw +Assert-Condition ($voyagerJob.Contains("SELECT @StartDateTime = MAX(EndDateTime)")) "Voyager2 job does not resume from the latest completed endpoint." +Assert-Condition ($voyagerJob.Contains("AND RunStatus = N''Completed''")) "Voyager2 job checkpoint is not restricted to completed periods." +Assert-Condition ($voyagerJob.Contains("@StartDateTime = @StartDateTime")) "Voyager2 job does not pass its checkpoint start to the archive procedure." +Assert-Condition ($voyagerJob.Contains("@EndDateTime = @EndDateTime")) "Voyager2 job does not pass its requested endpoint to the archive procedure." + +foreach ($tableName in @( + "query_store_runtime_stats_contributor", + "query_store_runtime_stats_canonical", + "query_store_wait_stats_contributor", + "query_store_wait_stats_canonical" +)) { + Assert-Condition ($voyagerJob.Contains("GRANT INSERT, ALTER ON dbo.$tableName TO [queryvault_executor];")) "Voyager2 executor permission is missing for $tableName." + Assert-Condition ($voyagerJob.Contains("GRANT ALTER ON dbo.${tableName}_PartitionMaintenance TO [queryvault_executor];")) "Voyager2 maintenance permission is missing for $tableName." +} + $queryMetricsView = Get-Content -LiteralPath (Join-Path $projectRoot "Views\qv_report.query_period_metrics.sql") -Raw $waitMetricsView = Get-Content -LiteralPath (Join-Path $projectRoot "Views\qv_report.wait_period_metrics.sql") -Raw Assert-Condition ($queryMetricsView.Contains("query_store_runtime_stats_canonical")) "Reporting runtime metrics do not prefer canonical observations." diff --git a/README.md b/README.md index 2683bd8..7f45764 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,12 @@ EXEC QueryVaultDB.dbo.usp_ArchiveQueryStore @DoNotDelete = 1; ``` +`usp_ArchiveQueryStore` accepts either an exact UTC window through +`@StartDateTime` and `@EndDateTime`, or a relative safely completed duration +through `@LookbackMinutes` (`60` for one hour, `1440` for one day). Supported +SQL Agent jobs resume from the latest completed endpoint so a failed run does +not leave an uncollected gap. + ## Safety notes - Test restores and deployments outside production first. diff --git a/docs/QueryVault-Guide.md b/docs/QueryVault-Guide.md index 11160da..a996a68 100644 --- a/docs/QueryVault-Guide.md +++ b/docs/QueryVault-Guide.md @@ -39,6 +39,22 @@ EXEC dbo.usp_ArchiveQueryStore @DoNotDelete = 1; ``` +For a relative period, pass the duration in minutes. The duration is measured +backward from the safely flushed Query Store endpoint, so `60` captures one +complete hour and `1440` captures one complete day: + +```sql +EXEC dbo.usp_ArchiveQueryStore + @SourceDatabaseName = N'YourDatabase', + @RunName = N'Last safely completed hour', + @LookbackMinutes = 60, + @DoNotDelete = 0; +``` + +Do not combine `@LookbackMinutes` with `@StartDateTime`. An explicit +`@EndDateTime` may be combined with `@LookbackMinutes` to anchor a relative +period in the past. + The effective end is capped at one source Query Store flush interval before current UTC time. If the requested range contains no safely flushed interval, the procedure fails. Every native runtime/wait contributor is retained, then @@ -48,6 +64,11 @@ native rows are not silently discarded or treated as complete observations. Use `@DoNotDelete = 1` for a deliberately protected baseline. QueryVault does not infer a period classification from `RunName`. +The supported SQL Agent jobs pass explicit start and end timestamps. They +resume from the latest `Completed` period endpoint for each source, so a failed +night is automatically included in the next successful run. For a source with +no completed period, the job falls back to `DefaultDaysToArchive`. + ## Inspect archive state The administrative summary remains available: