diff --git a/lib/entry-points.js b/lib/entry-points.js index c10b0ce2ea..12bdf418f8 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -147882,6 +147882,36 @@ var featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN", minimumVersion: void 0 }, + ["overlay_analysis_min_disk_8_gb" /* OverlayAnalysisMinDisk8Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_9_gb" /* OverlayAnalysisMinDisk9Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_10_gb" /* OverlayAnalysisMinDisk10Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_11_gb" /* OverlayAnalysisMinDisk11Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_12_gb" /* OverlayAnalysisMinDisk12Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_13_gb" /* OverlayAnalysisMinDisk13Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB", + minimumVersion: void 0 + }, ["overlay_analysis_status_check" /* OverlayAnalysisStatusCheck */]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK", @@ -150113,7 +150143,14 @@ async function cachePrefix(codeql, language) { // src/config-utils.ts var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14e3; -var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1e6; +var OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE = { + ["overlay_analysis_min_disk_8_gb" /* OverlayAnalysisMinDisk8Gb */]: 8e3, + ["overlay_analysis_min_disk_9_gb" /* OverlayAnalysisMinDisk9Gb */]: 9e3, + ["overlay_analysis_min_disk_10_gb" /* OverlayAnalysisMinDisk10Gb */]: 1e4, + ["overlay_analysis_min_disk_11_gb" /* OverlayAnalysisMinDisk11Gb */]: 11e3, + ["overlay_analysis_min_disk_12_gb" /* OverlayAnalysisMinDisk12Gb */]: 12e3, + ["overlay_analysis_min_disk_13_gb" /* OverlayAnalysisMinDisk13Gb */]: 13e3 +}; var OVERLAY_MINIMUM_MEMORY_MB = 5 * 1024; var CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE = "2.24.3"; async function getSupportedLanguageMap(codeql, logger) { @@ -150363,16 +150400,28 @@ async function checkOverlayAnalysisFeatureEnabled(features, codeql, languages, c } return new Success(void 0); } -function runnerHasSufficientDiskSpace(diskUsage, logger) { - const minimumDiskSpaceBytes = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES; - if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) { - const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1e6); - const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1e6); +async function getMinimumDiskSpaceMb(features) { + let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB; + for (const [feature, thresholdMb] of Object.entries( + OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE + )) { + if (await features.getValue(feature)) { + minimumMb = Math.min(minimumMb, thresholdMb); + } + } + return minimumMb; +} +function runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb) { + const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1e6); + if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1e6) { logger.info( `Setting overlay database mode to ${"none" /* None */} due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).` ); return false; } + logger.debug( + `Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the minimum of ${minimumDiskSpaceMb} MB.` + ); return true; } async function runnerHasSufficientMemory(codeql, ramInput, logger) { @@ -150393,12 +150442,13 @@ async function runnerHasSufficientMemory(codeql, ramInput, logger) { return false; } logger.debug( - `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.` + `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.` ); return true; } -async function checkRunnerResources(codeql, diskUsage, ramInput, logger) { - if (!runnerHasSufficientDiskSpace(diskUsage, logger)) { +async function checkRunnerResources(codeql, features, diskUsage, ramInput, logger) { + const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features); + if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) { return new Failure("insufficient-disk-space" /* InsufficientDiskSpace */); } if (!await runnerHasSufficientMemory(codeql, ramInput, logger)) { @@ -150457,7 +150507,13 @@ async function checkOverlayEnablement(codeql, features, languages, sourceRoot, b ); return new Failure("unable-to-determine-disk-usage" /* UnableToDetermineDiskUsage */); } - const resourceResult = performResourceChecks && diskUsage !== void 0 ? await checkRunnerResources(codeql, diskUsage, ramInput, logger) : new Success(void 0); + const resourceResult = performResourceChecks && diskUsage !== void 0 ? await checkRunnerResources( + codeql, + features, + diskUsage, + ramInput, + logger + ) : new Success(void 0); if (resourceResult.isFailure()) { return resourceResult; } diff --git a/src/config-utils.test.ts b/src/config-utils.test.ts index aec214cd64..29d72f3af4 100644 --- a/src/config-utils.test.ts +++ b/src/config-utils.test.ts @@ -1333,6 +1333,101 @@ checkOverlayEnablementMacro.serial( }, ); +// Check that each feature flag lowers the limit to the threshold that its name +// declares. Both sides of the boundary are needed to pin the threshold down: a +// mapping to a lower value would still pass the case at the limit, and one to a +// higher value would still fail the case below it. +for (const [feature, thresholdGb] of [ + [Feature.OverlayAnalysisMinDisk8Gb, 8], + [Feature.OverlayAnalysisMinDisk9Gb, 9], + [Feature.OverlayAnalysisMinDisk10Gb, 10], + [Feature.OverlayAnalysisMinDisk11Gb, 11], + [Feature.OverlayAnalysisMinDisk12Gb, 12], + [Feature.OverlayAnalysisMinDisk13Gb, 13], +] as Array<[Feature, number]>) { + const features = [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisCodeScanningJavascript, + feature, + ]; + + checkOverlayEnablementMacro.serial( + `Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`, + { + languages: [BuiltInLanguage.javascript], + features, + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: thresholdGb * 1_000_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + overlayDatabaseMode: OverlayDatabaseMode.OverlayBase, + useOverlayDatabaseCaching: true, + }, + ); + + checkOverlayEnablementMacro.serial( + `No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`, + { + languages: [BuiltInLanguage.javascript], + features, + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + disabledReason: OverlayDisabledReason.InsufficientDiskSpace, + }, + ); +} + +checkOverlayEnablementMacro.serial( + "Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag", + { + languages: [BuiltInLanguage.javascript], + features: [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisCodeScanningJavascript, + Feature.OverlayAnalysisMinDisk9Gb, + Feature.OverlayAnalysisMinDisk12Gb, + ], + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: 9_000_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + overlayDatabaseMode: OverlayDatabaseMode.OverlayBase, + useOverlayDatabaseCaching: true, + }, +); + +checkOverlayEnablementMacro.serial( + "No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag", + { + languages: [BuiltInLanguage.javascript], + features: [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisCodeScanningJavascript, + Feature.OverlayAnalysisMinDisk9Gb, + Feature.OverlayAnalysisMinDisk12Gb, + ], + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: 8_500_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + disabledReason: OverlayDisabledReason.InsufficientDiskSpace, + }, +); + checkOverlayEnablementMacro.serial( "No overlay-base database on default branch if memory flag is too low", { diff --git a/src/config-utils.ts b/src/config-utils.ts index 6d1efaa1ba..0a6ced00aa 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -48,7 +48,7 @@ import { import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils"; import { EnvVar } from "./environment"; import * as errorMessages from "./error-messages"; -import { Feature, FeatureEnablement } from "./feature-flags"; +import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags"; import { RepositoryProperties, RepositoryPropertyName, @@ -101,10 +101,23 @@ export { type Config } from "./config/action-config"; * whether to perform overlay analysis, then the action will not perform overlay * analysis unless overlay analysis has been explicitly enabled via environment * variable. + * + * This threshold can be lowered by the feature flags in + * `OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE`. */ const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000; -const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES = - OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000; + +/** + * Minimum available disk space (in MB) enabled by each overlay feature flag. + */ +const OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE = { + [Feature.OverlayAnalysisMinDisk8Gb]: 8000, + [Feature.OverlayAnalysisMinDisk9Gb]: 9000, + [Feature.OverlayAnalysisMinDisk10Gb]: 10000, + [Feature.OverlayAnalysisMinDisk11Gb]: 11000, + [Feature.OverlayAnalysisMinDisk12Gb]: 12000, + [Feature.OverlayAnalysisMinDisk13Gb]: 13000, +} satisfies Partial>; /** * The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If @@ -579,21 +592,44 @@ async function checkOverlayAnalysisFeatureEnabled( return new Success(undefined); } +/** + * Returns the minimum available disk space (in MB) required to perform overlay + * analysis, which is the lowest threshold enabled by a feature flag, or the + * default threshold if no such feature flag is enabled. + */ +async function getMinimumDiskSpaceMb( + features: FeatureEnablement, +): Promise { + let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB; + for (const [feature, thresholdMb] of Object.entries( + OVERLAY_MINIMUM_DISK_SPACE_MB_BY_FEATURE, + )) { + if (await features.getValue(feature as FeatureWithoutCLI)) { + minimumMb = Math.min(minimumMb, thresholdMb); + } + } + return minimumMb; +} + /** Checks if the runner has enough disk space for overlay analysis. */ function runnerHasSufficientDiskSpace( diskUsage: DiskUsage, logger: Logger, + minimumDiskSpaceMb: number, ): boolean { - const minimumDiskSpaceBytes = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES; - if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) { - const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000); - const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000); + const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000); + if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) { logger.info( `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + `due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`, ); return false; } + + logger.debug( + `Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` + + `minimum of ${minimumDiskSpaceMb} MB.`, + ); return true; } @@ -625,7 +661,7 @@ async function runnerHasSufficientMemory( } logger.debug( - `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`, + `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`, ); return true; } @@ -636,11 +672,13 @@ async function runnerHasSufficientMemory( */ async function checkRunnerResources( codeql: CodeQL, + features: FeatureEnablement, diskUsage: DiskUsage, ramInput: string | undefined, logger: Logger, ): Promise> { - if (!runnerHasSufficientDiskSpace(diskUsage, logger)) { + const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features); + if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) { return new Failure(OverlayDisabledReason.InsufficientDiskSpace); } if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) { @@ -752,7 +790,13 @@ export async function checkOverlayEnablement( } const resourceResult = performResourceChecks && diskUsage !== undefined - ? await checkRunnerResources(codeql, diskUsage, ramInput, logger) + ? await checkRunnerResources( + codeql, + features, + diskUsage, + ramInput, + logger, + ) : new Success(undefined); if (resourceResult.isFailure()) { return resourceResult; diff --git a/src/feature-flags.ts b/src/feature-flags.ts index fff7ef0440..7abccf60cb 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -121,6 +121,36 @@ export enum Feature { * `OverlayAnalysisMatchCodeqlVersion` overrides this flag. */ OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run", + /** + * Lowers the overlay minimum available disk space to 8 GB. The lowest enabled limit wins; if + * none are enabled, the default applies. + */ + OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb", + /** + * Lowers the overlay minimum available disk space to 9 GB. The lowest enabled limit wins; if + * none are enabled, the default applies. + */ + OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb", + /** + * Lowers the overlay minimum available disk space to 10 GB. The lowest enabled limit wins; if + * none are enabled, the default applies. + */ + OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb", + /** + * Lowers the overlay minimum available disk space to 11 GB. The lowest enabled limit wins; if + * none are enabled, the default applies. + */ + OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb", + /** + * Lowers the overlay minimum available disk space to 12 GB. The lowest enabled limit wins; if + * none are enabled, the default applies. + */ + OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb", + /** + * Lowers the overlay minimum available disk space to 13 GB. The lowest enabled limit wins; if + * none are enabled, the default applies. + */ + OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb", OverlayAnalysisPython = "overlay_analysis_python", OverlayAnalysisRuby = "overlay_analysis_ruby", /** Controls whether hardware checks are skipped for overlay analysis. */ @@ -349,6 +379,36 @@ export const featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN", minimumVersion: undefined, }, + [Feature.OverlayAnalysisMinDisk8Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk9Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk10Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk11Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk12Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk13Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB", + minimumVersion: undefined, + }, [Feature.OverlayAnalysisStatusCheck]: { defaultValue: false, envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_STATUS_CHECK",