From fe49308c789e1378b3c02db9bf4d83390bb7d77c Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Thu, 13 Mar 2025 15:07:26 -0400 Subject: [PATCH] feat(core): provide default value for max cache size (#30351) ## Current Behavior The max cache size is disabled by default ## Expected Behavior Max cache size is set to 10% of the current disk by default, and can be disabled by specifying 0. Information about this shows up in `nx report`. image ## Related Issue(s) Fixes # --- Cargo.lock | 87 +++++++++++++- packages/nx/Cargo.toml | 1 + packages/nx/src/command-line/report/report.ts | 33 ++++++ packages/nx/src/native/cache/cache.rs | 107 ++++++++++++------ packages/nx/src/native/index.d.ts | 3 + packages/nx/src/native/native-bindings.js | 1 + packages/nx/src/tasks-runner/cache.spec.ts | 31 ++++- packages/nx/src/tasks-runner/cache.ts | 40 +++++-- 8 files changed, 256 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c85c158bb..ac4b7b5c92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,6 +332,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "crossbeam-channel" version = "0.5.12" @@ -1142,9 +1148,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libloading" @@ -1418,6 +1424,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1520,6 +1535,7 @@ dependencies = [ "swc_ecma_dep_graph", "swc_ecma_parser", "swc_ecma_visit", + "sysinfo", "tempfile", "thiserror", "tokio", @@ -2418,6 +2434,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + [[package]] name = "tap" version = "1.0.1" @@ -2874,6 +2904,59 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index 91786451b8..f39de05853 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -43,6 +43,7 @@ swc_common = "0.31.16" swc_ecma_parser = { version = "0.137.1", features = ["typescript"] } swc_ecma_visit = "0.93.0" swc_ecma_ast = "0.107.0" +sysinfo = "0.33.1" rand = "0.9.0" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["fileapi"] } diff --git a/packages/nx/src/command-line/report/report.ts b/packages/nx/src/command-line/report/report.ts index e42701781b..8e88be1de6 100644 --- a/packages/nx/src/command-line/report/report.ts +++ b/packages/nx/src/command-line/report/report.ts @@ -29,6 +29,14 @@ import { createNxKeyLicenseeInformation, } from '../../utils/nx-key'; import { type NxKey } from '@nx/key'; +import { + DbCache, + dbCacheEnabled, + formatCacheSize, + parseMaxCacheSize, +} from '../../tasks-runner/cache'; +import { getDefaultMaxCacheSize } from '../../native'; +import { cacheDir } from '../../utils/cache-directory'; const nxPackageJson = readJsonFile( join(__dirname, '../../../package.json') @@ -75,6 +83,7 @@ export async function reportHandler() { outOfSyncPackageGroup, projectGraphError, nativeTarget, + cache, } = await getReportData(); const fields = [ @@ -191,6 +200,15 @@ export async function reportHandler() { } } + if (cache) { + bodyLines.push(LINE_SEPARATOR); + bodyLines.push( + `Cache Usage: ${formatCacheSize(cache.used)} / ${ + cache.max === 0 ? '∞' : formatCacheSize(cache.max) + }` + ); + } + if (outOfSyncPackageGroup) { bodyLines.push(LINE_SEPARATOR); bodyLines.push( @@ -241,6 +259,10 @@ export interface ReportData { }; projectGraphError?: Error | null; nativeTarget: string | null; + cache: { + max: number; + used: number; + } | null; } export async function getReportData(): Promise { @@ -281,6 +303,16 @@ export async function getReportData(): Promise { } } + let cache = dbCacheEnabled(nxJson) + ? { + max: + nxJson.maxCacheSize !== undefined + ? parseMaxCacheSize(nxJson.maxCacheSize) + : getDefaultMaxCacheSize(cacheDir), + used: new DbCache({ nxCloudRemoteCache: null }).getUsedCacheSpace(), + } + : null; + return { pm, nxKey, @@ -294,6 +326,7 @@ export async function getReportData(): Promise { outOfSyncPackageGroup, projectGraphError, nativeTarget: native ? native.getBinaryTarget() : null, + cache, }; } diff --git a/packages/nx/src/native/cache/cache.rs b/packages/nx/src/native/cache/cache.rs index 5654fa907e..1201d4c00b 100644 --- a/packages/nx/src/native/cache/cache.rs +++ b/packages/nx/src/native/cache/cache.rs @@ -6,6 +6,7 @@ use fs_extra::remove_items; use napi::bindgen_prelude::*; use regex::Regex; use rusqlite::params; +use sysinfo::Disks; use tracing::trace; use crate::native::cache::expand_outputs::_expand_outputs; @@ -29,7 +30,7 @@ pub struct NxCache { cache_path: PathBuf, db: External, link_task_details: bool, - max_cache_size: Option, + max_cache_size: i64, } #[napi] @@ -47,6 +48,8 @@ impl NxCache { create_dir_all(&cache_path)?; create_dir_all(cache_path.join("terminalOutputs"))?; + let max_cache_size = max_cache_size.unwrap_or(0); + let r = Self { db: db_connection, workspace_root: PathBuf::from(workspace_root), @@ -207,48 +210,63 @@ impl NxCache { "INSERT OR REPLACE INTO cache_outputs (hash, code, size) VALUES (?1, ?2, ?3)", params![hash, code, size], )?; - if self.max_cache_size.is_some() { + if self.max_cache_size != 0 { self.ensure_cache_size_within_limit()? } Ok(()) } - fn ensure_cache_size_within_limit(&self) -> anyhow::Result<()> { - if let Some(user_specified_max_cache_size) = self.max_cache_size { - let buffer_amount = (0.1 * user_specified_max_cache_size as f64) as i64; - let target_cache_size = user_specified_max_cache_size - buffer_amount; + #[napi] + pub fn get_cache_size(&self) -> anyhow::Result { + self.db + .query_row("SELECT SUM(size) FROM cache_outputs", [], |row| { + row.get::<_, Option>(0) + // If there are no cache entries, the result is + // a single row with a NULL value. This would look like: + // Ok(None). We need to convert this to Ok(0). + .transpose() + .unwrap_or(Ok(0)) + }) + // The query_row returns an Result> to account for + // a query that returned no rows. This isn't possible when using + // SUM, so we can safely unwrap the Option, but need to transpose + // to access it. The result represents a db error or mapping error. + .transpose() + .unwrap_or(Ok(0)) + } - let full_cache_size = self - .db - .query_row("SELECT SUM(size) FROM cache_outputs", [], |row| { - row.get::<_, i64>(0) - })? - .unwrap_or(0); - if user_specified_max_cache_size < full_cache_size { - let mut cache_size = full_cache_size; - let mut stmt = self.db.prepare( - "SELECT hash, size FROM cache_outputs ORDER BY accessed_at ASC LIMIT 100", - )?; - 'outer: while cache_size > target_cache_size { - let rows = stmt.query_map([], |r| { - let hash: String = r.get(0)?; - let size: i64 = r.get(1)?; - Ok((hash, size)) - })?; - for row in rows { - if let Ok((hash, size)) = row { - cache_size -= size; - self.db.execute( - "DELETE FROM cache_outputs WHERE hash = ?1", - params![hash], - )?; - remove_items(&[self.cache_path.join(&hash)])?; - } - // We've deleted enough cache entries to be under the - // target cache size, stop looking for more. - if cache_size < target_cache_size { - break 'outer; - } + fn ensure_cache_size_within_limit(&self) -> anyhow::Result<()> { + // 0 is equivalent to being unlimited. + if self.max_cache_size == 0 { + return Ok(()); + } + let user_specified_max_cache_size = self.max_cache_size; + let buffer_amount = (0.1 * user_specified_max_cache_size as f64) as i64; + let target_cache_size = user_specified_max_cache_size - buffer_amount; + + let full_cache_size = self.get_cache_size()?; + if user_specified_max_cache_size < full_cache_size { + let mut cache_size = full_cache_size; + let mut stmt = self.db.prepare( + "SELECT hash, size FROM cache_outputs ORDER BY accessed_at ASC LIMIT 100", + )?; + 'outer: while cache_size > target_cache_size { + let rows = stmt.query_map([], |r| { + let hash: String = r.get(0)?; + let size: i64 = r.get(1)?; + Ok((hash, size)) + })?; + for row in rows { + if let Ok((hash, size)) = row { + cache_size -= size; + self.db + .execute("DELETE FROM cache_outputs WHERE hash = ?1", params![hash])?; + remove_items(&[self.cache_path.join(&hash)])?; + } + // We've deleted enough cache entries to be under the + // target cache size, stop looking for more. + if cache_size < target_cache_size { + break 'outer; } } } @@ -346,6 +364,21 @@ impl NxCache { } } +#[napi] +fn get_default_max_cache_size(cache_path: String) -> i64 { + let disks = Disks::new_with_refreshed_list(); + let cache_path = PathBuf::from(cache_path); + + for disk in disks.list() { + if cache_path.starts_with(disk.mount_point()) { + return (disk.total_space() as f64 * 0.1) as i64; + } + } + + // Default to 100gb + 100 * 1024 * 1024 * 1024 +} + fn try_and_retry(mut f: F) -> anyhow::Result where F: FnMut() -> anyhow::Result, diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index 2e5d3bfd2e..bbe62915d8 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -42,6 +42,7 @@ export declare class NxCache { put(hash: string, terminalOutput: string, outputs: Array, code: number): void applyRemoteCacheResults(hash: string, result: CachedResult, outputs: Array): void getTaskOutputsPath(hash: string): string + getCacheSize(): number copyFilesFromCache(cachedResult: CachedResult, outputs: Array): number removeOldCacheRecords(): void checkCacheFsInSync(): boolean @@ -166,6 +167,8 @@ export declare export function findImports(projectFileMap: Record { describe('parseMaxCacheSize', () => { + it('should support numerical byte values', () => { + expect(parseMaxCacheSize('0')).toEqual(0); + expect(parseMaxCacheSize(0)).toEqual(0); + expect(parseMaxCacheSize('1')).toEqual(1); + expect(parseMaxCacheSize(1024)).toEqual(1024); + }); + it('should parse KB', () => { expect(parseMaxCacheSize('1KB')).toEqual(1024); }); @@ -38,4 +45,26 @@ describe('cache', () => { expect(() => parseMaxCacheSize('1.5.5KB')).toThrow; }); }); + + describe('formatCacheSize', () => { + it('should format bytes', () => { + expect(formatCacheSize(1)).toEqual('1.00 B'); + }); + + it('should format KB', () => { + expect(formatCacheSize(1024)).toEqual('1.00 KB'); + }); + + it('should format MB', () => { + expect(formatCacheSize(1024 * 1024)).toEqual('1.00 MB'); + }); + + it('should format GB', () => { + expect(formatCacheSize(1024 * 1024 * 1024)).toEqual('1.00 GB'); + }); + + it('should format partial units', () => { + expect(formatCacheSize(1024 * 88.5)).toEqual('88.50 KB'); + }); + }); }); diff --git a/packages/nx/src/tasks-runner/cache.ts b/packages/nx/src/tasks-runner/cache.ts index f96d78ec22..49e9edca8e 100644 --- a/packages/nx/src/tasks-runner/cache.ts +++ b/packages/nx/src/tasks-runner/cache.ts @@ -12,7 +12,12 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { cacheDir } from '../utils/cache-directory'; import { Task } from '../config/task-graph'; import { machineId } from 'node-machine-id'; -import { NxCache, CachedResult as NativeCacheResult, IS_WASM } from '../native'; +import { + NxCache, + CachedResult as NativeCacheResult, + IS_WASM, + getDefaultMaxCacheSize, +} from '../native'; import { getDbConnection } from '../utils/db-connection'; import { isNxCloudUsed } from '../utils/nx-cloud-utils'; import { NxJsonConfiguration, readNxJson } from '../config/nx-json'; @@ -95,7 +100,9 @@ export class DbCache { cacheDir, getDbConnection(), undefined, - parseMaxCacheSize(this.nxJson.maxCacheSize) + this.nxJson.maxCacheSize !== undefined + ? parseMaxCacheSize(this.nxJson.maxCacheSize) + : getDefaultMaxCacheSize(cacheDir) ); private remoteCache: RemoteCacheV2 | null; @@ -150,6 +157,10 @@ export class DbCache { } } + getUsedCacheSpace() { + return this.cache.getCacheSize(); + } + private applyRemoteCacheResults( hash: string, res: NativeCacheResult, @@ -603,13 +614,17 @@ function tryAndRetry(fn: () => Promise): Promise { * * @param maxCacheSize Max cache size as specified in nx.json */ -export function parseMaxCacheSize(maxCacheSize: string): number | undefined { - if (!maxCacheSize) { +export function parseMaxCacheSize( + maxCacheSize: string | number +): number | undefined { + if (maxCacheSize === null || maxCacheSize === undefined) { return undefined; } - let regexResult = maxCacheSize.match( - /^(?[\d|.]+)\s?((?[KMG]?B)?)$/ - ); + let regexResult = maxCacheSize + // Covers folks who accidentally specify as a number rather than a string + .toString() + // Match a number followed by an optional unit (KB, MB, GB), with optional whitespace between the number and unit + .match(/^(?[\d|.]+)\s?((?[KMG]?B)?)$/); if (!regexResult) { throw new Error( `Invalid max cache size specified in nx.json: ${maxCacheSize}. Must be a number followed by an optional unit (KB, MB, GB)` @@ -639,3 +654,14 @@ export function parseMaxCacheSize(maxCacheSize: string): number | undefined { return size; } } + +export function formatCacheSize(maxCacheSize: number, decimals = 2): string { + const exponents = ['B', 'KB', 'MB', 'GB']; + let exponent = 0; + let size = maxCacheSize; + while (size >= 1024 && exponent < exponents.length - 1) { + size /= 1024; + exponent++; + } + return `${size.toFixed(decimals)} ${exponents[exponent]}`; +}