feat(core): provide default value for max cache size (#30351)

<!-- Please make sure you have read the submission guidelines before
posting an PR -->
<!--
https://github.com/nrwl/nx/blob/master/CONTRIBUTING.md#-submitting-a-pr
-->

<!-- Please make sure that your commit message follows our format -->
<!-- Example: `fix(nx): must begin with lowercase` -->

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
The max cache size is disabled by default

## Expected Behavior
Max cache size is set to 10% of the current disk by default, and can be
disabled by specifying 0. Information about this shows up in `nx
report`.

<img width="331" alt="image"
src="https://github.com/user-attachments/assets/ee937101-9915-49d1-b3f1-c2f0d929c140"
/>


## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

Cargo.lock

@@ -332,6 +332,12 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.12"
@@ -1142,9 +1148,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.161"
+version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "libloading"
@@ -1418,6 +1424,15 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -1520,6 +1535,7 @@ dependencies = [
  "swc_ecma_dep_graph",
  "swc_ecma_parser",
  "swc_ecma_visit",
+ "sysinfo",
  "tempfile",
  "thiserror",
  "tokio",
@@ -2418,6 +2434,20 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "sysinfo"
+version = "0.33.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+ "memchr",
+ "ntapi",
+ "rayon",
+ "windows",
+]
+
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -2874,6 +2904,59 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-result",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.53",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.53",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"

packages/nx/Cargo.toml

@@ -43,6 +43,7 @@ swc_common = "0.31.16"
 swc_ecma_parser = { version = "0.137.1", features = ["typescript"] }
 swc_ecma_visit = "0.93.0"
 swc_ecma_ast = "0.107.0"
+sysinfo = "0.33.1"
 rand = "0.9.0"
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["fileapi"] }

packages/nx/src/command-line/report/report.ts

@@ -29,6 +29,14 @@ import {
   createNxKeyLicenseeInformation,
 } from '../../utils/nx-key';
 import { type NxKey } from '@nx/key';
+import {
+  DbCache,
+  dbCacheEnabled,
+  formatCacheSize,
+  parseMaxCacheSize,
+} from '../../tasks-runner/cache';
+import { getDefaultMaxCacheSize } from '../../native';
+import { cacheDir } from '../../utils/cache-directory';
 
 const nxPackageJson = readJsonFile<typeof import('../../../package.json')>(
   join(__dirname, '../../../package.json')
@@ -75,6 +83,7 @@ export async function reportHandler() {
     outOfSyncPackageGroup,
     projectGraphError,
     nativeTarget,
+    cache,
   } = await getReportData();
 
   const fields = [
@@ -191,6 +200,15 @@ export async function reportHandler() {
     }
   }
 
+  if (cache) {
+    bodyLines.push(LINE_SEPARATOR);
+    bodyLines.push(
+      `Cache Usage: ${formatCacheSize(cache.used)} / ${
+        cache.max === 0 ? '∞' : formatCacheSize(cache.max)
+      }`
+    );
+  }
+
   if (outOfSyncPackageGroup) {
     bodyLines.push(LINE_SEPARATOR);
     bodyLines.push(
@@ -241,6 +259,10 @@ export interface ReportData {
   };
   projectGraphError?: Error | null;
   nativeTarget: string | null;
+  cache: {
+    max: number;
+    used: number;
+  } | null;
 }
 
 export async function getReportData(): Promise<ReportData> {
@@ -281,6 +303,16 @@ export async function getReportData(): Promise<ReportData> {
     }
   }
 
+  let cache = dbCacheEnabled(nxJson)
+    ? {
+        max:
+          nxJson.maxCacheSize !== undefined
+            ? parseMaxCacheSize(nxJson.maxCacheSize)
+            : getDefaultMaxCacheSize(cacheDir),
+        used: new DbCache({ nxCloudRemoteCache: null }).getUsedCacheSpace(),
+      }
+    : null;
+
   return {
     pm,
     nxKey,
@@ -294,6 +326,7 @@ export async function getReportData(): Promise<ReportData> {
     outOfSyncPackageGroup,
     projectGraphError,
     nativeTarget: native ? native.getBinaryTarget() : null,
+    cache,
   };
 }
 

packages/nx/src/native/cache/cache.rs

@@ -6,6 +6,7 @@ use fs_extra::remove_items;
 use napi::bindgen_prelude::*;
 use regex::Regex;
 use rusqlite::params;
+use sysinfo::Disks;
 use tracing::trace;
 
 use crate::native::cache::expand_outputs::_expand_outputs;
@@ -29,7 +30,7 @@ pub struct NxCache {
     cache_path: PathBuf,
     db: External<NxDbConnection>,
     link_task_details: bool,
-    max_cache_size: Option<i64>,
+    max_cache_size: i64,
 }
 
 #[napi]
@@ -47,6 +48,8 @@ impl NxCache {
         create_dir_all(&cache_path)?;
         create_dir_all(cache_path.join("terminalOutputs"))?;
 
+        let max_cache_size = max_cache_size.unwrap_or(0);
+
         let r = Self {
             db: db_connection,
             workspace_root: PathBuf::from(workspace_root),
@@ -207,48 +210,63 @@ impl NxCache {
             "INSERT OR REPLACE INTO cache_outputs (hash, code, size) VALUES (?1, ?2, ?3)",
             params![hash, code, size],
         )?;
-        if self.max_cache_size.is_some() {
+        if self.max_cache_size != 0 {
             self.ensure_cache_size_within_limit()?
         }
         Ok(())
     }
 
-    fn ensure_cache_size_within_limit(&self) -> anyhow::Result<()> {
+    #[napi]
-        if let Some(user_specified_max_cache_size) = self.max_cache_size {
+    pub fn get_cache_size(&self) -> anyhow::Result<i64> {
-            let buffer_amount = (0.1 * user_specified_max_cache_size as f64) as i64;
+        self.db
-            let target_cache_size = user_specified_max_cache_size - buffer_amount;
+            .query_row("SELECT SUM(size) FROM cache_outputs", [], |row| {
+                row.get::<_, Option<i64>>(0)
+                    // If there are no cache entries, the result is
+                    // a single row with a NULL value. This would look like:
+                    // Ok(None). We need to convert this to Ok(0).
+                    .transpose()
+                    .unwrap_or(Ok(0))
+            })
+            // The query_row returns an Result<Option<T>> to account for
+            // a query that returned no rows. This isn't possible when using
+            // SUM, so we can safely unwrap the Option, but need to transpose
+            // to access it. The result represents a db error or mapping error.
+            .transpose()
+            .unwrap_or(Ok(0))
+    }
 
-            let full_cache_size = self
+    fn ensure_cache_size_within_limit(&self) -> anyhow::Result<()> {
-                .db
+        // 0 is equivalent to being unlimited.
-                .query_row("SELECT SUM(size) FROM cache_outputs", [], |row| {
+        if self.max_cache_size == 0 {
-                    row.get::<_, i64>(0)
+            return Ok(());
-                })?
+        }
-                .unwrap_or(0);
+        let user_specified_max_cache_size = self.max_cache_size;
-            if user_specified_max_cache_size < full_cache_size {
+        let buffer_amount = (0.1 * user_specified_max_cache_size as f64) as i64;
-                let mut cache_size = full_cache_size;
+        let target_cache_size = user_specified_max_cache_size - buffer_amount;
-                let mut stmt = self.db.prepare(
+
-                    "SELECT hash, size FROM cache_outputs ORDER BY accessed_at ASC LIMIT 100",
+        let full_cache_size = self.get_cache_size()?;
-                )?;
+        if user_specified_max_cache_size < full_cache_size {
-                'outer: while cache_size > target_cache_size {
+            let mut cache_size = full_cache_size;
-                    let rows = stmt.query_map([], |r| {
+            let mut stmt = self.db.prepare(
-                        let hash: String = r.get(0)?;
+                "SELECT hash, size FROM cache_outputs ORDER BY accessed_at ASC LIMIT 100",
-                        let size: i64 = r.get(1)?;
+            )?;
-                        Ok((hash, size))
+            'outer: while cache_size > target_cache_size {
-                    })?;
+                let rows = stmt.query_map([], |r| {
-                    for row in rows {
+                    let hash: String = r.get(0)?;
-                        if let Ok((hash, size)) = row {
+                    let size: i64 = r.get(1)?;
-                            cache_size -= size;
+                    Ok((hash, size))
-                            self.db.execute(
+                })?;
-                                "DELETE FROM cache_outputs WHERE hash = ?1",
+                for row in rows {
-                                params![hash],
+                    if let Ok((hash, size)) = row {
-                            )?;
+                        cache_size -= size;
-                            remove_items(&[self.cache_path.join(&hash)])?;
+                        self.db
-                        }
+                            .execute("DELETE FROM cache_outputs WHERE hash = ?1", params![hash])?;
-                        // We've deleted enough cache entries to be under the
+                        remove_items(&[self.cache_path.join(&hash)])?;
-                        // target cache size, stop looking for more.
+                    }
-                        if cache_size < target_cache_size {
+                    // We've deleted enough cache entries to be under the
-                            break 'outer;
+                    // target cache size, stop looking for more.
-                        }
+                    if cache_size < target_cache_size {
+                        break 'outer;
                     }
                 }
             }
@@ -346,6 +364,21 @@ impl NxCache {
     }
 }
 
+#[napi]
+fn get_default_max_cache_size(cache_path: String) -> i64 {
+    let disks = Disks::new_with_refreshed_list();
+    let cache_path = PathBuf::from(cache_path);
+
+    for disk in disks.list() {
+        if cache_path.starts_with(disk.mount_point()) {
+            return (disk.total_space() as f64 * 0.1) as i64;
+        }
+    }
+
+    // Default to 100gb
+    100 * 1024 * 1024 * 1024
+}
+
 fn try_and_retry<T, F>(mut f: F) -> anyhow::Result<T>
 where
     F: FnMut() -> anyhow::Result<T>,

packages/nx/src/native/index.d.ts

@@ -42,6 +42,7 @@ export declare class NxCache {
   put(hash: string, terminalOutput: string, outputs: Array<string>, code: number): void
   applyRemoteCacheResults(hash: string, result: CachedResult, outputs: Array<string>): void
   getTaskOutputsPath(hash: string): string
+  getCacheSize(): number
   copyFilesFromCache(cachedResult: CachedResult, outputs: Array<string>): number
   removeOldCacheRecords(): void
   checkCacheFsInSync(): boolean
@@ -166,6 +167,8 @@ export declare export function findImports(projectFileMap: Record<string, Array<
 
 export declare export function getBinaryTarget(): string
 
+export declare export function getDefaultMaxCacheSize(cachePath: string): number
+
 /**
  * Expands the given outputs into a list of existing files.
  * This is used when hashing outputs

packages/nx/src/native/native-bindings.js

@@ -379,6 +379,7 @@ module.exports.EventType = nativeBinding.EventType
 module.exports.expandOutputs = nativeBinding.expandOutputs
 module.exports.findImports = nativeBinding.findImports
 module.exports.getBinaryTarget = nativeBinding.getBinaryTarget
+module.exports.getDefaultMaxCacheSize = nativeBinding.getDefaultMaxCacheSize
 module.exports.getFilesForOutputs = nativeBinding.getFilesForOutputs
 module.exports.getTransformableOutputs = nativeBinding.getTransformableOutputs
 module.exports.hashArray = nativeBinding.hashArray

packages/nx/src/tasks-runner/cache.spec.ts

@@ -1,7 +1,14 @@
-import { parseMaxCacheSize } from './cache';
+import { formatCacheSize, parseMaxCacheSize } from './cache';
 
 describe('cache', () => {
   describe('parseMaxCacheSize', () => {
+    it('should support numerical byte values', () => {
+      expect(parseMaxCacheSize('0')).toEqual(0);
+      expect(parseMaxCacheSize(0)).toEqual(0);
+      expect(parseMaxCacheSize('1')).toEqual(1);
+      expect(parseMaxCacheSize(1024)).toEqual(1024);
+    });
+
     it('should parse KB', () => {
       expect(parseMaxCacheSize('1KB')).toEqual(1024);
     });
@@ -38,4 +45,26 @@ describe('cache', () => {
       expect(() => parseMaxCacheSize('1.5.5KB')).toThrow;
     });
   });
+
+  describe('formatCacheSize', () => {
+    it('should format bytes', () => {
+      expect(formatCacheSize(1)).toEqual('1.00 B');
+    });
+
+    it('should format KB', () => {
+      expect(formatCacheSize(1024)).toEqual('1.00 KB');
+    });
+
+    it('should format MB', () => {
+      expect(formatCacheSize(1024 * 1024)).toEqual('1.00 MB');
+    });
+
+    it('should format GB', () => {
+      expect(formatCacheSize(1024 * 1024 * 1024)).toEqual('1.00 GB');
+    });
+
+    it('should format partial units', () => {
+      expect(formatCacheSize(1024 * 88.5)).toEqual('88.50 KB');
+    });
+  });
 });

packages/nx/src/tasks-runner/cache.ts

@@ -12,7 +12,12 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import { cacheDir } from '../utils/cache-directory';
 import { Task } from '../config/task-graph';
 import { machineId } from 'node-machine-id';
-import { NxCache, CachedResult as NativeCacheResult, IS_WASM } from '../native';
+import {
+  NxCache,
+  CachedResult as NativeCacheResult,
+  IS_WASM,
+  getDefaultMaxCacheSize,
+} from '../native';
 import { getDbConnection } from '../utils/db-connection';
 import { isNxCloudUsed } from '../utils/nx-cloud-utils';
 import { NxJsonConfiguration, readNxJson } from '../config/nx-json';
@@ -95,7 +100,9 @@ export class DbCache {
     cacheDir,
     getDbConnection(),
     undefined,
-    parseMaxCacheSize(this.nxJson.maxCacheSize)
+    this.nxJson.maxCacheSize !== undefined
+      ? parseMaxCacheSize(this.nxJson.maxCacheSize)
+      : getDefaultMaxCacheSize(cacheDir)
   );
 
   private remoteCache: RemoteCacheV2 | null;
@@ -150,6 +157,10 @@ export class DbCache {
     }
   }
 
+  getUsedCacheSpace() {
+    return this.cache.getCacheSize();
+  }
+
   private applyRemoteCacheResults(
     hash: string,
     res: NativeCacheResult,
@@ -603,13 +614,17 @@ function tryAndRetry<T>(fn: () => Promise<T>): Promise<T> {
  *
  * @param maxCacheSize Max cache size as specified in nx.json
  */
-export function parseMaxCacheSize(maxCacheSize: string): number | undefined {
+export function parseMaxCacheSize(
-  if (!maxCacheSize) {
+  maxCacheSize: string | number
+): number | undefined {
+  if (maxCacheSize === null || maxCacheSize === undefined) {
     return undefined;
   }
-  let regexResult = maxCacheSize.match(
+  let regexResult = maxCacheSize
-    /^(?<size>[\d|.]+)\s?((?<unit>[KMG]?B)?)$/
+    // Covers folks who accidentally specify as a number rather than a string
-  );
+    .toString()
+    // Match a number followed by an optional unit (KB, MB, GB), with optional whitespace between the number and unit
+    .match(/^(?<size>[\d|.]+)\s?((?<unit>[KMG]?B)?)$/);
   if (!regexResult) {
     throw new Error(
       `Invalid max cache size specified in nx.json: ${maxCacheSize}. Must be a number followed by an optional unit (KB, MB, GB)`
@@ -639,3 +654,14 @@ export function parseMaxCacheSize(maxCacheSize: string): number | undefined {
       return size;
   }
 }
+
+export function formatCacheSize(maxCacheSize: number, decimals = 2): string {
+  const exponents = ['B', 'KB', 'MB', 'GB'];
+  let exponent = 0;
+  let size = maxCacheSize;
+  while (size >= 1024 && exponent < exponents.length - 1) {
+    size /= 1024;
+    exponent++;
+  }
+  return `${size.toFixed(decimals)} ${exponents[exponent]}`;
+}