diff --git a/docs/generated/devkit/createNodesFromFiles.md b/docs/generated/devkit/createNodesFromFiles.md index 4c3b09737d..b9dda82a10 100644 --- a/docs/generated/devkit/createNodesFromFiles.md +++ b/docs/generated/devkit/createNodesFromFiles.md @@ -10,12 +10,12 @@ #### Parameters -| Name | Type | -| :------------ | :------------------------------------------------------------------------- | -| `createNodes` | [`CreateNodesFunction`](../../devkit/documents/CreateNodesFunction)\<`T`\> | -| `configFiles` | readonly `string`[] | -| `options` | `T` | -| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) | +| Name | Type | +| :------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `createNodes` | (`projectConfigurationFile`: `string`, `options`: `T`, `context`: [`CreateNodesContext`](../../devkit/documents/CreateNodesContext), `idx`: `number`) => [`CreateNodesResult`](../../devkit/documents/CreateNodesResult) \| `Promise`\<[`CreateNodesResult`](../../devkit/documents/CreateNodesResult)\> | +| `configFiles` | readonly `string`[] | +| `options` | `T` | +| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) | #### Returns diff --git a/docs/generated/devkit/isDaemonEnabled.md b/docs/generated/devkit/isDaemonEnabled.md index 9eb4dddd3c..b53aacae7c 100644 --- a/docs/generated/devkit/isDaemonEnabled.md +++ b/docs/generated/devkit/isDaemonEnabled.md @@ -1,6 +1,12 @@ # Function: isDaemonEnabled -▸ **isDaemonEnabled**(): `boolean` +▸ **isDaemonEnabled**(`nxJson?`): `boolean` + +#### Parameters + +| Name | Type | +| :------- | :----------------------------------------------------------------------------------------- | +| `nxJson` | [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\> | #### Returns diff --git a/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts b/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts index c4bfee85dd..93be89931f 100644 --- a/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts +++ b/packages/devkit/src/utils/calculate-hash-for-create-nodes.ts @@ -5,7 +5,11 @@ import { hashArray, } from 'nx/src/devkit-exports'; -import { hashObject, hashWithWorkspaceContext } from 'nx/src/devkit-internals'; +import { + hashMultiGlobWithWorkspaceContext, + hashObject, + hashWithWorkspaceContext, +} from 'nx/src/devkit-internals'; export async function calculateHashForCreateNodes( projectRoot: string, @@ -21,3 +25,28 @@ export async function calculateHashForCreateNodes( hashObject(options), ]); } + +export async function calculateHashesForCreateNodes( + projectRoots: string[], + options: object, + context: CreateNodesContext | CreateNodesContextV2, + additionalGlobs: string[][] = [] +): Promise { + if ( + additionalGlobs.length && + additionalGlobs.length !== projectRoots.length + ) { + throw new Error( + 'If additionalGlobs is provided, it must be the same length as projectRoots' + ); + } + return hashMultiGlobWithWorkspaceContext( + context.workspaceRoot, + projectRoots.map((projectRoot, idx) => [ + join(projectRoot, '**/*'), + ...(additionalGlobs.length ? additionalGlobs[idx] : []), + ]) + ).then((hashes) => { + return hashes.map((hash) => hashArray([hash, hashObject(options)])); + }); +} diff --git a/packages/eslint/src/plugins/plugin.ts b/packages/eslint/src/plugins/plugin.ts index 62398f4755..913df91680 100644 --- a/packages/eslint/src/plugins/plugin.ts +++ b/packages/eslint/src/plugins/plugin.ts @@ -11,7 +11,10 @@ import { TargetConfiguration, writeJsonFile, } from '@nx/devkit'; -import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; +import { + calculateHashesForCreateNodes, + calculateHashForCreateNodes, +} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; import { existsSync } from 'node:fs'; import { basename, dirname, join, normalize, sep } from 'node:path/posix'; import { hashObject } from 'nx/src/hasher/file-hasher'; @@ -186,10 +189,10 @@ const internalCreateNodesV2 = async ( configFilePath: string, options: EslintPluginOptions, context: CreateNodesContextV2, - eslintConfigFiles: string[], projectRootsByEslintRoots: Map, lintableFilesPerProjectRoot: Map, - projectsCache: Record + projectsCache: Record, + hashByRoot: Map ): Promise => { const configDir = dirname(configFilePath); @@ -201,19 +204,7 @@ const internalCreateNodesV2 = async ( const projects: CreateNodesResult['projects'] = {}; await Promise.all( projectRootsByEslintRoots.get(configDir).map(async (projectRoot) => { - const parentConfigs = eslintConfigFiles.filter((eslintConfig) => - isSubDir(projectRoot, dirname(eslintConfig)) - ); - const hash = await calculateHashForCreateNodes( - projectRoot, - options, - { - configFiles: eslintConfigFiles, - nxJsonConfiguration: context.nxJsonConfiguration, - workspaceRoot: context.workspaceRoot, - }, - [...parentConfigs, join(projectRoot, '.eslintignore')] - ); + const hash = hashByRoot.get(projectRoot); if (projectsCache[hash]) { // We can reuse the projects in the cache. @@ -280,6 +271,20 @@ export const createNodesV2: CreateNodesV2 = [ options, context ); + const hashes = await calculateHashesForCreateNodes( + projectRoots, + options, + context, + projectRoots.map((root) => { + const parentConfigs = eslintConfigFiles.filter((eslintConfig) => + isSubDir(root, dirname(eslintConfig)) + ); + return [...parentConfigs, join(root, '.eslintignore')]; + }) + ); + const hashByRoot = new Map( + projectRoots.map((r, i) => [r, hashes[i]]) + ); try { return await createNodesFromFiles( (configFile, options, context) => @@ -287,10 +292,10 @@ export const createNodesV2: CreateNodesV2 = [ configFile, options, context, - eslintConfigFiles, projectRootsByEslintRoots, lintableFilesPerProjectRoot, - targetsCache + targetsCache, + hashByRoot ), eslintConfigFiles, options, diff --git a/packages/jest/src/plugins/plugin.ts b/packages/jest/src/plugins/plugin.ts index 96027aa670..455fbdb4b9 100644 --- a/packages/jest/src/plugins/plugin.ts +++ b/packages/jest/src/plugins/plugin.ts @@ -1,6 +1,7 @@ import { CreateNodes, CreateNodesContext, + CreateNodesContextV2, createNodesFromFiles, CreateNodesV2, getPackageManagerCommand, @@ -13,7 +14,10 @@ import { TargetConfiguration, writeJsonFile, } from '@nx/devkit'; -import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; +import { + calculateHashesForCreateNodes, + calculateHashForCreateNodes, +} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; import { clearRequireCache, loadConfigFile, @@ -72,17 +76,70 @@ export const createNodesV2: CreateNodesV2 = [ // Cache jest preset(s) to avoid penalties of module load times. Most of jest configs will use the same preset. const presetCache: Record = {}; + const packageManagerWorkspacesGlob = combineGlobPatterns( + getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot) + ); + options = normalizeOptions(options); + + const { roots: projectRoots, configFiles: validConfigFiles } = + configFiles.reduce( + (acc, configFile) => { + const potentialRoot = dirname(configFile); + if ( + checkIfConfigFileShouldBeProject( + configFile, + potentialRoot, + packageManagerWorkspacesGlob, + context + ) + ) { + acc.roots.push(potentialRoot); + acc.configFiles.push(configFile); + } + return acc; + }, + { + roots: [], + configFiles: [], + } as { + roots: string[]; + configFiles: string[]; + } + ); + + const hashes = await calculateHashesForCreateNodes( + projectRoots, + options, + context + ); + try { return await createNodesFromFiles( - (configFile, options, context) => - createNodesInternal( - configFile, + async (configFilePath, options, context, idx) => { + const projectRoot = projectRoots[idx]; + const hash = hashes[idx]; + + targetsCache[hash] ??= await buildJestTargets( + configFilePath, + projectRoot, options, context, - targetsCache, presetCache - ), - configFiles, + ); + + const { targets, metadata } = targetsCache[hash]; + + return { + projects: { + [projectRoot]: { + root: projectRoot, + targets, + metadata, + }, + }, + }; + }, + validConfigFiles, options, context ); @@ -98,35 +155,63 @@ export const createNodesV2: CreateNodesV2 = [ */ export const createNodes: CreateNodes = [ jestConfigGlob, - (...args) => { + async (configFilePath, options, context) => { logger.warn( '`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will change to the createNodesV2 API.' ); - return createNodesInternal(...args, {}, {}); + const projectRoot = dirname(configFilePath); + + const packageManagerWorkspacesGlob = combineGlobPatterns( + getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot) + ); + + if ( + !checkIfConfigFileShouldBeProject( + configFilePath, + projectRoot, + packageManagerWorkspacesGlob, + context + ) + ) { + return {}; + } + + options = normalizeOptions(options); + + const { targets, metadata } = await buildJestTargets( + configFilePath, + projectRoot, + options, + context, + {} + ); + + return { + projects: { + [projectRoot]: { + root: projectRoot, + targets, + metadata, + }, + }, + }; }, ]; -async function createNodesInternal( +function checkIfConfigFileShouldBeProject( configFilePath: string, - options: JestPluginOptions, - context: CreateNodesContext, - targetsCache: Record, - presetCache: Record -) { - const projectRoot = dirname(configFilePath); - - const packageManagerWorkspacesGlob = combineGlobPatterns( - getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot) - ); - + projectRoot: string, + packageManagerWorkspacesGlob: string, + context: CreateNodesContext | CreateNodesContextV2 +): boolean { // Do not create a project if package.json and project.json isn't there. const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot)); if ( !siblingFiles.includes('package.json') && !siblingFiles.includes('project.json') ) { - return {}; + return false; } else if ( !siblingFiles.includes('project.json') && siblingFiles.includes('package.json') @@ -136,7 +221,7 @@ async function createNodesInternal( const isPackageJsonProject = minimatch(path, packageManagerWorkspacesGlob); if (!isPackageJsonProject) { - return {}; + return false; } } @@ -148,31 +233,9 @@ async function createNodesInternal( // The `getJestProjectsAsync` function uses the project graph, which leads to a // circular dependency. We can skip this since it's no intended to be used for // an Nx project. - return {}; + return false; } - - options = normalizeOptions(options); - - const hash = await calculateHashForCreateNodes(projectRoot, options, context); - targetsCache[hash] ??= await buildJestTargets( - configFilePath, - projectRoot, - options, - context, - presetCache - ); - - const { targets, metadata } = targetsCache[hash]; - - return { - projects: { - [projectRoot]: { - root: projectRoot, - targets, - metadata, - }, - }, - }; + return true; } async function buildJestTargets( diff --git a/packages/nx/src/config/nx-json.ts b/packages/nx/src/config/nx-json.ts index d02f78eb53..cc081f4aad 100644 --- a/packages/nx/src/config/nx-json.ts +++ b/packages/nx/src/config/nx-json.ts @@ -2,15 +2,16 @@ import { existsSync } from 'fs'; import { dirname, join } from 'path'; import type { ChangelogRenderOptions } from '../../release/changelog-renderer'; -import { readJsonFile } from '../utils/fileutils'; -import { PackageManager } from '../utils/package-manager'; -import { workspaceRoot } from '../utils/workspace-root'; -import { +import type { PackageManager } from '../utils/package-manager'; +import type { InputDefinition, TargetConfiguration, TargetDependencyConfig, } from './workspace-json-project-json'; +import { readJsonFile } from '../utils/fileutils'; +import { workspaceRoot } from '../utils/workspace-root'; + export type ImplicitDependencyEntry = { [key: string]: T | ImplicitJsonSubsetDependency; }; diff --git a/packages/nx/src/daemon/client/client.ts b/packages/nx/src/daemon/client/client.ts index 35a955efb7..345c46890e 100644 --- a/packages/nx/src/daemon/client/client.ts +++ b/packages/nx/src/daemon/client/client.ts @@ -16,7 +16,6 @@ import { getFullOsSocketPath, killSocketOrPath } from '../socket-utils'; import { DAEMON_DIR_FOR_CURRENT_WORKSPACE, DAEMON_OUTPUT_LOG_FILE, - isDaemonDisabled, removeSocketDir, } from '../tmp-dir'; import { FileData, ProjectGraph } from '../../config/project-graph'; @@ -50,7 +49,12 @@ import { GET_FILES_IN_DIRECTORY, HandleGetFilesInDirectoryMessage, } from '../message-types/get-files-in-directory'; -import { HASH_GLOB, HandleHashGlobMessage } from '../message-types/hash-glob'; +import { + HASH_GLOB, + HASH_MULTI_GLOB, + HandleHashGlobMessage, + HandleHashMultiGlobMessage, +} from '../message-types/hash-glob'; import { GET_ESTIMATED_TASK_TIMINGS, GET_FLAKY_TASKS, @@ -91,6 +95,7 @@ import { POST_TASKS_EXECUTION, PRE_TASKS_EXECUTION, } from '../message-types/run-tasks-execution-hooks'; +import { isDaemonEnabled } from './enabled'; const DAEMON_ENV_SETTINGS = { NX_PROJECT_GLOB_CACHE: 'false', @@ -136,48 +141,7 @@ export class DaemonClient { private _err: FileHandle = null; enabled() { - if (this._enabled === undefined) { - const useDaemonProcessOption = this.nxJson?.useDaemonProcess; - const env = process.env.NX_DAEMON; - - // env takes precedence - // option=true,env=false => no daemon - // option=false,env=undefined => no daemon - // option=false,env=false => no daemon - - // option=undefined,env=undefined => daemon - // option=true,env=true => daemon - // option=false,env=true => daemon - - // CI=true,env=undefined => no daemon - // CI=true,env=false => no daemon - // CI=true,env=true => daemon - - // docker=true,env=undefined => no daemon - // docker=true,env=false => no daemon - // docker=true,env=true => daemon - // WASM => no daemon because file watching does not work - if ( - ((isCI() || isDocker()) && env !== 'true') || - isDaemonDisabled() || - nxJsonIsNotPresent() || - (useDaemonProcessOption === undefined && env === 'false') || - (useDaemonProcessOption === true && env === 'false') || - (useDaemonProcessOption === false && env === undefined) || - (useDaemonProcessOption === false && env === 'false') - ) { - this._enabled = false; - } else if (IS_WASM) { - output.warn({ - title: - 'The Nx Daemon is unsupported in WebAssembly environments. Some things may be slower than or not function as expected.', - }); - this._enabled = false; - } else { - this._enabled = true; - } - } - return this._enabled; + return isDaemonEnabled(this.nxJson); } reset() { @@ -385,6 +349,14 @@ export class DaemonClient { return this.sendToDaemonViaQueue(message); } + hashMultiGlob(globGroups: string[][]): Promise { + const message: HandleHashMultiGlobMessage = { + type: HASH_MULTI_GLOB, + globGroups: globGroups, + }; + return this.sendToDaemonViaQueue(message); + } + getFlakyTasks(hashes: string[]): Promise { const message: HandleGetFlakyTasks = { type: GET_FLAKY_TASKS, @@ -705,27 +677,6 @@ export class DaemonClient { export const daemonClient = new DaemonClient(); -export function isDaemonEnabled() { - return daemonClient.enabled(); -} - -function isDocker() { - try { - statSync('/.dockerenv'); - return true; - } catch { - try { - return readFileSync('/proc/self/cgroup', 'utf8')?.includes('docker'); - } catch {} - - return false; - } -} - -function nxJsonIsNotPresent() { - return !hasNxJson(workspaceRoot); -} - function daemonProcessException(message: string) { try { let log = readFileSync(DAEMON_OUTPUT_LOG_FILE).toString().split('\n'); diff --git a/packages/nx/src/daemon/client/enabled.ts b/packages/nx/src/daemon/client/enabled.ts new file mode 100644 index 0000000000..05084cc866 --- /dev/null +++ b/packages/nx/src/daemon/client/enabled.ts @@ -0,0 +1,79 @@ +import { + hasNxJson, + readNxJson, + type NxJsonConfiguration, +} from '../../config/nx-json'; + +import { readFileSync, statSync } from 'node:fs'; + +import { isCI } from '../../utils/is-ci'; +import { workspaceRoot } from '../../utils/workspace-root'; +import { isDaemonDisabled } from '../tmp-dir'; + +let _enabled: boolean | undefined; + +export function isDaemonEnabled(nxJson: NxJsonConfiguration = readNxJson()) { + if (_enabled === undefined) { + const useDaemonProcessOption = nxJson?.useDaemonProcess; + const env = process.env.NX_DAEMON; + + // env takes precedence + // option=true,env=false => no daemon + // option=false,env=undefined => no daemon + // option=false,env=false => no daemon + + // option=undefined,env=undefined => daemon + // option=true,env=true => daemon + // option=false,env=true => daemon + + // CI=true,env=undefined => no daemon + // CI=true,env=false => no daemon + // CI=true,env=true => daemon + + // docker=true,env=undefined => no daemon + // docker=true,env=false => no daemon + // docker=true,env=true => daemon + // WASM => no daemon because file watching does not work + if ( + ((isCI() || isDocker()) && env !== 'true') || + isDaemonDisabled() || + nxJsonIsNotPresent() || + (useDaemonProcessOption === undefined && env === 'false') || + (useDaemonProcessOption === true && env === 'false') || + (useDaemonProcessOption === false && env === undefined) || + (useDaemonProcessOption === false && env === 'false') + ) { + _enabled = false; + } else if ( + (require('../../native') as typeof import('../../native')).IS_WASM + ) { + ( + require('../../utils/output') as typeof import('../../utils/output') + ).output.warn({ + title: + 'The Nx Daemon is unsupported in WebAssembly environments. Some things may be slower than or not function as expected.', + }); + _enabled = false; + } else { + _enabled = true; + } + } + return _enabled; +} + +function isDocker() { + try { + statSync('/.dockerenv'); + return true; + } catch { + try { + return readFileSync('/proc/self/cgroup', 'utf8')?.includes('docker'); + } catch {} + + return false; + } +} + +function nxJsonIsNotPresent() { + return !hasNxJson(workspaceRoot); +} diff --git a/packages/nx/src/daemon/message-types/hash-glob.ts b/packages/nx/src/daemon/message-types/hash-glob.ts index 9bec1f566c..e097fae4ba 100644 --- a/packages/nx/src/daemon/message-types/hash-glob.ts +++ b/packages/nx/src/daemon/message-types/hash-glob.ts @@ -16,3 +16,21 @@ export function isHandleHashGlobMessage( message['type'] === HASH_GLOB ); } + +export const HASH_MULTI_GLOB = 'HASH_MULTI_GLOB' as const; + +export type HandleHashMultiGlobMessage = { + type: typeof HASH_MULTI_GLOB; + globGroups: string[][]; +}; + +export function isHandleHashMultiGlobMessage( + message: unknown +): message is HandleHashMultiGlobMessage { + return ( + typeof message === 'object' && + message !== null && + 'type' in message && + message['type'] === HASH_MULTI_GLOB + ); +} diff --git a/packages/nx/src/daemon/server/handle-hash-glob.ts b/packages/nx/src/daemon/server/handle-hash-glob.ts index 1c283005f3..3703d3e781 100644 --- a/packages/nx/src/daemon/server/handle-hash-glob.ts +++ b/packages/nx/src/daemon/server/handle-hash-glob.ts @@ -1,5 +1,8 @@ import { workspaceRoot } from '../../utils/workspace-root'; -import { hashWithWorkspaceContext } from '../../utils/workspace-context'; +import { + hashMultiGlobWithWorkspaceContext, + hashWithWorkspaceContext, +} from '../../utils/workspace-context'; import { HandlerResult } from './server'; export async function handleHashGlob( @@ -12,3 +15,13 @@ export async function handleHashGlob( description: 'handleHashGlob', }; } + +export async function handleHashMultiGlob( + globs: string[][] +): Promise { + const files = await hashMultiGlobWithWorkspaceContext(workspaceRoot, globs); + return { + response: JSON.stringify(files), + description: 'handleHashMultiGlob', + }; +} diff --git a/packages/nx/src/daemon/server/server.ts b/packages/nx/src/daemon/server/server.ts index f47937ad27..de0a5c9a32 100644 --- a/packages/nx/src/daemon/server/server.ts +++ b/packages/nx/src/daemon/server/server.ts @@ -77,8 +77,12 @@ import { isHandleGetFilesInDirectoryMessage, } from '../message-types/get-files-in-directory'; import { handleGetFilesInDirectory } from './handle-get-files-in-directory'; -import { HASH_GLOB, isHandleHashGlobMessage } from '../message-types/hash-glob'; -import { handleHashGlob } from './handle-hash-glob'; +import { + HASH_GLOB, + isHandleHashGlobMessage, + isHandleHashMultiGlobMessage, +} from '../message-types/hash-glob'; +import { handleHashGlob, handleHashMultiGlob } from './handle-hash-glob'; import { GET_ESTIMATED_TASK_TIMINGS, GET_FLAKY_TASKS, @@ -264,6 +268,10 @@ async function handleMessage(socket, data: string) { await handleResult(socket, HASH_GLOB, () => handleHashGlob(payload.globs, payload.exclude) ); + } else if (isHandleHashMultiGlobMessage(payload)) { + await handleResult(socket, HASH_GLOB, () => + handleHashMultiGlob(payload.globGroups) + ); } else if (isHandleGetFlakyTasksMessage(payload)) { await handleResult(socket, GET_FLAKY_TASKS, () => handleGetFlakyTasks(payload.hashes) diff --git a/packages/nx/src/devkit-exports.ts b/packages/nx/src/devkit-exports.ts index 44151f6d88..082bcbe03d 100644 --- a/packages/nx/src/devkit-exports.ts +++ b/packages/nx/src/devkit-exports.ts @@ -260,4 +260,4 @@ export { cacheDir } from './utils/cache-directory'; */ export { createProjectFileMapUsingProjectGraph } from './project-graph/file-map-utils'; -export { isDaemonEnabled } from './daemon/client/client'; +export { isDaemonEnabled } from './daemon/client/enabled'; diff --git a/packages/nx/src/devkit-internals.ts b/packages/nx/src/devkit-internals.ts index ceda5e4fc9..61d2ad470f 100644 --- a/packages/nx/src/devkit-internals.ts +++ b/packages/nx/src/devkit-internals.ts @@ -20,7 +20,10 @@ export { stripIndent } from './utils/logger'; export { readModulePackageJson } from './utils/package-json'; export { splitByColons } from './utils/split-target'; export { hashObject } from './hasher/file-hasher'; -export { hashWithWorkspaceContext } from './utils/workspace-context'; +export { + hashWithWorkspaceContext, + hashMultiGlobWithWorkspaceContext, +} from './utils/workspace-context'; export { createProjectRootMappingsFromProjectConfigurations, findProjectForPath, diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index 2e8e7da4ca..6f6b9c5973 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -100,6 +100,7 @@ export declare class WorkspaceContext { * as the input globs. */ multiGlob(globs: Array, exclude?: Array | undefined | null): Array> + hashFilesMatchingGlobs(globGroups: Array>): Array hashFilesMatchingGlob(globs: Array, exclude?: Array | undefined | null): string incrementalUpdate(updatedFiles: Array, deletedFiles: Array): Record updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject, globalFiles: ExternalObject>, updatedFiles: Record, deletedFiles: Array): UpdatedWorkspaceFiles diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index 6dc5236781..90d82a601e 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -253,6 +253,29 @@ impl WorkspaceContext { .collect() } + #[napi] + pub fn hash_files_matching_globs( + &self, + glob_groups: Vec>, + ) -> napi::Result> { + let files = &self.all_file_data(); + let hashes = glob_groups + .into_iter() + .map(|globs| { + let globbed_files = + config_files::glob_files(files, globs, None)?.collect::>(); + let mut hasher = xxh3::Xxh3::new(); + for file in globbed_files { + hasher.update(file.file.as_bytes()); + hasher.update(file.hash.as_bytes()); + } + Ok(hasher.digest().to_string()) + }) + .collect::>>()?; + + Ok(hashes) + } + #[napi] pub fn hash_files_matching_glob( &self, diff --git a/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts b/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts index f5e32dd07e..ff1385003a 100644 --- a/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts +++ b/packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts @@ -1,3 +1,7 @@ +import { performance } from 'node:perf_hooks'; + +performance.mark(`plugin worker ${process.pid} code loading -- start`); + import { consumeMessage, isPluginWorkerMessage } from './messaging'; import { createSerializableError } from '../../../utils/serializable-error'; import { consumeMessagesFromSocket } from '../../../utils/consume-messages-from-socket'; @@ -10,6 +14,13 @@ if (process.env.NX_PERF_LOGGING === 'true') { require('../../../utils/perf-logging'); } +performance.mark(`plugin worker ${process.pid} code loading -- end`); +performance.measure( + `plugin worker ${process.pid} code loading`, + `plugin worker ${process.pid} code loading -- start`, + `plugin worker ${process.pid} code loading -- end` +); + global.NX_GRAPH_CREATION = true; global.NX_PLUGIN_WORKER = true; let connected = false; diff --git a/packages/nx/src/project-graph/plugins/loaded-nx-plugin.ts b/packages/nx/src/project-graph/plugins/loaded-nx-plugin.ts index 150be0d688..7cd83846e6 100644 --- a/packages/nx/src/project-graph/plugins/loaded-nx-plugin.ts +++ b/packages/nx/src/project-graph/plugins/loaded-nx-plugin.ts @@ -1,5 +1,5 @@ import type { ProjectGraph } from '../../config/project-graph'; -import type { PluginConfiguration } from '../../config/nx-json'; +import { readNxJson, type PluginConfiguration } from '../../config/nx-json'; import { AggregateCreateNodesError, isAggregateCreateNodesError, @@ -17,7 +17,7 @@ import type { } from './public-api'; import { createNodesFromFiles } from './utils'; import { isIsolationEnabled } from './isolation/enabled'; -import { isDaemonEnabled } from '../../daemon/client/client'; +import { isDaemonEnabled } from '../../daemon/client/enabled'; export class LoadedNxPlugin { index?: number; @@ -123,7 +123,10 @@ export class LoadedNxPlugin { this.preTasksExecution = async (context: PreTasksExecutionContext) => { const updates = {}; let originalEnv = process.env; - if (isIsolationEnabled() || isDaemonEnabled()) { + if ( + isIsolationEnabled() || + isDaemonEnabled(context.nxJsonConfiguration) + ) { process.env = new Proxy(originalEnv, { set: (target, key: string, value) => { target[key] = value; diff --git a/packages/nx/src/project-graph/plugins/tasks-execution-hooks.ts b/packages/nx/src/project-graph/plugins/tasks-execution-hooks.ts index 261baeb3a6..9a2e68adfd 100644 --- a/packages/nx/src/project-graph/plugins/tasks-execution-hooks.ts +++ b/packages/nx/src/project-graph/plugins/tasks-execution-hooks.ts @@ -4,12 +4,13 @@ import type { } from './public-api'; import { getPlugins } from './get-plugins'; import { isOnDaemon } from '../../daemon/is-on-daemon'; -import { daemonClient, isDaemonEnabled } from '../../daemon/client/client'; +import { daemonClient } from '../../daemon/client/client'; +import { isDaemonEnabled } from '../../daemon/client/enabled'; export async function runPreTasksExecution( pluginContext: PreTasksExecutionContext ) { - if (isOnDaemon() || !isDaemonEnabled()) { + if (isOnDaemon() || !isDaemonEnabled(pluginContext.nxJsonConfiguration)) { performance.mark(`preTasksExecution:start`); const plugins = await getPlugins(pluginContext.workspaceRoot); const envs = await Promise.all( @@ -30,7 +31,7 @@ export async function runPreTasksExecution( }) ); - if (!isDaemonEnabled()) { + if (!isDaemonEnabled(pluginContext.nxJsonConfiguration)) { applyProcessEnvs(envs); } performance.mark(`preTasksExecution:end`); @@ -57,7 +58,7 @@ function applyProcessEnvs(envs: NodeJS.ProcessEnv[]) { export async function runPostTasksExecution( context: PostTasksExecutionContext ) { - if (isOnDaemon() || !isDaemonEnabled()) { + if (isOnDaemon() || !isDaemonEnabled(context.nxJsonConfiguration)) { performance.mark(`postTasksExecution:start`); const plugins = await getPlugins(); await Promise.all( diff --git a/packages/nx/src/project-graph/plugins/utils.ts b/packages/nx/src/project-graph/plugins/utils.ts index cb5600a856..f209c4a4f6 100644 --- a/packages/nx/src/project-graph/plugins/utils.ts +++ b/packages/nx/src/project-graph/plugins/utils.ts @@ -1,11 +1,16 @@ import { + CreateNodesContext, CreateNodesContextV2, - CreateNodesFunction, CreateNodesResult, } from './public-api'; import { AggregateCreateNodesError } from '../error-types'; export async function createNodesFromFiles( - createNodes: CreateNodesFunction, + createNodes: ( + projectConfigurationFile: string, + options: T | undefined, + context: CreateNodesContext, + idx: number + ) => CreateNodesResult | Promise, configFiles: readonly string[], options: T, context: CreateNodesContextV2 @@ -14,12 +19,17 @@ export async function createNodesFromFiles( const errors: Array<[file: string, error: Error]> = []; await Promise.all( - configFiles.map(async (file) => { + configFiles.map(async (file, idx) => { try { - const value = await createNodes(file, options, { - ...context, - configFiles, - }); + const value = await createNodes( + file, + options, + { + ...context, + configFiles, + }, + idx + ); if (value) { results.push([file, value] as const); } diff --git a/packages/nx/src/utils/workspace-context.ts b/packages/nx/src/utils/workspace-context.ts index 14193cb07c..6cf6ef5fbc 100644 --- a/packages/nx/src/utils/workspace-context.ts +++ b/packages/nx/src/utils/workspace-context.ts @@ -86,6 +86,17 @@ export async function hashWithWorkspaceContext( return daemonClient.hashGlob(globs, exclude); } +export async function hashMultiGlobWithWorkspaceContext( + workspaceRoot: string, + globGroups: string[][] +) { + if (isOnDaemon() || !daemonClient.enabled()) { + ensureContextAvailable(workspaceRoot); + return workspaceContext.hashFilesMatchingGlobs(globGroups); + } + return daemonClient.hashMultiGlob(globGroups); +} + export async function updateContextWithChangedFiles( workspaceRoot: string, createdFiles: string[], diff --git a/packages/rspack/src/plugins/utils/plugins/rspack-nx-build-coordination-plugin.ts b/packages/rspack/src/plugins/utils/plugins/rspack-nx-build-coordination-plugin.ts index fb13e7589f..4b73808d01 100644 --- a/packages/rspack/src/plugins/utils/plugins/rspack-nx-build-coordination-plugin.ts +++ b/packages/rspack/src/plugins/utils/plugins/rspack-nx-build-coordination-plugin.ts @@ -1,6 +1,7 @@ import { exec } from 'child_process'; import type { Compiler } from '@rspack/core'; -import { daemonClient, isDaemonEnabled } from 'nx/src/daemon/client/client'; +import { daemonClient } from 'nx/src/daemon/client/client'; +import { isDaemonEnabled } from 'nx/src/daemon/client/enabled'; import { BatchFunctionRunner } from 'nx/src/command-line/watch/watch'; import { output } from 'nx/src/utils/output'; diff --git a/packages/vite/src/plugins/plugin.ts b/packages/vite/src/plugins/plugin.ts index 685388e6e9..9b3973b65a 100644 --- a/packages/vite/src/plugins/plugin.ts +++ b/packages/vite/src/plugins/plugin.ts @@ -2,6 +2,7 @@ import { CreateDependencies, CreateNodes, CreateNodesContext, + CreateNodesContextV2, createNodesFromFiles, CreateNodesV2, detectPackageManager, @@ -16,7 +17,10 @@ import { import { dirname, isAbsolute, join, relative } from 'path'; import { getNamedInputs } from '@nx/devkit/src/utils/get-named-inputs'; import { existsSync, readdirSync } from 'fs'; -import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; +import { + calculateHashesForCreateNodes, + calculateHashForCreateNodes, +} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; import { workspaceDataDirectory } from 'nx/src/utils/cache-directory'; import { getLockFileName } from '@nx/js'; import { loadViteDynamicImport } from '../utils/executor-utils'; @@ -42,7 +46,10 @@ export interface VitePluginOptions { buildDepsTargetName?: string; } -type ViteTargets = Pick; +type ViteTargets = Pick< + ProjectConfiguration, + 'targets' | 'metadata' | 'projectType' +>; function readTargetsCache(cachePath: string): Record { return process.env.NX_CACHE_PROJECT_GRAPH !== 'false' && existsSync(cachePath) @@ -67,20 +74,88 @@ export const createNodesV2: CreateNodesV2 = [ viteVitestConfigGlob, async (configFilePaths, options, context) => { const optionsHash = hashObject(options); + const normalizedOptions = normalizeOptions(options); const cachePath = join(workspaceDataDirectory, `vite-${optionsHash}.hash`); const targetsCache = readTargetsCache(cachePath); const isUsingTsSolutionSetup = _isUsingTsSolutionSetup(); + + const { roots: projectRoots, configFiles: validConfigFiles } = + configFilePaths.reduce( + (acc, configFile) => { + const potentialRoot = dirname(configFile); + if (checkIfConfigFileShouldBeProject(potentialRoot, context)) { + acc.roots.push(potentialRoot); + acc.configFiles.push(configFile); + } + return acc; + }, + { + roots: [], + configFiles: [], + } as { + roots: string[]; + configFiles: string[]; + } + ); + + const lockfile = getLockFileName( + detectPackageManager(context.workspaceRoot) + ); + const hashes = await calculateHashesForCreateNodes( + projectRoots, + { ...normalizedOptions, isUsingTsSolutionSetup }, + context, + projectRoots.map((r) => [lockfile]) + ); + try { return await createNodesFromFiles( - (configFile, options, context) => - createNodesInternal( - configFile, - options, - context, - targetsCache, - isUsingTsSolutionSetup - ), - configFilePaths, + async (configFile, _, context, idx) => { + const projectRoot = dirname(configFile); + // Do not create a project if package.json and project.json isn't there. + const siblingFiles = readdirSync( + join(context.workspaceRoot, projectRoot) + ); + + const tsConfigFiles = + siblingFiles.filter((p) => + minimatch(p, 'tsconfig*{.json,.*.json}') + ) ?? []; + + // results from vitest.config.js will be different from results of vite.config.js + // but the hash will be the same because it is based on the files under the project root. + // Adding the config file path to the hash ensures that the final hash value is different + // for different config files. + const hash = hashes[idx] + configFile; + const { projectType, metadata, targets } = (targetsCache[hash] ??= + await buildViteTargets( + configFile, + projectRoot, + normalizedOptions, + tsConfigFiles, + isUsingTsSolutionSetup, + context + )); + + const project: ProjectConfiguration = { + root: projectRoot, + targets, + metadata, + }; + + // If project is buildable, then the project type. + // If it is not buildable, then leave it to other plugins/project.json to set the project type. + if (project.targets[normalizedOptions.buildTargetName]) { + project.projectType = projectType; + } + + return { + projects: { + [projectRoot]: project, + }, + }; + }, + validConfigFiles, options, context ); @@ -96,78 +171,52 @@ export const createNodes: CreateNodes = [ logger.warn( '`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will change to the createNodesV2 API.' ); - return createNodesInternal( + const projectRoot = dirname(configFilePath); + // Do not create a project if package.json and project.json isn't there. + const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot)); + if ( + !siblingFiles.includes('package.json') && + !siblingFiles.includes('project.json') + ) { + return {}; + } + + const tsConfigFiles = + siblingFiles.filter((p) => minimatch(p, 'tsconfig*{.json,.*.json}')) ?? + []; + + const normalizedOptions = normalizeOptions(options); + + const isUsingTsSolutionSetup = _isUsingTsSolutionSetup(); + + const { projectType, metadata, targets } = await buildViteTargets( configFilePath, - options, - context, - {}, - _isUsingTsSolutionSetup() + projectRoot, + normalizedOptions, + tsConfigFiles, + isUsingTsSolutionSetup, + context ); + const project: ProjectConfiguration = { + root: projectRoot, + targets, + metadata, + }; + + // If project is buildable, then the project type. + // If it is not buildable, then leave it to other plugins/project.json to set the project type. + if (project.targets[normalizedOptions.buildTargetName]) { + project.projectType = projectType; + } + + return { + projects: { + [projectRoot]: project, + }, + }; }, ]; -async function createNodesInternal( - configFilePath: string, - options: VitePluginOptions, - context: CreateNodesContext, - targetsCache: Record, - isUsingTsSolutionSetup: boolean -) { - const projectRoot = dirname(configFilePath); - // Do not create a project if package.json and project.json isn't there. - const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot)); - if ( - !siblingFiles.includes('package.json') && - !siblingFiles.includes('project.json') - ) { - return {}; - } - - const tsConfigFiles = - siblingFiles.filter((p) => minimatch(p, 'tsconfig*{.json,.*.json}')) ?? []; - - const normalizedOptions = normalizeOptions(options); - - // We do not want to alter how the hash is calculated, so appending the config file path to the hash - // to prevent vite/vitest files overwriting the target cache created by the other - const hash = - (await calculateHashForCreateNodes( - projectRoot, - { ...normalizedOptions, isUsingTsSolutionSetup }, - context, - [getLockFileName(detectPackageManager(context.workspaceRoot))] - )) + configFilePath; - - const { isLibrary, ...viteTargets } = await buildViteTargets( - configFilePath, - projectRoot, - normalizedOptions, - tsConfigFiles, - isUsingTsSolutionSetup, - context - ); - targetsCache[hash] ??= viteTargets; - - const { targets, metadata } = targetsCache[hash]; - const project: ProjectConfiguration = { - root: projectRoot, - targets, - metadata, - }; - - // If project is buildable, then the project type. - // If it is not buildable, then leave it to other plugins/project.json to set the project type. - if (project.targets[normalizedOptions.buildTargetName]) { - project.projectType = isLibrary ? 'library' : 'application'; - } - - return { - projects: { - [projectRoot]: project, - }, - }; -} - async function buildViteTargets( configFilePath: string, projectRoot: string, @@ -175,7 +224,7 @@ async function buildViteTargets( tsConfigFiles: string[], isUsingTsSolutionSetup: boolean, context: CreateNodesContext -): Promise { +): Promise { const absoluteConfigFilePath = joinPathFragments( context.workspaceRoot, configFilePath @@ -304,7 +353,11 @@ async function buildViteTargets( ); const metadata = {}; - return { targets, metadata, isLibrary: Boolean(viteBuildConfig.build?.lib) }; + return { + targets, + metadata, + projectType: viteBuildConfig.build?.lib ? 'library' : 'application', + }; } async function buildTarget( @@ -538,3 +591,19 @@ function normalizeOptions(options: VitePluginOptions): VitePluginOptions { options.typecheckTargetName ??= 'typecheck'; return options; } + +function checkIfConfigFileShouldBeProject( + projectRoot: string, + context: CreateNodesContext | CreateNodesContextV2 +): boolean { + // Do not create a project if package.json and project.json isn't there. + const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot)); + if ( + !siblingFiles.includes('package.json') && + !siblingFiles.includes('project.json') + ) { + return false; + } + + return true; +} diff --git a/packages/webpack/src/plugins/webpack-nx-build-coordination-plugin.ts b/packages/webpack/src/plugins/webpack-nx-build-coordination-plugin.ts index 1fe05acfe2..ab799a771d 100644 --- a/packages/webpack/src/plugins/webpack-nx-build-coordination-plugin.ts +++ b/packages/webpack/src/plugins/webpack-nx-build-coordination-plugin.ts @@ -1,8 +1,9 @@ import { exec } from 'child_process'; import type { Compiler } from 'webpack'; -import { daemonClient, isDaemonEnabled } from 'nx/src/daemon/client/client'; +import { daemonClient } from 'nx/src/daemon/client/client'; import { BatchFunctionRunner } from 'nx/src/command-line/watch/watch'; import { output } from 'nx/src/utils/output'; +import { isDaemonEnabled } from 'nx/src/daemon/client/enabled'; type PluginOptions = { skipInitialBuild?: boolean;