miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

feat(core): add multi hash fn (#29935)

Browse Source 
Adds function to compute multiple glob hashes in native code at the same time, greatly speeding up certain plugin performance.
This commit is contained in:
Craigory Coppola 2025-02-13 14:21:54 -05:00 committed by GitHub
parent 2ebdd2e5a2
commit c2e89f87b5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 550 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/generated/devkit/createNodesFromFiles.md
View File

@ -10,12 +10,12 @@
#### Parameters
| Name | Type |
| :------------ | :------------------------------------------------------------------------- |
| `createNodes` | [`CreateNodesFunction`](../../devkit/documents/CreateNodesFunction)\<`T`\> |
| `configFiles` | readonly `string`[] |
| `options` | `T` |
| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) |
| Name | Type |
| :------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `createNodes` | (`projectConfigurationFile`: `string`, `options`: `T`, `context`: [`CreateNodesContext`](../../devkit/documents/CreateNodesContext), `idx`: `number`) => [`CreateNodesResult`](../../devkit/documents/CreateNodesResult) \| `Promise`\<[`CreateNodesResult`](../../devkit/documents/CreateNodesResult)\> |
| `configFiles` | readonly `string`[] |
| `options` | `T` |
| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) |
#### Returns

8
docs/generated/devkit/isDaemonEnabled.md
View File

@ -1,6 +1,12 @@
# Function: isDaemonEnabled
▸ **isDaemonEnabled**(): `boolean`
▸ **isDaemonEnabled**(`nxJson?`): `boolean`
#### Parameters
| Name | Type |
| :------- | :----------------------------------------------------------------------------------------- |
| `nxJson` | [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\> |
#### Returns

31
packages/devkit/src/utils/calculate-hash-for-create-nodes.ts
View File

@ -5,7 +5,11 @@ import {
hashArray,
} from 'nx/src/devkit-exports';
import { hashObject, hashWithWorkspaceContext } from 'nx/src/devkit-internals';
import {
hashMultiGlobWithWorkspaceContext,
hashObject,
hashWithWorkspaceContext,
} from 'nx/src/devkit-internals';
export async function calculateHashForCreateNodes(
projectRoot: string,
@ -21,3 +25,28 @@ export async function calculateHashForCreateNodes(
hashObject(options),
]);
}
export async function calculateHashesForCreateNodes(
projectRoots: string[],
options: object,
context: CreateNodesContext | CreateNodesContextV2,
additionalGlobs: string[][] = []
): Promise<string[]> {
if (
additionalGlobs.length &&
additionalGlobs.length !== projectRoots.length
) {
throw new Error(
'If additionalGlobs is provided, it must be the same length as projectRoots'
);
}
return hashMultiGlobWithWorkspaceContext(
context.workspaceRoot,
projectRoots.map((projectRoot, idx) => [
join(projectRoot, '**/*'),
...(additionalGlobs.length ? additionalGlobs[idx] : []),
])
).then((hashes) => {
return hashes.map((hash) => hashArray([hash, hashObject(options)]));
});
}

41
packages/eslint/src/plugins/plugin.ts
View File

@ -11,7 +11,10 @@ import {
TargetConfiguration,
writeJsonFile,
} from '@nx/devkit';
import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import {
calculateHashesForCreateNodes,
calculateHashForCreateNodes,
} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import { existsSync } from 'node:fs';
import { basename, dirname, join, normalize, sep } from 'node:path/posix';
import { hashObject } from 'nx/src/hasher/file-hasher';
@ -186,10 +189,10 @@ const internalCreateNodesV2 = async (
configFilePath: string,
options: EslintPluginOptions,
context: CreateNodesContextV2,
eslintConfigFiles: string[],
projectRootsByEslintRoots: Map<string, string[]>,
lintableFilesPerProjectRoot: Map<string, string[]>,
projectsCache: Record<string, CreateNodesResult['projects']>
projectsCache: Record<string, CreateNodesResult['projects']>,
hashByRoot: Map<string, string>
): Promise<CreateNodesResult> => {
const configDir = dirname(configFilePath);
@ -201,19 +204,7 @@ const internalCreateNodesV2 = async (
const projects: CreateNodesResult['projects'] = {};
await Promise.all(
projectRootsByEslintRoots.get(configDir).map(async (projectRoot) => {
const parentConfigs = eslintConfigFiles.filter((eslintConfig) =>
isSubDir(projectRoot, dirname(eslintConfig))
);
const hash = await calculateHashForCreateNodes(
projectRoot,
options,
{
configFiles: eslintConfigFiles,
nxJsonConfiguration: context.nxJsonConfiguration,
workspaceRoot: context.workspaceRoot,
},
[...parentConfigs, join(projectRoot, '.eslintignore')]
);
const hash = hashByRoot.get(projectRoot);
if (projectsCache[hash]) {
// We can reuse the projects in the cache.
@ -280,6 +271,20 @@ export const createNodesV2: CreateNodesV2<EslintPluginOptions> = [
options,
context
);
const hashes = await calculateHashesForCreateNodes(
projectRoots,
options,
context,
projectRoots.map((root) => {
const parentConfigs = eslintConfigFiles.filter((eslintConfig) =>
isSubDir(root, dirname(eslintConfig))
);
return [...parentConfigs, join(root, '.eslintignore')];
})
);
const hashByRoot = new Map<string, string>(
projectRoots.map((r, i) => [r, hashes[i]])
);
try {
return await createNodesFromFiles(
(configFile, options, context) =>
@ -287,10 +292,10 @@ export const createNodesV2: CreateNodesV2<EslintPluginOptions> = [
configFile,
options,
context,
eslintConfigFiles,
projectRootsByEslintRoots,
lintableFilesPerProjectRoot,
targetsCache
targetsCache,
hashByRoot
),
eslintConfigFiles,
options,

157
packages/jest/src/plugins/plugin.ts
View File

@ -1,6 +1,7 @@
import {
CreateNodes,
CreateNodesContext,
CreateNodesContextV2,
createNodesFromFiles,
CreateNodesV2,
getPackageManagerCommand,
@ -13,7 +14,10 @@ import {
TargetConfiguration,
writeJsonFile,
} from '@nx/devkit';
import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import {
calculateHashesForCreateNodes,
calculateHashForCreateNodes,
} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import {
clearRequireCache,
loadConfigFile,
@ -72,17 +76,70 @@ export const createNodesV2: CreateNodesV2<JestPluginOptions> = [
// Cache jest preset(s) to avoid penalties of module load times. Most of jest configs will use the same preset.
const presetCache: Record<string, unknown> = {};
const packageManagerWorkspacesGlob = combineGlobPatterns(
getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot)
);
options = normalizeOptions(options);
const { roots: projectRoots, configFiles: validConfigFiles } =
configFiles.reduce(
(acc, configFile) => {
const potentialRoot = dirname(configFile);
if (
checkIfConfigFileShouldBeProject(
configFile,
potentialRoot,
packageManagerWorkspacesGlob,
context
)
) {
acc.roots.push(potentialRoot);
acc.configFiles.push(configFile);
}
return acc;
},
{
roots: [],
configFiles: [],
} as {
roots: string[];
configFiles: string[];
}
);
const hashes = await calculateHashesForCreateNodes(
projectRoots,
options,
context
);
try {
return await createNodesFromFiles(
(configFile, options, context) =>
createNodesInternal(
configFile,
async (configFilePath, options, context, idx) => {
const projectRoot = projectRoots[idx];
const hash = hashes[idx];
targetsCache[hash] ??= await buildJestTargets(
configFilePath,
projectRoot,
options,
context,
targetsCache,
presetCache
),
configFiles,
);
const { targets, metadata } = targetsCache[hash];
return {
projects: {
[projectRoot]: {
root: projectRoot,
targets,
metadata,
},
},
};
},
validConfigFiles,
options,
context
);
@ -98,35 +155,63 @@ export const createNodesV2: CreateNodesV2<JestPluginOptions> = [
*/
export const createNodes: CreateNodes<JestPluginOptions> = [
jestConfigGlob,
(...args) => {
async (configFilePath, options, context) => {
logger.warn(
'`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will change to the createNodesV2 API.'
);
return createNodesInternal(...args, {}, {});
const projectRoot = dirname(configFilePath);
const packageManagerWorkspacesGlob = combineGlobPatterns(
getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot)
);
if (
!checkIfConfigFileShouldBeProject(
configFilePath,
projectRoot,
packageManagerWorkspacesGlob,
context
)
) {
return {};
}
options = normalizeOptions(options);
const { targets, metadata } = await buildJestTargets(
configFilePath,
projectRoot,
options,
context,
{}
);
return {
projects: {
[projectRoot]: {
root: projectRoot,
targets,
metadata,
},
},
};
},
];
async function createNodesInternal(
function checkIfConfigFileShouldBeProject(
configFilePath: string,
options: JestPluginOptions,
context: CreateNodesContext,
targetsCache: Record<string, JestTargets>,
presetCache: Record<string, unknown>
) {
const projectRoot = dirname(configFilePath);
const packageManagerWorkspacesGlob = combineGlobPatterns(
getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot)
);
projectRoot: string,
packageManagerWorkspacesGlob: string,
context: CreateNodesContext | CreateNodesContextV2
): boolean {
// Do not create a project if package.json and project.json isn't there.
const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot));
if (
!siblingFiles.includes('package.json') &&
!siblingFiles.includes('project.json')
) {
return {};
return false;
} else if (
!siblingFiles.includes('project.json') &&
siblingFiles.includes('package.json')
@ -136,7 +221,7 @@ async function createNodesInternal(
const isPackageJsonProject = minimatch(path, packageManagerWorkspacesGlob);
if (!isPackageJsonProject) {
return {};
return false;
}
}
@ -148,31 +233,9 @@ async function createNodesInternal(
// The `getJestProjectsAsync` function uses the project graph, which leads to a
// circular dependency. We can skip this since it's no intended to be used for
// an Nx project.
return {};
return false;
}
options = normalizeOptions(options);
const hash = await calculateHashForCreateNodes(projectRoot, options, context);
targetsCache[hash] ??= await buildJestTargets(
configFilePath,
projectRoot,
options,
context,
presetCache
);
const { targets, metadata } = targetsCache[hash];
return {
projects: {
[projectRoot]: {
root: projectRoot,
targets,
metadata,
},
},
};
return true;
}
async function buildJestTargets(

9
packages/nx/src/config/nx-json.ts
View File

@ -2,15 +2,16 @@ import { existsSync } from 'fs';
import { dirname, join } from 'path';
import type { ChangelogRenderOptions } from '../../release/changelog-renderer';
import { readJsonFile } from '../utils/fileutils';
import { PackageManager } from '../utils/package-manager';
import { workspaceRoot } from '../utils/workspace-root';
import {
import type { PackageManager } from '../utils/package-manager';
import type {
InputDefinition,
TargetConfiguration,
TargetDependencyConfig,
} from './workspace-json-project-json';
import { readJsonFile } from '../utils/fileutils';
import { workspaceRoot } from '../utils/workspace-root';
export type ImplicitDependencyEntry<T = '*' | string[]> = {
[key: string]: T | ImplicitJsonSubsetDependency<T>;
};

81
packages/nx/src/daemon/client/client.ts
View File

@ -16,7 +16,6 @@ import { getFullOsSocketPath, killSocketOrPath } from '../socket-utils';
import {
DAEMON_DIR_FOR_CURRENT_WORKSPACE,
DAEMON_OUTPUT_LOG_FILE,
isDaemonDisabled,
removeSocketDir,
} from '../tmp-dir';
import { FileData, ProjectGraph } from '../../config/project-graph';
@ -50,7 +49,12 @@ import {
GET_FILES_IN_DIRECTORY,
HandleGetFilesInDirectoryMessage,
} from '../message-types/get-files-in-directory';
import { HASH_GLOB, HandleHashGlobMessage } from '../message-types/hash-glob';
import {
HASH_GLOB,
HASH_MULTI_GLOB,
HandleHashGlobMessage,
HandleHashMultiGlobMessage,
} from '../message-types/hash-glob';
import {
GET_ESTIMATED_TASK_TIMINGS,
GET_FLAKY_TASKS,
@ -91,6 +95,7 @@ import {
POST_TASKS_EXECUTION,
PRE_TASKS_EXECUTION,
} from '../message-types/run-tasks-execution-hooks';
import { isDaemonEnabled } from './enabled';
const DAEMON_ENV_SETTINGS = {
NX_PROJECT_GLOB_CACHE: 'false',
@ -136,48 +141,7 @@ export class DaemonClient {
private _err: FileHandle = null;
enabled() {
if (this._enabled === undefined) {
const useDaemonProcessOption = this.nxJson?.useDaemonProcess;
const env = process.env.NX_DAEMON;
// env takes precedence
// option=true,env=false => no daemon
// option=false,env=undefined => no daemon
// option=false,env=false => no daemon
// option=undefined,env=undefined => daemon
// option=true,env=true => daemon
// option=false,env=true => daemon
// CI=true,env=undefined => no daemon
// CI=true,env=false => no daemon
// CI=true,env=true => daemon
// docker=true,env=undefined => no daemon
// docker=true,env=false => no daemon
// docker=true,env=true => daemon
// WASM => no daemon because file watching does not work
if (
((isCI() || isDocker()) && env !== 'true') ||
isDaemonDisabled() ||
nxJsonIsNotPresent() ||
(useDaemonProcessOption === undefined && env === 'false') ||
(useDaemonProcessOption === true && env === 'false') ||
(useDaemonProcessOption === false && env === undefined) ||
(useDaemonProcessOption === false && env === 'false')
) {
this._enabled = false;
} else if (IS_WASM) {
output.warn({
title:
'The Nx Daemon is unsupported in WebAssembly environments. Some things may be slower than or not function as expected.',
});
this._enabled = false;
} else {
this._enabled = true;
}
}
return this._enabled;
return isDaemonEnabled(this.nxJson);
}
reset() {
@ -385,6 +349,14 @@ export class DaemonClient {
return this.sendToDaemonViaQueue(message);
}
hashMultiGlob(globGroups: string[][]): Promise<string[]> {
const message: HandleHashMultiGlobMessage = {
type: HASH_MULTI_GLOB,
globGroups: globGroups,
};
return this.sendToDaemonViaQueue(message);
}
getFlakyTasks(hashes: string[]): Promise<string[]> {
const message: HandleGetFlakyTasks = {
type: GET_FLAKY_TASKS,
@ -705,27 +677,6 @@ export class DaemonClient {
export const daemonClient = new DaemonClient();
export function isDaemonEnabled() {
return daemonClient.enabled();
}
function isDocker() {
try {
statSync('/.dockerenv');
return true;
} catch {
try {
return readFileSync('/proc/self/cgroup', 'utf8')?.includes('docker');
} catch {}
return false;
}
}
function nxJsonIsNotPresent() {
return !hasNxJson(workspaceRoot);
}
function daemonProcessException(message: string) {
try {
let log = readFileSync(DAEMON_OUTPUT_LOG_FILE).toString().split('\n');

79
packages/nx/src/daemon/client/enabled.ts Normal file
View File

@ -0,0 +1,79 @@
import {
hasNxJson,
readNxJson,
type NxJsonConfiguration,
} from '../../config/nx-json';
import { readFileSync, statSync } from 'node:fs';
import { isCI } from '../../utils/is-ci';
import { workspaceRoot } from '../../utils/workspace-root';
import { isDaemonDisabled } from '../tmp-dir';
let _enabled: boolean | undefined;
export function isDaemonEnabled(nxJson: NxJsonConfiguration = readNxJson()) {
if (_enabled === undefined) {
const useDaemonProcessOption = nxJson?.useDaemonProcess;
const env = process.env.NX_DAEMON;
// env takes precedence
// option=true,env=false => no daemon
// option=false,env=undefined => no daemon
// option=false,env=false => no daemon
// option=undefined,env=undefined => daemon
// option=true,env=true => daemon
// option=false,env=true => daemon
// CI=true,env=undefined => no daemon
// CI=true,env=false => no daemon
// CI=true,env=true => daemon
// docker=true,env=undefined => no daemon
// docker=true,env=false => no daemon
// docker=true,env=true => daemon
// WASM => no daemon because file watching does not work
if (
((isCI() || isDocker()) && env !== 'true') ||
isDaemonDisabled() ||
nxJsonIsNotPresent() ||
(useDaemonProcessOption === undefined && env === 'false') ||
(useDaemonProcessOption === true && env === 'false') ||
(useDaemonProcessOption === false && env === undefined) ||
(useDaemonProcessOption === false && env === 'false')
) {
_enabled = false;
} else if (
(require('../../native') as typeof import('../../native')).IS_WASM
) {
(
require('../../utils/output') as typeof import('../../utils/output')
).output.warn({
title:
'The Nx Daemon is unsupported in WebAssembly environments. Some things may be slower than or not function as expected.',
});
_enabled = false;
} else {
_enabled = true;
}
}
return _enabled;
}
function isDocker() {
try {
statSync('/.dockerenv');
return true;
} catch {
try {
return readFileSync('/proc/self/cgroup', 'utf8')?.includes('docker');
} catch {}
return false;
}
}
function nxJsonIsNotPresent() {
return !hasNxJson(workspaceRoot);
}

18
packages/nx/src/daemon/message-types/hash-glob.ts
View File

@ -16,3 +16,21 @@ export function isHandleHashGlobMessage(
message['type'] === HASH_GLOB
);
}
export const HASH_MULTI_GLOB = 'HASH_MULTI_GLOB' as const;
export type HandleHashMultiGlobMessage = {
type: typeof HASH_MULTI_GLOB;
globGroups: string[][];
};
export function isHandleHashMultiGlobMessage(
message: unknown
): message is HandleHashMultiGlobMessage {
return (
typeof message === 'object' &&
message !== null &&
'type' in message &&
message['type'] === HASH_MULTI_GLOB
);
}

15
packages/nx/src/daemon/server/handle-hash-glob.ts
View File

@ -1,5 +1,8 @@
import { workspaceRoot } from '../../utils/workspace-root';
import { hashWithWorkspaceContext } from '../../utils/workspace-context';
import {
hashMultiGlobWithWorkspaceContext,
hashWithWorkspaceContext,
} from '../../utils/workspace-context';
import { HandlerResult } from './server';
export async function handleHashGlob(
@ -12,3 +15,13 @@ export async function handleHashGlob(
description: 'handleHashGlob',
};
}
export async function handleHashMultiGlob(
globs: string[][]
): Promise<HandlerResult> {
const files = await hashMultiGlobWithWorkspaceContext(workspaceRoot, globs);
return {
response: JSON.stringify(files),
description: 'handleHashMultiGlob',
};
}

12
packages/nx/src/daemon/server/server.ts
View File

@ -77,8 +77,12 @@ import {
isHandleGetFilesInDirectoryMessage,
} from '../message-types/get-files-in-directory';
import { handleGetFilesInDirectory } from './handle-get-files-in-directory';
import { HASH_GLOB, isHandleHashGlobMessage } from '../message-types/hash-glob';
import { handleHashGlob } from './handle-hash-glob';
import {
HASH_GLOB,
isHandleHashGlobMessage,
isHandleHashMultiGlobMessage,
} from '../message-types/hash-glob';
import { handleHashGlob, handleHashMultiGlob } from './handle-hash-glob';
import {
GET_ESTIMATED_TASK_TIMINGS,
GET_FLAKY_TASKS,
@ -264,6 +268,10 @@ async function handleMessage(socket, data: string) {
await handleResult(socket, HASH_GLOB, () =>
handleHashGlob(payload.globs, payload.exclude)
);
} else if (isHandleHashMultiGlobMessage(payload)) {
await handleResult(socket, HASH_GLOB, () =>
handleHashMultiGlob(payload.globGroups)
);
} else if (isHandleGetFlakyTasksMessage(payload)) {
await handleResult(socket, GET_FLAKY_TASKS, () =>
handleGetFlakyTasks(payload.hashes)

2
packages/nx/src/devkit-exports.ts
View File

@ -260,4 +260,4 @@ export { cacheDir } from './utils/cache-directory';
*/
export { createProjectFileMapUsingProjectGraph } from './project-graph/file-map-utils';
export { isDaemonEnabled } from './daemon/client/client';
export { isDaemonEnabled } from './daemon/client/enabled';

5
packages/nx/src/devkit-internals.ts
View File

@ -20,7 +20,10 @@ export { stripIndent } from './utils/logger';
export { readModulePackageJson } from './utils/package-json';
export { splitByColons } from './utils/split-target';
export { hashObject } from './hasher/file-hasher';
export { hashWithWorkspaceContext } from './utils/workspace-context';
export {
hashWithWorkspaceContext,
hashMultiGlobWithWorkspaceContext,
} from './utils/workspace-context';
export {
createProjectRootMappingsFromProjectConfigurations,
findProjectForPath,

1
packages/nx/src/native/index.d.ts vendored
View File

@ -100,6 +100,7 @@ export declare class WorkspaceContext {
* as the input globs.
*/
multiGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<Array<string>>
hashFilesMatchingGlobs(globGroups: Array<Array<string>>): Array<string>
hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string
incrementalUpdate(updatedFiles: Array<string>, deletedFiles: Array<string>): Record<string, string>
updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject<ProjectFiles>, globalFiles: ExternalObject<Array<FileData>>, updatedFiles: Record<string, string>, deletedFiles: Array<string>): UpdatedWorkspaceFiles

23
packages/nx/src/native/workspace/context.rs
View File

@ -253,6 +253,29 @@ impl WorkspaceContext {
.collect()
}
#[napi]
pub fn hash_files_matching_globs(
&self,
glob_groups: Vec<Vec<String>>,
) -> napi::Result<Vec<String>> {
let files = &self.all_file_data();
let hashes = glob_groups
.into_iter()
.map(|globs| {
let globbed_files =
config_files::glob_files(files, globs, None)?.collect::<Vec<_>>();
let mut hasher = xxh3::Xxh3::new();
for file in globbed_files {
hasher.update(file.file.as_bytes());
hasher.update(file.hash.as_bytes());
}
Ok(hasher.digest().to_string())
})
.collect::<napi::Result<Vec<_>>>()?;
Ok(hashes)
}
#[napi]
pub fn hash_files_matching_glob(
&self,

11
packages/nx/src/project-graph/plugins/isolation/plugin-worker.ts
View File

@ -1,3 +1,7 @@
import { performance } from 'node:perf_hooks';
performance.mark(`plugin worker ${process.pid} code loading -- start`);
import { consumeMessage, isPluginWorkerMessage } from './messaging';
import { createSerializableError } from '../../../utils/serializable-error';
import { consumeMessagesFromSocket } from '../../../utils/consume-messages-from-socket';
@ -10,6 +14,13 @@ if (process.env.NX_PERF_LOGGING === 'true') {
require('../../../utils/perf-logging');
}
performance.mark(`plugin worker ${process.pid} code loading -- end`);
performance.measure(
`plugin worker ${process.pid} code loading`,
`plugin worker ${process.pid} code loading -- start`,
`plugin worker ${process.pid} code loading -- end`
);
global.NX_GRAPH_CREATION = true;
global.NX_PLUGIN_WORKER = true;
let connected = false;

9
packages/nx/src/project-graph/plugins/loaded-nx-plugin.ts
View File

@ -1,5 +1,5 @@
import type { ProjectGraph } from '../../config/project-graph';
import type { PluginConfiguration } from '../../config/nx-json';
import { readNxJson, type PluginConfiguration } from '../../config/nx-json';
import {
AggregateCreateNodesError,
isAggregateCreateNodesError,
@ -17,7 +17,7 @@ import type {
} from './public-api';
import { createNodesFromFiles } from './utils';
import { isIsolationEnabled } from './isolation/enabled';
import { isDaemonEnabled } from '../../daemon/client/client';
import { isDaemonEnabled } from '../../daemon/client/enabled';
export class LoadedNxPlugin {
index?: number;
@ -123,7 +123,10 @@ export class LoadedNxPlugin {
this.preTasksExecution = async (context: PreTasksExecutionContext) => {
const updates = {};
let originalEnv = process.env;
if (isIsolationEnabled() || isDaemonEnabled()) {
if (
isIsolationEnabled() ||
isDaemonEnabled(context.nxJsonConfiguration)
) {
process.env = new Proxy<NodeJS.ProcessEnv>(originalEnv, {
set: (target, key: string, value) => {
target[key] = value;

9
packages/nx/src/project-graph/plugins/tasks-execution-hooks.ts
View File

@ -4,12 +4,13 @@ import type {
} from './public-api';
import { getPlugins } from './get-plugins';
import { isOnDaemon } from '../../daemon/is-on-daemon';
import { daemonClient, isDaemonEnabled } from '../../daemon/client/client';
import { daemonClient } from '../../daemon/client/client';
import { isDaemonEnabled } from '../../daemon/client/enabled';
export async function runPreTasksExecution(
pluginContext: PreTasksExecutionContext
) {
if (isOnDaemon() || !isDaemonEnabled()) {
if (isOnDaemon() || !isDaemonEnabled(pluginContext.nxJsonConfiguration)) {
performance.mark(`preTasksExecution:start`);
const plugins = await getPlugins(pluginContext.workspaceRoot);
const envs = await Promise.all(
@ -30,7 +31,7 @@ export async function runPreTasksExecution(
})
);
if (!isDaemonEnabled()) {
if (!isDaemonEnabled(pluginContext.nxJsonConfiguration)) {
applyProcessEnvs(envs);
}
performance.mark(`preTasksExecution:end`);
@ -57,7 +58,7 @@ function applyProcessEnvs(envs: NodeJS.ProcessEnv[]) {
export async function runPostTasksExecution(
context: PostTasksExecutionContext
) {
if (isOnDaemon() || !isDaemonEnabled()) {
if (isOnDaemon() || !isDaemonEnabled(context.nxJsonConfiguration)) {
performance.mark(`postTasksExecution:start`);
const plugins = await getPlugins();
await Promise.all(

24
packages/nx/src/project-graph/plugins/utils.ts
View File

@ -1,11 +1,16 @@
import {
CreateNodesContext,
CreateNodesContextV2,
CreateNodesFunction,
CreateNodesResult,
} from './public-api';
import { AggregateCreateNodesError } from '../error-types';
export async function createNodesFromFiles<T = unknown>(
createNodes: CreateNodesFunction<T>,
createNodes: (
projectConfigurationFile: string,
options: T | undefined,
context: CreateNodesContext,
idx: number
) => CreateNodesResult | Promise<CreateNodesResult>,
configFiles: readonly string[],
options: T,
context: CreateNodesContextV2
@ -14,12 +19,17 @@ export async function createNodesFromFiles<T = unknown>(
const errors: Array<[file: string, error: Error]> = [];
await Promise.all(
configFiles.map(async (file) => {
configFiles.map(async (file, idx) => {
try {
const value = await createNodes(file, options, {
...context,
configFiles,
});
const value = await createNodes(
file,
options,
{
...context,
configFiles,
},
idx
);
if (value) {
results.push([file, value] as const);
}

11
packages/nx/src/utils/workspace-context.ts
View File

@ -86,6 +86,17 @@ export async function hashWithWorkspaceContext(
return daemonClient.hashGlob(globs, exclude);
}
export async function hashMultiGlobWithWorkspaceContext(
workspaceRoot: string,
globGroups: string[][]
) {
if (isOnDaemon() || !daemonClient.enabled()) {
ensureContextAvailable(workspaceRoot);
return workspaceContext.hashFilesMatchingGlobs(globGroups);
}
return daemonClient.hashMultiGlob(globGroups);
}
export async function updateContextWithChangedFiles(
workspaceRoot: string,
createdFiles: string[],

3
packages/rspack/src/plugins/utils/plugins/rspack-nx-build-coordination-plugin.ts
View File

@ -1,6 +1,7 @@
import { exec } from 'child_process';
import type { Compiler } from '@rspack/core';
import { daemonClient, isDaemonEnabled } from 'nx/src/daemon/client/client';
import { daemonClient } from 'nx/src/daemon/client/client';
import { isDaemonEnabled } from 'nx/src/daemon/client/enabled';
import { BatchFunctionRunner } from 'nx/src/command-line/watch/watch';
import { output } from 'nx/src/utils/output';

229
packages/vite/src/plugins/plugin.ts
View File

@ -2,6 +2,7 @@ import {
CreateDependencies,
CreateNodes,
CreateNodesContext,
CreateNodesContextV2,
createNodesFromFiles,
CreateNodesV2,
detectPackageManager,
@ -16,7 +17,10 @@ import {
import { dirname, isAbsolute, join, relative } from 'path';
import { getNamedInputs } from '@nx/devkit/src/utils/get-named-inputs';
import { existsSync, readdirSync } from 'fs';
import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import {
calculateHashesForCreateNodes,
calculateHashForCreateNodes,
} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import { workspaceDataDirectory } from 'nx/src/utils/cache-directory';
import { getLockFileName } from '@nx/js';
import { loadViteDynamicImport } from '../utils/executor-utils';
@ -42,7 +46,10 @@ export interface VitePluginOptions {
buildDepsTargetName?: string;
}
type ViteTargets = Pick<ProjectConfiguration, 'targets' | 'metadata'>;
type ViteTargets = Pick<
ProjectConfiguration,
'targets' | 'metadata' | 'projectType'
>;
function readTargetsCache(cachePath: string): Record<string, ViteTargets> {
return process.env.NX_CACHE_PROJECT_GRAPH !== 'false' && existsSync(cachePath)
@ -67,20 +74,88 @@ export const createNodesV2: CreateNodesV2<VitePluginOptions> = [
viteVitestConfigGlob,
async (configFilePaths, options, context) => {
const optionsHash = hashObject(options);
const normalizedOptions = normalizeOptions(options);
const cachePath = join(workspaceDataDirectory, `vite-${optionsHash}.hash`);
const targetsCache = readTargetsCache(cachePath);
const isUsingTsSolutionSetup = _isUsingTsSolutionSetup();
const { roots: projectRoots, configFiles: validConfigFiles } =
configFilePaths.reduce(
(acc, configFile) => {
const potentialRoot = dirname(configFile);
if (checkIfConfigFileShouldBeProject(potentialRoot, context)) {
acc.roots.push(potentialRoot);
acc.configFiles.push(configFile);
}
return acc;
},
{
roots: [],
configFiles: [],
} as {
roots: string[];
configFiles: string[];
}
);
const lockfile = getLockFileName(
detectPackageManager(context.workspaceRoot)
);
const hashes = await calculateHashesForCreateNodes(
projectRoots,
{ ...normalizedOptions, isUsingTsSolutionSetup },
context,
projectRoots.map((r) => [lockfile])
);
try {
return await createNodesFromFiles(
(configFile, options, context) =>
createNodesInternal(
configFile,
options,
context,
targetsCache,
isUsingTsSolutionSetup
),
configFilePaths,
async (configFile, _, context, idx) => {
const projectRoot = dirname(configFile);
// Do not create a project if package.json and project.json isn't there.
const siblingFiles = readdirSync(
join(context.workspaceRoot, projectRoot)
);
const tsConfigFiles =
siblingFiles.filter((p) =>
minimatch(p, 'tsconfig*{.json,.*.json}')
) ?? [];
// results from vitest.config.js will be different from results of vite.config.js
// but the hash will be the same because it is based on the files under the project root.
// Adding the config file path to the hash ensures that the final hash value is different
// for different config files.
const hash = hashes[idx] + configFile;
const { projectType, metadata, targets } = (targetsCache[hash] ??=
await buildViteTargets(
configFile,
projectRoot,
normalizedOptions,
tsConfigFiles,
isUsingTsSolutionSetup,
context
));
const project: ProjectConfiguration = {
root: projectRoot,
targets,
metadata,
};
// If project is buildable, then the project type.
// If it is not buildable, then leave it to other plugins/project.json to set the project type.
if (project.targets[normalizedOptions.buildTargetName]) {
project.projectType = projectType;
}
return {
projects: {
[projectRoot]: project,
},
};
},
validConfigFiles,
options,
context
);
@ -96,78 +171,52 @@ export const createNodes: CreateNodes<VitePluginOptions> = [
logger.warn(
'`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will change to the createNodesV2 API.'
);
return createNodesInternal(
const projectRoot = dirname(configFilePath);
// Do not create a project if package.json and project.json isn't there.
const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot));
if (
!siblingFiles.includes('package.json') &&
!siblingFiles.includes('project.json')
) {
return {};
}
const tsConfigFiles =
siblingFiles.filter((p) => minimatch(p, 'tsconfig*{.json,.*.json}')) ??
[];
const normalizedOptions = normalizeOptions(options);
const isUsingTsSolutionSetup = _isUsingTsSolutionSetup();
const { projectType, metadata, targets } = await buildViteTargets(
configFilePath,
options,
context,
{},
_isUsingTsSolutionSetup()
projectRoot,
normalizedOptions,
tsConfigFiles,
isUsingTsSolutionSetup,
context
);
const project: ProjectConfiguration = {
root: projectRoot,
targets,
metadata,
};
// If project is buildable, then the project type.
// If it is not buildable, then leave it to other plugins/project.json to set the project type.
if (project.targets[normalizedOptions.buildTargetName]) {
project.projectType = projectType;
}
return {
projects: {
[projectRoot]: project,
},
};
},
];
async function createNodesInternal(
configFilePath: string,
options: VitePluginOptions,
context: CreateNodesContext,
targetsCache: Record<string, ViteTargets>,
isUsingTsSolutionSetup: boolean
) {
const projectRoot = dirname(configFilePath);
// Do not create a project if package.json and project.json isn't there.
const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot));
if (
!siblingFiles.includes('package.json') &&
!siblingFiles.includes('project.json')
) {
return {};
}
const tsConfigFiles =
siblingFiles.filter((p) => minimatch(p, 'tsconfig*{.json,.*.json}')) ?? [];
const normalizedOptions = normalizeOptions(options);
// We do not want to alter how the hash is calculated, so appending the config file path to the hash
// to prevent vite/vitest files overwriting the target cache created by the other
const hash =
(await calculateHashForCreateNodes(
projectRoot,
{ ...normalizedOptions, isUsingTsSolutionSetup },
context,
[getLockFileName(detectPackageManager(context.workspaceRoot))]
)) + configFilePath;
const { isLibrary, ...viteTargets } = await buildViteTargets(
configFilePath,
projectRoot,
normalizedOptions,
tsConfigFiles,
isUsingTsSolutionSetup,
context
);
targetsCache[hash] ??= viteTargets;
const { targets, metadata } = targetsCache[hash];
const project: ProjectConfiguration = {
root: projectRoot,
targets,
metadata,
};
// If project is buildable, then the project type.
// If it is not buildable, then leave it to other plugins/project.json to set the project type.
if (project.targets[normalizedOptions.buildTargetName]) {
project.projectType = isLibrary ? 'library' : 'application';
}
return {
projects: {
[projectRoot]: project,
},
};
}
async function buildViteTargets(
configFilePath: string,
projectRoot: string,
@ -175,7 +224,7 @@ async function buildViteTargets(
tsConfigFiles: string[],
isUsingTsSolutionSetup: boolean,
context: CreateNodesContext
): Promise<ViteTargets & { isLibrary: boolean }> {
): Promise<ViteTargets> {
const absoluteConfigFilePath = joinPathFragments(
context.workspaceRoot,
configFilePath
@ -304,7 +353,11 @@ async function buildViteTargets(
);
const metadata = {};
return { targets, metadata, isLibrary: Boolean(viteBuildConfig.build?.lib) };
return {
targets,
metadata,
projectType: viteBuildConfig.build?.lib ? 'library' : 'application',
};
}
async function buildTarget(
@ -538,3 +591,19 @@ function normalizeOptions(options: VitePluginOptions): VitePluginOptions {
options.typecheckTargetName ??= 'typecheck';
return options;
}
function checkIfConfigFileShouldBeProject(
projectRoot: string,
context: CreateNodesContext | CreateNodesContextV2
): boolean {
// Do not create a project if package.json and project.json isn't there.
const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot));
if (
!siblingFiles.includes('package.json') &&
!siblingFiles.includes('project.json')
) {
return false;
}
return true;
}

3
packages/webpack/src/plugins/webpack-nx-build-coordination-plugin.ts
View File

@ -1,8 +1,9 @@
import { exec } from 'child_process';
import type { Compiler } from 'webpack';
import { daemonClient, isDaemonEnabled } from 'nx/src/daemon/client/client';
import { daemonClient } from 'nx/src/daemon/client/client';
import { BatchFunctionRunner } from 'nx/src/command-line/watch/watch';
import { output } from 'nx/src/utils/output';
import { isDaemonEnabled } from 'nx/src/daemon/client/enabled';
type PluginOptions = {
skipInitialBuild?: boolean;