feat(core): add create nodes v2 for batch processing config files (#26250)

This commit is contained in:
Craigory Coppola 2024-05-30 15:28:59 -04:00 committed by GitHub
parent 1277b22ce4
commit a5682d1ca5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 885 additions and 317 deletions

View File

@ -0,0 +1,197 @@
# Class: AggregateCreateNodesError
This error should be thrown when a `createNodesV2` function hits a recoverable error.
It allows Nx to recieve partial results and continue processing for better UX.
## Hierarchy
- `Error`
**`AggregateCreateNodesError`**
## Table of contents
### Constructors
- [constructor](../../devkit/documents/AggregateCreateNodesError#constructor)
### Properties
- [cause](../../devkit/documents/AggregateCreateNodesError#cause): unknown
- [errors](../../devkit/documents/AggregateCreateNodesError#errors): [file: string, error: Error][]
- [message](../../devkit/documents/AggregateCreateNodesError#message): string
- [name](../../devkit/documents/AggregateCreateNodesError#name): string
- [partialResults](../../devkit/documents/AggregateCreateNodesError#partialresults): CreateNodesResultV2
- [stack](../../devkit/documents/AggregateCreateNodesError#stack): string
- [prepareStackTrace](../../devkit/documents/AggregateCreateNodesError#preparestacktrace): Function
- [stackTraceLimit](../../devkit/documents/AggregateCreateNodesError#stacktracelimit): number
### Methods
- [captureStackTrace](../../devkit/documents/AggregateCreateNodesError#capturestacktrace)
## Constructors
### constructor
**new AggregateCreateNodesError**(`errors`, `partialResults`): [`AggregateCreateNodesError`](../../devkit/documents/AggregateCreateNodesError)
Throwing this error from a `createNodesV2` function will allow Nx to continue processing and recieve partial results from your plugin.
#### Parameters
| Name | Type | Description |
| :--------------- | :------------------------------------------------------------------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `errors` | [file: string, error: Error][] | An array of tuples that represent errors encountered when processing a given file. An example entry might look like ['path/to/project.json', [Error: 'Invalid JSON. Unexpected token 'a' in JSON at position 0]] |
| `partialResults` | [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) | The partial results of the `createNodesV2` function. This should be the results for each file that didn't encounter an issue. |
#### Returns
[`AggregateCreateNodesError`](../../devkit/documents/AggregateCreateNodesError)
**`Example`**
```ts
export async function createNodesV2(files: string[]) {
const partialResults = [];
const errors = [];
await Promise.all(
files.map(async (file) => {
try {
const result = await createNodes(file);
partialResults.push(result);
} catch (e) {
errors.push([file, e]);
}
})
);
if (errors.length > 0) {
throw new AggregateCreateNodesError(errors, partialResults);
}
return partialResults;
}
```
#### Overrides
Error.constructor
## Properties
### cause
`Optional` **cause**: `unknown`
#### Inherited from
Error.cause
---
### errors
`Readonly` **errors**: [file: string, error: Error][]
An array of tuples that represent errors encountered when processing a given file. An example entry might look like ['path/to/project.json', [Error: 'Invalid JSON. Unexpected token 'a' in JSON at position 0]]
---
### message
**message**: `string`
#### Inherited from
Error.message
---
### name
**name**: `string`
#### Inherited from
Error.name
---
### partialResults
`Readonly` **partialResults**: [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)
The partial results of the `createNodesV2` function. This should be the results for each file that didn't encounter an issue.
---
### stack
`Optional` **stack**: `string`
#### Inherited from
Error.stack
---
### prepareStackTrace
`Static` `Optional` **prepareStackTrace**: (`err`: `Error`, `stackTraces`: `CallSite`[]) => `any`
Optional override for formatting stack traces
**`See`**
https://v8.dev/docs/stack-trace-api#customizing-stack-traces
#### Type declaration
▸ (`err`, `stackTraces`): `any`
##### Parameters
| Name | Type |
| :------------ | :----------- |
| `err` | `Error` |
| `stackTraces` | `CallSite`[] |
##### Returns
`any`
#### Inherited from
Error.prepareStackTrace
---
### stackTraceLimit
`Static` **stackTraceLimit**: `number`
#### Inherited from
Error.stackTraceLimit
## Methods
### captureStackTrace
**captureStackTrace**(`targetObject`, `constructorOpt?`): `void`
Create .stack property on a target object
#### Parameters
| Name | Type |
| :---------------- | :--------- |
| `targetObject` | `object` |
| `constructorOpt?` | `Function` |
#### Returns
`void`
#### Inherited from
Error.captureStackTrace

View File

@ -4,6 +4,18 @@
A pair of file patterns and [CreateNodesFunction](../../devkit/documents/CreateNodesFunction)
Nx 19.2+: Both original `CreateNodes` and `CreateNodesV2` are supported. Nx will only invoke `CreateNodesV2` if it is present.
Nx 20.X : The `CreateNodesV2` will be the only supported API. This typing will still exist, but be identical to `CreateNodesV2`.
Nx **will not** invoke the original `plugin.createNodes` callback. This should give plugin authors a window to transition.
Plugin authors should update their plugin's `createNodes` function to align with `CreateNodesV2` / the updated `CreateNodes`.
The plugin should contain something like: `export createNodes = createNodesV2;` during this period. This will allow the plugin
to maintain compatibility with Nx 19.2 and up.
Nx 21.X : The `CreateNodesV2` typing will be removed, as it has replaced `CreateNodes`.
**`Deprecated`**
Use [CreateNodesV2](../../devkit/documents/CreateNodesV2) instead. CreateNodesV2 will replace this API. Read more about the transition above.
#### Type parameters
| Name | Type |

View File

@ -2,6 +2,12 @@
Context for [CreateNodesFunction](../../devkit/documents/CreateNodesFunction)
## Hierarchy
- [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2)
**`CreateNodesContext`**
## Table of contents
### Properties
@ -24,8 +30,16 @@ The subset of configuration files which match the createNodes pattern
`Readonly` **nxJsonConfiguration**: [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\>
#### Inherited from
[CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2).[nxJsonConfiguration](../../devkit/documents/CreateNodesContextV2#nxjsonconfiguration)
---
### workspaceRoot
`Readonly` **workspaceRoot**: `string`
#### Inherited from
[CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2).[workspaceRoot](../../devkit/documents/CreateNodesContextV2#workspaceroot)

View File

@ -0,0 +1,26 @@
# Interface: CreateNodesContextV2
## Hierarchy
- **`CreateNodesContextV2`**
↳ [`CreateNodesContext`](../../devkit/documents/CreateNodesContext)
## Table of contents
### Properties
- [nxJsonConfiguration](../../devkit/documents/CreateNodesContextV2#nxjsonconfiguration): NxJsonConfiguration<string[] | "\*">
- [workspaceRoot](../../devkit/documents/CreateNodesContextV2#workspaceroot): string
## Properties
### nxJsonConfiguration
`Readonly` **nxJsonConfiguration**: [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\>
---
### workspaceRoot
`Readonly` **workspaceRoot**: `string`

View File

@ -0,0 +1,25 @@
# Type alias: CreateNodesFunctionV2\<T\>
Ƭ **CreateNodesFunctionV2**\<`T`\>: (`projectConfigurationFiles`: readonly `string`[], `options`: `T` \| `undefined`, `context`: [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2)) => [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) \| `Promise`\<[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)\>
#### Type parameters
| Name | Type |
| :--- | :-------- |
| `T` | `unknown` |
#### Type declaration
▸ (`projectConfigurationFiles`, `options`, `context`): [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) \| `Promise`\<[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)\>
##### Parameters
| Name | Type |
| :-------------------------- | :-------------------------------------------------------------------- |
| `projectConfigurationFiles` | readonly `string`[] |
| `options` | `T` \| `undefined` |
| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) |
##### Returns
[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) \| `Promise`\<[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)\>

View File

@ -0,0 +1,3 @@
# Type alias: CreateNodesResultV2
Ƭ **CreateNodesResultV2**: readonly [configFileSource: string, result: CreateNodesResult][]

View File

@ -0,0 +1,12 @@
# Type alias: CreateNodesV2\<T\>
Ƭ **CreateNodesV2**\<`T`\>: readonly [projectFilePattern: string, createNodesFunction: CreateNodesFunctionV2\<T\>]
A pair of file patterns and [CreateNodesFunctionV2](../../devkit/documents/CreateNodesFunctionV2)
In Nx 20 [CreateNodes](../../devkit/documents/CreateNodes) will be replaced with this type. In Nx 21, this type will be removed.
#### Type parameters
| Name | Type |
| :--- | :-------- |
| `T` | `unknown` |

View File

@ -13,8 +13,9 @@ A plugin for Nx which creates nodes and dependencies for the [ProjectGraph](../.
#### Type declaration
| Name | Type | Description |
| :-------------------- | :------------------------------------------------------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------- |
| :-------------------- | :------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `createDependencies?` | [`CreateDependencies`](../../devkit/documents/CreateDependencies)\<`TOptions`\> | Provides a function to analyze files to create dependencies for the [ProjectGraph](../../devkit/documents/ProjectGraph) |
| `createMetadata?` | [`CreateMetadata`](../../devkit/documents/CreateMetadata)\<`TOptions`\> | Provides a function to create metadata for the [ProjectGraph](../../devkit/documents/ProjectGraph) |
| `createNodes?` | [`CreateNodes`](../../devkit/documents/CreateNodes)\<`TOptions`\> | Provides a file pattern and function that retrieves configuration info from those files. e.g. { '\*_/_.csproj': buildProjectsFromCsProjFile } |
| `createNodes?` | [`CreateNodes`](../../devkit/documents/CreateNodes)\<`TOptions`\> | Provides a file pattern and function that retrieves configuration info from those files. e.g. { '**/\*.csproj': buildProjectsFromCsProjFile } **`Deprecated`\*\* Use createNodesV2 instead. In Nx 20 support for calling createNodes with a single file for the first argument will be removed. |
| `createNodesV2?` | [`CreateNodesV2`](../../devkit/documents/CreateNodesV2)\<`TOptions`\> | Provides a file pattern and function that retrieves configuration info from those files. e.g. { '\*_/_.csproj': buildProjectsFromCsProjFiles } In Nx 20 createNodes will be replaced with this property. In Nx 21, this property will be removed. |
| `name` | `string` | - |

View File

@ -18,12 +18,14 @@ It only uses language primitives and immutable objects
### Classes
- [AggregateCreateNodesError](../../devkit/documents/AggregateCreateNodesError)
- [ProjectGraphBuilder](../../devkit/documents/ProjectGraphBuilder)
### Interfaces
- [CreateDependenciesContext](../../devkit/documents/CreateDependenciesContext)
- [CreateNodesContext](../../devkit/documents/CreateNodesContext)
- [CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2)
- [CreateNodesResult](../../devkit/documents/CreateNodesResult)
- [DefaultTasksRunnerOptions](../../devkit/documents/DefaultTasksRunnerOptions)
- [ExecutorContext](../../devkit/documents/ExecutorContext)
@ -67,6 +69,9 @@ It only uses language primitives and immutable objects
- [CreateMetadataContext](../../devkit/documents/CreateMetadataContext)
- [CreateNodes](../../devkit/documents/CreateNodes)
- [CreateNodesFunction](../../devkit/documents/CreateNodesFunction)
- [CreateNodesFunctionV2](../../devkit/documents/CreateNodesFunctionV2)
- [CreateNodesResultV2](../../devkit/documents/CreateNodesResultV2)
- [CreateNodesV2](../../devkit/documents/CreateNodesV2)
- [CustomHasher](../../devkit/documents/CustomHasher)
- [DynamicDependency](../../devkit/documents/DynamicDependency)
- [Executor](../../devkit/documents/Executor)
@ -109,6 +114,7 @@ It only uses language primitives and immutable objects
- [applyChangesToString](../../devkit/documents/applyChangesToString)
- [convertNxExecutor](../../devkit/documents/convertNxExecutor)
- [convertNxGenerator](../../devkit/documents/convertNxGenerator)
- [createNodesFromFiles](../../devkit/documents/createNodesFromFiles)
- [createProjectFileMapUsingProjectGraph](../../devkit/documents/createProjectFileMapUsingProjectGraph)
- [createProjectGraphAsync](../../devkit/documents/createProjectGraphAsync)
- [defaultTasksRunner](../../devkit/documents/defaultTasksRunner)

View File

@ -0,0 +1,22 @@
# Function: createNodesFromFiles
**createNodesFromFiles**\<`T`\>(`createNodes`, `configFiles`, `options`, `context`): `Promise`\<[file: string, value: CreateNodesResult][]\>
#### Type parameters
| Name | Type |
| :--- | :-------- |
| `T` | `unknown` |
#### Parameters
| Name | Type |
| :------------ | :-------------------------------------------------------------------- |
| `createNodes` | [`CreateNodesFunction`](../../devkit/documents/CreateNodesFunction) |
| `configFiles` | readonly `string`[] |
| `options` | `T` |
| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) |
#### Returns
`Promise`\<[file: string, value: CreateNodesResult][]\>

View File

@ -18,12 +18,14 @@ It only uses language primitives and immutable objects
### Classes
- [AggregateCreateNodesError](../../devkit/documents/AggregateCreateNodesError)
- [ProjectGraphBuilder](../../devkit/documents/ProjectGraphBuilder)
### Interfaces
- [CreateDependenciesContext](../../devkit/documents/CreateDependenciesContext)
- [CreateNodesContext](../../devkit/documents/CreateNodesContext)
- [CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2)
- [CreateNodesResult](../../devkit/documents/CreateNodesResult)
- [DefaultTasksRunnerOptions](../../devkit/documents/DefaultTasksRunnerOptions)
- [ExecutorContext](../../devkit/documents/ExecutorContext)
@ -67,6 +69,9 @@ It only uses language primitives and immutable objects
- [CreateMetadataContext](../../devkit/documents/CreateMetadataContext)
- [CreateNodes](../../devkit/documents/CreateNodes)
- [CreateNodesFunction](../../devkit/documents/CreateNodesFunction)
- [CreateNodesFunctionV2](../../devkit/documents/CreateNodesFunctionV2)
- [CreateNodesResultV2](../../devkit/documents/CreateNodesResultV2)
- [CreateNodesV2](../../devkit/documents/CreateNodesV2)
- [CustomHasher](../../devkit/documents/CustomHasher)
- [DynamicDependency](../../devkit/documents/DynamicDependency)
- [Executor](../../devkit/documents/Executor)
@ -109,6 +114,7 @@ It only uses language primitives and immutable objects
- [applyChangesToString](../../devkit/documents/applyChangesToString)
- [convertNxExecutor](../../devkit/documents/convertNxExecutor)
- [convertNxGenerator](../../devkit/documents/convertNxGenerator)
- [createNodesFromFiles](../../devkit/documents/createNodesFromFiles)
- [createProjectFileMapUsingProjectGraph](../../devkit/documents/createProjectFileMapUsingProjectGraph)
- [createProjectGraphAsync](../../devkit/documents/createProjectGraphAsync)
- [defaultTasksRunner](../../devkit/documents/defaultTasksRunner)

View File

@ -10,11 +10,9 @@ import { readFileSync } from 'node:fs';
import { basename } from 'node:path';
import {
getGradleReport,
invalidateGradleReportCache,
getCurrentGradleReport,
newLineSeparator,
} from '../utils/get-gradle-report';
import { writeTargetsToCache } from './nodes';
export const createDependencies: CreateDependencies = async (
_,
@ -31,7 +29,7 @@ export const createDependencies: CreateDependencies = async (
gradleFileToGradleProjectMap,
gradleProjectToProjectName,
buildFileToDepsMap,
} = getGradleReport();
} = getCurrentGradleReport();
for (const gradleFile of gradleFiles) {
const gradleProject = gradleFileToGradleProjectMap.get(gradleFile);
@ -59,10 +57,6 @@ export const createDependencies: CreateDependencies = async (
gradleDependenciesEnd.name
);
writeTargetsToCache();
if (dependencies.length) {
invalidateGradleReportCache();
}
return dependencies;
};

View File

@ -1,12 +1,13 @@
import { CreateNodesContext } from '@nx/devkit';
import { TempFs } from 'nx/src/internal-testing-utils/temp-fs';
import type { GradleReport } from '../utils/get-gradle-report';
import { type GradleReport } from '../utils/get-gradle-report';
let gradleReport: GradleReport;
jest.mock('../utils/get-gradle-report.ts', () => {
return {
getGradleReport: jest.fn().mockImplementation(() => gradleReport),
populateGradleReport: jest.fn().mockImplementation(() => void 0),
getCurrentGradleReport: jest.fn().mockImplementation(() => gradleReport),
};
});

View File

@ -1,10 +1,16 @@
import {
CreateNodes,
CreateNodesV2,
CreateNodesContext,
CreateNodesContextV2,
ProjectConfiguration,
TargetConfiguration,
createNodesFromFiles,
readJsonFile,
writeJsonFile,
CreateNodesResultV2,
CreateNodesFunction,
logger,
} from '@nx/devkit';
import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
import { existsSync } from 'node:fs';
@ -12,7 +18,13 @@ import { dirname, join } from 'node:path';
import { projectGraphCacheDirectory } from 'nx/src/utils/cache-directory';
import { getGradleExecFile } from '../utils/exec-gradle';
import { getGradleReport } from '../utils/get-gradle-report';
import {
populateGradleReport,
getCurrentGradleReport,
GradleReport,
gradleConfigGlob,
} from '../utils/get-gradle-report';
import { hashObject } from 'nx/src/hasher/file-hasher';
const cacheableTaskType = new Set(['Build', 'Verification']);
const dependsOnMap = {
@ -33,8 +45,6 @@ export interface GradlePluginOptions {
[taskTargetName: string]: string | undefined;
}
const cachePath = join(projectGraphCacheDirectory, 'gradle.hash');
const targetsCache = readTargetsCache();
type GradleTargets = Record<
string,
{
@ -44,20 +54,45 @@ type GradleTargets = Record<
}
>;
function readTargetsCache(): GradleTargets {
function readTargetsCache(cachePath: string): GradleTargets {
return existsSync(cachePath) ? readJsonFile(cachePath) : {};
}
export function writeTargetsToCache() {
const oldCache = readTargetsCache();
writeJsonFile(cachePath, {
...oldCache,
...targetsCache,
});
export function writeTargetsToCache(cachePath: string, results: GradleTargets) {
writeJsonFile(cachePath, results);
}
export const createNodes: CreateNodes<GradlePluginOptions> = [
'**/build.{gradle.kts,gradle}',
export const createNodesV2: CreateNodesV2<GradlePluginOptions> = [
gradleConfigGlob,
async (configFiles, options, context) => {
const optionsHash = hashObject(options);
const cachePath = join(
projectGraphCacheDirectory,
`gradle-${optionsHash}.hash`
);
const targetsCache = readTargetsCache(cachePath);
populateGradleReport(context.workspaceRoot);
const gradleReport = getCurrentGradleReport();
try {
return await createNodesFromFiles(
makeCreateNodes(gradleReport, targetsCache),
configFiles,
options,
context
);
} finally {
writeTargetsToCache(cachePath, targetsCache);
}
},
];
export const makeCreateNodes =
(
gradleReport: GradleReport,
targetsCache: GradleTargets
): CreateNodesFunction =>
(
gradleFilePath,
options: GradlePluginOptions | undefined,
@ -71,6 +106,7 @@ export const createNodes: CreateNodes<GradlePluginOptions> = [
context
);
targetsCache[hash] ??= createGradleProject(
gradleReport,
gradleFilePath,
options,
context
@ -84,10 +120,26 @@ export const createNodes: CreateNodes<GradlePluginOptions> = [
[projectRoot]: project,
},
};
};
/**
* @deprecated `{@link createNodesV2} is replacing this. Update your plugin to export its own `createNodesV2` function that wraps this one instead.`
*/
export const createNodes: CreateNodes<GradlePluginOptions> = [
gradleConfigGlob,
(configFile, options, context) => {
logger.warn(
'`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will error.'
);
populateGradleReport(context.workspaceRoot);
const gradleReport = getCurrentGradleReport();
const internalCreateNodes = makeCreateNodes(gradleReport, {});
return internalCreateNodes(configFile, options, context);
},
];
function createGradleProject(
gradleReport: GradleReport,
gradleFilePath: string,
options: GradlePluginOptions | undefined,
context: CreateNodesContext
@ -98,7 +150,7 @@ function createGradleProject(
gradleFileToOutputDirsMap,
gradleFileToGradleProjectMap,
gradleProjectToProjectName,
} = getGradleReport();
} = gradleReport;
const gradleProject = gradleFileToGradleProjectMap.get(
gradleFilePath

View File

@ -4,6 +4,7 @@ import { join, relative } from 'node:path';
import { normalizePath, workspaceRoot } from '@nx/devkit';
import { execGradle } from './exec-gradle';
import { hashWithWorkspaceContext } from 'nx/src/utils/workspace-context';
export const fileSeparator = process.platform.startsWith('win')
? 'file:///'
@ -22,14 +23,25 @@ export interface GradleReport {
}
let gradleReportCache: GradleReport;
let gradleCurrentConfigHash: string;
export function invalidateGradleReportCache() {
gradleReportCache = undefined;
export const gradleConfigGlob = '**/build.{gradle.kts,gradle}';
export function getCurrentGradleReport() {
if (!gradleReportCache) {
throw new Error(
'Expected cached gradle report. Please open an issue at https://github.com/nrwl/nx/issues/new/choose'
);
}
return gradleReportCache;
}
export function getGradleReport(): GradleReport {
if (gradleReportCache) {
return gradleReportCache;
export function populateGradleReport(workspaceRoot: string): void {
const gradleConfigHash = hashWithWorkspaceContext(workspaceRoot, [
gradleConfigGlob,
]);
if (gradleReportCache && gradleConfigHash === gradleCurrentConfigHash) {
return;
}
const gradleProjectReportStart = performance.mark(
@ -47,7 +59,6 @@ export function getGradleReport(): GradleReport {
gradleProjectReportEnd.name
);
gradleReportCache = processProjectReports(projectReportLines);
return gradleReportCache;
}
export function processProjectReports(

View File

@ -46,6 +46,10 @@ export type {
CreateNodesFunction,
CreateNodesResult,
CreateNodesContext,
CreateNodesContextV2,
CreateNodesFunctionV2,
CreateNodesResultV2,
CreateNodesV2,
CreateDependencies,
CreateDependenciesContext,
CreateMetadata,
@ -53,6 +57,10 @@ export type {
ProjectsMetadata,
} from './project-graph/plugins';
export { AggregateCreateNodesError } from './project-graph/error-types';
export { createNodesFromFiles } from './project-graph/plugins';
export type {
NxPluginV1,
ProjectTargetConfigurator,

View File

@ -4,7 +4,6 @@ import { ProjectConfiguration } from '../../../config/workspace-json-project-jso
import { toProjectName } from '../../../config/to-project-name';
import { readJsonFile } from '../../../utils/fileutils';
import { NxPluginV2 } from '../../../project-graph/plugins';
import { CreateNodesError } from '../../../project-graph/error-types';
export const ProjectJsonProjectsPlugin: NxPluginV2 = {
name: 'nx/core/project-json',

View File

@ -1,14 +1,14 @@
import { CreateNodesResultWithContext } from './plugins/internal-api';
import {
ConfigurationResult,
ConfigurationSourceMaps,
} from './utils/project-configuration-utils';
import { ProjectConfiguration } from '../config/workspace-json-project-json';
import { ProjectGraph } from '../config/project-graph';
import { CreateNodesFunctionV2 } from './plugins';
export class ProjectGraphError extends Error {
readonly #errors: Array<
| CreateNodesError
| AggregateCreateNodesError
| MergeNodesError
| CreateMetadataError
| ProjectsWithNoNameError
@ -22,7 +22,7 @@ export class ProjectGraphError extends Error {
constructor(
errors: Array<
| CreateNodesError
| AggregateCreateNodesError
| MergeNodesError
| ProjectsWithNoNameError
| MultipleProjectsWithSameNameError
@ -168,7 +168,7 @@ export class ProjectConfigurationsError extends Error {
constructor(
public readonly errors: Array<
| MergeNodesError
| CreateNodesError
| AggregateCreateNodesError
| ProjectsWithNoNameError
| MultipleProjectsWithSameNameError
>,
@ -190,34 +190,39 @@ export function isProjectConfigurationsError(
);
}
export class CreateNodesError extends Error {
file: string;
pluginName: string;
constructor({
file,
pluginName,
error,
}: {
file: string;
pluginName: string;
error: Error;
}) {
const msg = `The "${pluginName}" plugin threw an error while creating nodes from ${file}:`;
super(msg, { cause: error });
this.name = this.constructor.name;
this.file = file;
this.pluginName = pluginName;
this.stack = `${this.message}\n ${error.stack.split('\n').join('\n ')}`;
}
}
/**
* This error should be thrown when a `createNodesV2` function hits a recoverable error.
* It allows Nx to recieve partial results and continue processing for better UX.
*/
export class AggregateCreateNodesError extends Error {
/**
* Throwing this error from a `createNodesV2` function will allow Nx to continue processing and recieve partial results from your plugin.
* @example
* export async function createNodesV2(
* files: string[],
* ) {
* const partialResults = [];
* const errors = [];
* await Promise.all(files.map(async (file) => {
* try {
* const result = await createNodes(file);
* partialResults.push(result);
* } catch (e) {
* errors.push([file, e]);
* }
* }));
* if (errors.length > 0) {
* throw new AggregateCreateNodesError(errors, partialResults);
* }
* return partialResults;
* }
*
* @param errors An array of tuples that represent errors encountered when processing a given file. An example entry might look like ['path/to/project.json', [Error: 'Invalid JSON. Unexpected token 'a' in JSON at position 0]]
* @param partialResults The partial results of the `createNodesV2` function. This should be the results for each file that didn't encounter an issue.
*/
constructor(
public readonly pluginName: string,
public readonly errors: Array<CreateNodesError>,
public readonly partialResults: Array<CreateNodesResultWithContext>
public readonly errors: Array<[file: string | null, error: Error]>,
public readonly partialResults: Awaited<ReturnType<CreateNodesFunctionV2>>
) {
super('Failed to create nodes');
this.name = this.constructor.name;
@ -335,13 +340,6 @@ export function isCreateMetadataError(e: unknown): e is CreateMetadataError {
);
}
export function isCreateNodesError(e: unknown): e is CreateNodesError {
return (
e instanceof CreateNodesError ||
(typeof e === 'object' && 'name' in e && e?.name === CreateNodesError.name)
);
}
export function isAggregateCreateNodesError(
e: unknown
): e is AggregateCreateNodesError {

View File

@ -1,3 +1,4 @@
export * from './public-api';
export { readPluginPackageJson, registerPluginTSTranspiler } from './loader';
export { createNodesFromFiles } from './utils';

View File

@ -13,7 +13,7 @@ import {
CreateDependenciesContext,
CreateMetadata,
CreateMetadataContext,
CreateNodesContext,
CreateNodesContextV2,
CreateNodesResult,
NxPluginV2,
} from './public-api';
@ -21,9 +21,13 @@ import {
ProjectGraph,
ProjectGraphProcessor,
} from '../../config/project-graph';
import { runCreateNodesInParallel } from './utils';
import { loadNxPluginInIsolation } from './isolation';
import { loadNxPlugin, unregisterPluginTSTranspiler } from './loader';
import { createNodesFromFiles } from './utils';
import {
AggregateCreateNodesError,
isAggregateCreateNodesError,
} from '../error-types';
export class LoadedNxPlugin {
readonly name: string;
@ -33,8 +37,10 @@ export class LoadedNxPlugin {
// the result's context.
fn: (
matchedFiles: string[],
context: CreateNodesContext
) => Promise<CreateNodesResultWithContext[]>
context: CreateNodesContextV2
) => Promise<
Array<readonly [plugin: string, file: string, result: CreateNodesResult]>
>
];
readonly createDependencies?: (
context: CreateDependenciesContext
@ -57,14 +63,56 @@ export class LoadedNxPlugin {
this.exclude = pluginDefinition.exclude;
}
if (plugin.createNodes) {
if (plugin.createNodes && !plugin.createNodesV2) {
this.createNodes = [
plugin.createNodes[0],
(files, context) =>
runCreateNodesInParallel(files, plugin, this.options, context),
(configFiles, context) =>
createNodesFromFiles(
plugin.createNodes[1],
configFiles,
this.options,
context
).then((results) => results.map((r) => [this.name, r[0], r[1]])),
];
}
if (plugin.createNodesV2) {
this.createNodes = [
plugin.createNodesV2[0],
async (configFiles, context) => {
const result = await plugin.createNodesV2[1](
configFiles,
this.options,
context
);
return result.map((r) => [this.name, r[0], r[1]]);
},
];
}
if (this.createNodes) {
const inner = this.createNodes[1];
this.createNodes[1] = async (...args) => {
performance.mark(`${plugin.name}:createNodes - start`);
try {
return await inner(...args);
} catch (e) {
if (isAggregateCreateNodesError(e)) {
throw e;
}
// The underlying plugin errored out. We can't know any partial results.
throw new AggregateCreateNodesError([null, e], []);
} finally {
performance.mark(`${plugin.name}:createNodes - end`);
performance.measure(
`${plugin.name}:createNodes`,
`${plugin.name}:createNodes - start`,
`${plugin.name}:createNodes - end`
);
}
};
}
if (plugin.createDependencies) {
this.createDependencies = (context) =>
plugin.createDependencies(this.options, context);

View File

@ -16,15 +16,18 @@ import { RawProjectGraphDependency } from '../project-graph-builder';
/**
* Context for {@link CreateNodesFunction}
*/
export interface CreateNodesContext {
readonly nxJsonConfiguration: NxJsonConfiguration;
readonly workspaceRoot: string;
export interface CreateNodesContext extends CreateNodesContextV2 {
/**
* The subset of configuration files which match the createNodes pattern
*/
readonly configFiles: readonly string[];
}
export interface CreateNodesContextV2 {
readonly nxJsonConfiguration: NxJsonConfiguration;
readonly workspaceRoot: string;
}
/**
* A function which parses a configuration file into a set of nodes.
* Used for creating nodes for the {@link ProjectGraph}
@ -35,6 +38,16 @@ export type CreateNodesFunction<T = unknown> = (
context: CreateNodesContext
) => CreateNodesResult | Promise<CreateNodesResult>;
export type CreateNodesResultV2 = Array<
readonly [configFileSource: string, result: CreateNodesResult]
>;
export type CreateNodesFunctionV2<T = unknown> = (
projectConfigurationFiles: readonly string[],
options: T | undefined,
context: CreateNodesContextV2
) => CreateNodesResultV2 | Promise<CreateNodesResultV2>;
export type Optional<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
export interface CreateNodesResult {
@ -51,12 +64,31 @@ export interface CreateNodesResult {
/**
* A pair of file patterns and {@link CreateNodesFunction}
*
* Nx 19.2+: Both original `CreateNodes` and `CreateNodesV2` are supported. Nx will only invoke `CreateNodesV2` if it is present.
* Nx 20.X : The `CreateNodesV2` will be the only supported API. This typing will still exist, but be identical to `CreateNodesV2`.
Nx **will not** invoke the original `plugin.createNodes` callback. This should give plugin authors a window to transition.
Plugin authors should update their plugin's `createNodes` function to align with `CreateNodesV2` / the updated `CreateNodes`.
The plugin should contain something like: `export createNodes = createNodesV2;` during this period. This will allow the plugin
to maintain compatibility with Nx 19.2 and up.
* Nx 21.X : The `CreateNodesV2` typing will be removed, as it has replaced `CreateNodes`.
*
* @deprecated Use {@link CreateNodesV2} instead. CreateNodesV2 will replace this API. Read more about the transition above.
*/
export type CreateNodes<T = unknown> = readonly [
projectFilePattern: string,
createNodesFunction: CreateNodesFunction<T>
];
/**
* A pair of file patterns and {@link CreateNodesFunctionV2}
* In Nx 20 {@link CreateNodes} will be replaced with this type. In Nx 21, this type will be removed.
*/
export type CreateNodesV2<T = unknown> = readonly [
projectFilePattern: string,
createNodesFunction: CreateNodesFunctionV2<T>
];
/**
* Context for {@link CreateDependencies}
*/
@ -123,9 +155,19 @@ export type NxPluginV2<TOptions = unknown> = {
/**
* Provides a file pattern and function that retrieves configuration info from
* those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFile }
*
* @deprecated Use {@link createNodesV2} instead. In Nx 20 support for calling createNodes with a single file for the first argument will be removed.
*/
createNodes?: CreateNodes<TOptions>;
/**
* Provides a file pattern and function that retrieves configuration info from
* those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFiles }
*
* In Nx 20 {@link createNodes} will be replaced with this property. In Nx 21, this property will be removed.
*/
createNodesV2?: CreateNodesV2<TOptions>;
/**
* Provides a function to analyze files to create dependencies for the {@link ProjectGraph}
*/

View File

@ -1,19 +1,16 @@
import { runCreateNodesInParallel } from './utils';
import { isAggregateCreateNodesError } from '../error-types';
import { createNodesFromFiles } from './utils';
const configFiles = ['file1', 'file2'] as const;
const context = {
file: 'file1',
nxJsonConfiguration: {},
workspaceRoot: '',
configFiles,
} as const;
describe('createNodesInParallel', () => {
describe('createNodesFromFiles', () => {
it('should return results with context', async () => {
const plugin = {
name: 'test',
createNodes: [
const createNodes = [
'*/**/*',
async (file: string) => {
return {
@ -24,100 +21,166 @@ describe('createNodesInParallel', () => {
},
};
},
],
} as const;
] as const;
const options = {};
const results = await runCreateNodesInParallel(
const results = await createNodesFromFiles(
createNodes[1],
configFiles,
plugin,
options,
context
);
expect(results).toMatchInlineSnapshot(`
[
[
"file1",
{
"file": "file1",
"pluginName": "test",
"projects": {
"file1": {
"root": "file1",
},
},
},
],
[
"file2",
{
"file": "file2",
"pluginName": "test",
"projects": {
"file2": {
"root": "file2",
},
},
},
],
]
`);
});
it('should handle async errors', async () => {
const plugin = {
name: 'test',
createNodes: [
const createNodes = [
'*/**/*',
async () => {
throw new Error('Async Error');
},
],
} as const;
] as const;
const options = {};
const error = await runCreateNodesInParallel(
let error;
await createNodesFromFiles(
createNodes[1],
configFiles,
plugin,
options,
context
).catch((e) => e);
).catch((e) => (error = e));
expect(error).toMatchInlineSnapshot(
`[AggregateCreateNodesError: Failed to create nodes]`
);
const isAggregateError = isAggregateCreateNodesError(error);
expect(isAggregateError).toBe(true);
if (isAggregateCreateNodesError(error)) {
expect(error.errors).toMatchInlineSnapshot(`
[
[CreateNodesError: The "test" plugin threw an error while creating nodes from file1:],
[CreateNodesError: The "test" plugin threw an error while creating nodes from file2:],
[
"file1",
[Error: Async Error],
],
[
"file2",
[Error: Async Error],
],
]
`);
}
});
it('should handle sync errors', async () => {
const plugin = {
name: 'test',
createNodes: [
const createNodes = [
'*/**/*',
() => {
throw new Error('Sync Error');
},
],
} as const;
] as const;
const options = {};
const error = await runCreateNodesInParallel(
let error;
await createNodesFromFiles(
createNodes[1],
configFiles,
plugin,
options,
context
).catch((e) => e);
).catch((e) => (error = e));
expect(error).toMatchInlineSnapshot(
`[AggregateCreateNodesError: Failed to create nodes]`
);
const isAggregateError = isAggregateCreateNodesError(error);
expect(isAggregateError).toBe(true);
if (isAggregateCreateNodesError(error)) {
expect(error.errors).toMatchInlineSnapshot(`
[
[CreateNodesError: The "test" plugin threw an error while creating nodes from file1:],
[CreateNodesError: The "test" plugin threw an error while creating nodes from file2:],
[
"file1",
[Error: Sync Error],
],
[
"file2",
[Error: Sync Error],
],
]
`);
}
});
it('should handle partial errors', async () => {
const createNodes = [
'*/**/*',
async (file: string) => {
if (file === 'file1') {
throw new Error('Error');
}
return {
projects: {
[file]: {
root: file,
},
},
};
},
] as const;
const options = {};
let error;
await createNodesFromFiles(
createNodes[1],
configFiles,
options,
context
).catch((e) => (error = e));
const isAggregateError = isAggregateCreateNodesError(error);
expect(isAggregateError).toBe(true);
if (isAggregateCreateNodesError(error)) {
expect(error.errors).toMatchInlineSnapshot(`
[
[
"file1",
[Error: Error],
],
]
`);
expect(error.partialResults).toMatchInlineSnapshot(`
[
[
"file2",
{
"projects": {
"file2": {
"root": "file2",
},
},
},
],
]
`);
}
});
});

View File

@ -4,19 +4,16 @@ import { toProjectName } from '../../config/to-project-name';
import { combineGlobPatterns } from '../../utils/globs';
import type { NxPluginV1 } from '../../utils/nx-plugin.deprecated';
import type {
CreateNodesResultWithContext,
LoadedNxPlugin,
NormalizedPlugin,
} from './internal-api';
import type { LoadedNxPlugin, NormalizedPlugin } from './internal-api';
import {
CreateNodesContextV2,
CreateNodesFunction,
CreateNodesFunctionV2,
CreateNodesResult,
type CreateNodesContext,
type NxPlugin,
type NxPluginV2,
} from './public-api';
import { AggregateCreateNodesError, CreateNodesError } from '../error-types';
import { performance } from 'perf_hooks';
import { AggregateCreateNodesError } from '../error-types';
export function isNxPluginV2(plugin: NxPlugin): plugin is NxPluginV2 {
return 'createNodes' in plugin || 'createDependencies' in plugin;
@ -54,49 +51,37 @@ export function normalizeNxPlugin(plugin: NxPlugin): NormalizedPlugin {
return plugin;
}
export async function runCreateNodesInParallel(
export type AsyncFn<T extends Function> = T extends (
...args: infer A
) => infer R
? (...args: A) => Promise<Awaited<R>>
: never;
export async function createNodesFromFiles<T = unknown>(
createNodes: CreateNodesFunction,
configFiles: readonly string[],
plugin: NormalizedPlugin,
options: unknown,
context: CreateNodesContext
): Promise<CreateNodesResultWithContext[]> {
performance.mark(`${plugin.name}:createNodes - start`);
options: T,
context: CreateNodesContextV2
) {
const results: Array<[file: string, value: CreateNodesResult]> = [];
const errors: Array<[file: string, error: Error]> = [];
const errors: CreateNodesError[] = [];
const results: CreateNodesResultWithContext[] = [];
const promises: Array<Promise<void>> = configFiles.map(async (file) => {
await Promise.all(
configFiles.map(async (file) => {
try {
const value = await plugin.createNodes[1](file, options, context);
if (value) {
results.push({
...value,
file,
pluginName: plugin.name,
const value = await createNodes(file, options, {
...context,
configFiles,
});
}
results.push([file, value] as const);
} catch (e) {
errors.push(
new CreateNodesError({
error: e,
pluginName: plugin.name,
file,
errors.push([file, e] as const);
}
})
);
}
});
await Promise.all(promises).then(() => {
performance.mark(`${plugin.name}:createNodes - end`);
performance.measure(
`${plugin.name}:createNodes`,
`${plugin.name}:createNodes - start`,
`${plugin.name}:createNodes - end`
);
});
if (errors.length > 0) {
throw new AggregateCreateNodesError(plugin.name, errors, results);
throw new AggregateCreateNodesError(errors, results);
}
return results;
}

View File

@ -17,15 +17,10 @@ import {
import { minimatch } from 'minimatch';
import { join } from 'path';
import { performance } from 'perf_hooks';
import { LoadedNxPlugin } from '../plugins/internal-api';
import {
CreateNodesResultWithContext,
LoadedNxPlugin,
} from '../plugins/internal-api';
import {
CreateNodesError,
MergeNodesError,
ProjectConfigurationsError,
isAggregateCreateNodesError,
ProjectsWithNoNameError,
MultipleProjectsWithSameNameError,
isMultipleProjectsWithSameNameError,
@ -34,7 +29,10 @@ import {
ProjectWithExistingNameError,
isProjectWithExistingNameError,
isProjectWithNoNameError,
isAggregateCreateNodesError,
AggregateCreateNodesError,
} from '../error-types';
import { CreateNodesResult } from '../plugins';
export type SourceInformation = [file: string | null, plugin: string];
export type ConfigurationSourceMaps = Record<
@ -347,9 +345,9 @@ export async function createProjectConfigurations(
): Promise<ConfigurationResult> {
performance.mark('build-project-configs:start');
const results: Array<Promise<Array<CreateNodesResultWithContext>>> = [];
const results: Array<ReturnType<LoadedNxPlugin['createNodes'][1]>> = [];
const errors: Array<
| CreateNodesError
| AggregateCreateNodesError
| MergeNodesError
| ProjectsWithNoNameError
| MultipleProjectsWithSameNameError
@ -357,10 +355,10 @@ export async function createProjectConfigurations(
// We iterate over plugins first - this ensures that plugins specified first take precedence.
for (const {
name: pluginName,
createNodes: createNodesTuple,
include,
exclude,
name: pluginName,
} of plugins) {
const [pattern, createNodes] = createNodesTuple ?? [];
@ -368,48 +366,85 @@ export async function createProjectConfigurations(
continue;
}
const matchingConfigFiles: string[] = [];
for (const file of projectFiles) {
if (minimatch(file, pattern, { dot: true })) {
if (include) {
const included = include.some((includedPattern) =>
minimatch(file, includedPattern, { dot: true })
const matchingConfigFiles: string[] = findMatchingConfigFiles(
projectFiles,
pattern,
include,
exclude
);
if (!included) {
continue;
}
}
if (exclude) {
const excluded = exclude.some((excludedPattern) =>
minimatch(file, excludedPattern, { dot: true })
);
if (excluded) {
continue;
}
}
matchingConfigFiles.push(file);
}
}
let r = createNodes(matchingConfigFiles, {
nxJsonConfiguration: nxJson,
workspaceRoot: root,
configFiles: matchingConfigFiles,
}).catch((e) => {
if (isAggregateCreateNodesError(e)) {
errors.push(...e.errors);
return e.partialResults;
} else {
throw e;
}
}).catch((e: Error) => {
const errorBodyLines = [
`An error occurred while processing files for the ${pluginName} plugin.`,
];
const error: AggregateCreateNodesError = isAggregateCreateNodesError(e)
? // This is an expected error if something goes wrong while processing files.
e
: // This represents a single plugin erroring out with a hard error.
new AggregateCreateNodesError([[null, e]], []);
errorBodyLines.push(
...error.errors.map(([file, e]) => ` - ${file}: ${e.message}`)
);
error.message = errorBodyLines.join('\n');
// This represents a single plugin erroring out with a hard error.
errors.push(error);
// The plugin didn't return partial results, so we return an empty array.
return error.partialResults.map((r) => [pluginName, r[0], r[1]] as const);
});
results.push(r);
}
return Promise.all(results).then((results) => {
const { projectRootMap, externalNodes, rootMap, configurationSourceMaps } =
mergeCreateNodesResults(results, errors);
performance.mark('build-project-configs:end');
performance.measure(
'build-project-configs',
'build-project-configs:start',
'build-project-configs:end'
);
if (errors.length === 0) {
return {
projects: projectRootMap,
externalNodes,
projectRootMap: rootMap,
sourceMaps: configurationSourceMaps,
matchingProjectFiles: projectFiles,
};
} else {
throw new ProjectConfigurationsError(errors, {
projects: projectRootMap,
externalNodes,
projectRootMap: rootMap,
sourceMaps: configurationSourceMaps,
matchingProjectFiles: projectFiles,
});
}
});
}
function mergeCreateNodesResults(
results: (readonly [
plugin: string,
file: string,
result: CreateNodesResult
])[][],
errors: (
| AggregateCreateNodesError
| MergeNodesError
| ProjectsWithNoNameError
| MultipleProjectsWithSameNameError
)[]
) {
performance.mark('createNodes:merge - start');
const projectRootMap: Record<string, ProjectConfiguration> = {};
const externalNodes: Record<string, ProjectGraphExternalNode> = {};
@ -419,12 +454,10 @@ export async function createProjectConfigurations(
> = {};
for (const result of results.flat()) {
const {
projects: projectNodes,
externalNodes: pluginExternalNodes,
file,
pluginName,
} = result;
const [file, pluginName, nodes] = result;
const { projects: projectNodes, externalNodes: pluginExternalNodes } =
nodes;
const sourceInfo: SourceInformation = [file, pluginName];
@ -482,32 +515,41 @@ export async function createProjectConfigurations(
'createNodes:merge - start',
'createNodes:merge - end'
);
performance.mark('build-project-configs:end');
performance.measure(
'build-project-configs',
'build-project-configs:start',
'build-project-configs:end'
);
if (errors.length === 0) {
return {
projects: projectRootMap,
externalNodes,
projectRootMap: rootMap,
sourceMaps: configurationSourceMaps,
matchingProjectFiles: projectFiles,
};
} else {
throw new ProjectConfigurationsError(errors, {
projects: projectRootMap,
externalNodes,
projectRootMap: rootMap,
sourceMaps: configurationSourceMaps,
matchingProjectFiles: projectFiles,
});
return { projectRootMap, externalNodes, rootMap, configurationSourceMaps };
}
});
function findMatchingConfigFiles(
projectFiles: string[],
pattern: string,
include: string[],
exclude: string[]
) {
const matchingConfigFiles: string[] = [];
for (const file of projectFiles) {
if (minimatch(file, pattern, { dot: true })) {
if (include) {
const included = include.some((includedPattern) =>
minimatch(file, includedPattern, { dot: true })
);
if (!included) {
continue;
}
}
if (exclude) {
const excluded = exclude.some((excludedPattern) =>
minimatch(file, excludedPattern, { dot: true })
);
if (excluded) {
continue;
}
}
matchingConfigFiles.push(file);
}
}
return matchingConfigFiles;
}
export function readProjectConfigurationsFromRootMap(