From 353d8d089dfecd7df466f8c2ee836b20aad0942c Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Wed, 30 Aug 2023 13:26:41 -0500 Subject: [PATCH] cleanup(core): move js lockfile parsing to v2 plugin (#18779) --- docs/generated/devkit/CreateNodes.md | 2 +- .../utils/package-json/update-package-json.ts | 18 +- .../next/src/executors/build/build.impl.ts | 18 +- packages/nx/src/plugins/js/index.ts | 266 +++++------ .../nx/src/plugins/js/lock-file/lock-file.ts | 171 +++---- .../plugins/js/lock-file/npm-parser.spec.ts | 307 +++++++++++-- .../nx/src/plugins/js/lock-file/npm-parser.ts | 136 ++++-- .../plugins/js/lock-file/pnpm-parser.spec.ts | 263 +++++++++-- .../src/plugins/js/lock-file/pnpm-parser.ts | 95 ++-- .../plugins/js/lock-file/yarn-parser.spec.ts | 430 ++++++++++++++---- .../src/plugins/js/lock-file/yarn-parser.ts | 100 +++- packages/nx/src/plugins/js/utils/config.ts | 80 ++++ packages/nx/src/utils/nx-plugin.ts | 44 +- .../vite/src/executors/build/build.impl.ts | 18 +- .../plugins/generate-package-json-plugin.ts | 14 +- 15 files changed, 1452 insertions(+), 510 deletions(-) create mode 100644 packages/nx/src/plugins/js/utils/config.ts diff --git a/docs/generated/devkit/CreateNodes.md b/docs/generated/devkit/CreateNodes.md index 64f95fff08..0264eb5535 100644 --- a/docs/generated/devkit/CreateNodes.md +++ b/docs/generated/devkit/CreateNodes.md @@ -1,5 +1,5 @@ # Type alias: CreateNodes -Ƭ **CreateNodes**: [projectFilePattern: string, createNodesFunction: CreateNodesFunction] +Ƭ **CreateNodes**: readonly [projectFilePattern: string, createNodesFunction: CreateNodesFunction] A pair of file patterns and [CreateNodesFunction](../../devkit/documents/CreateNodesFunction) diff --git a/packages/js/src/utils/package-json/update-package-json.ts b/packages/js/src/utils/package-json/update-package-json.ts index f78972a0e8..a516cef36d 100644 --- a/packages/js/src/utils/package-json/update-package-json.ts +++ b/packages/js/src/utils/package-json/update-package-json.ts @@ -7,6 +7,7 @@ import { import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json'; import { + detectPackageManager, ExecutorContext, getOutputsForTargetAndConfiguration, joinPathFragments, @@ -100,10 +101,19 @@ export function updatePackageJson( writeJsonFile(`${options.outputPath}/package.json`, packageJson); if (options.generateLockfile) { - const lockFile = createLockFile(packageJson); - writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, { - encoding: 'utf-8', - }); + const packageManager = detectPackageManager(context.root); + const lockFile = createLockFile( + packageJson, + context.projectGraph, + packageManager + ); + writeFileSync( + `${options.outputPath}/${getLockFileName(packageManager)}`, + lockFile, + { + encoding: 'utf-8', + } + ); } } diff --git a/packages/next/src/executors/build/build.impl.ts b/packages/next/src/executors/build/build.impl.ts index 10b8d44b0a..be2923efe4 100644 --- a/packages/next/src/executors/build/build.impl.ts +++ b/packages/next/src/executors/build/build.impl.ts @@ -1,4 +1,5 @@ import { + detectPackageManager, ExecutorContext, logger, readJsonFile, @@ -83,10 +84,19 @@ export default async function buildExecutor( writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson); if (options.generateLockfile) { - const lockFile = createLockFile(builtPackageJson); - writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, { - encoding: 'utf-8', - }); + const packageManager = detectPackageManager(context.root); + const lockFile = createLockFile( + builtPackageJson, + context.projectGraph, + packageManager + ); + writeFileSync( + `${options.outputPath}/${getLockFileName(packageManager)}`, + lockFile, + { + encoding: 'utf-8', + } + ); } // If output path is different from source path, then copy over the config and public files. diff --git a/packages/nx/src/plugins/js/index.ts b/packages/nx/src/plugins/js/index.ts index 898ef77872..62c13b89a3 100644 --- a/packages/nx/src/plugins/js/index.ts +++ b/packages/nx/src/plugins/js/index.ts @@ -1,98 +1,123 @@ -import { - ProjectGraph, - ProjectGraphProcessor, -} from '../../config/project-graph'; -import { - ProjectGraphBuilder, - ProjectGraphDependencyWithFile, -} from '../../project-graph/project-graph-builder'; -import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies'; -import { readNxJson } from '../../config/configuration'; -import { fileExists, readJsonFile } from '../../utils/fileutils'; -import { PackageJson } from '../../utils/package-json'; -import { - lockFileExists, - lockFileHash, - parseLockFile, -} from './lock-file/lock-file'; -import { NrwlJsPluginConfig, NxJsonConfiguration } from '../../config/nx-json'; -import { dirname, join } from 'path'; -import { projectGraphCacheDirectory } from '../../utils/cache-directory'; -import { existsSync, readFileSync, writeFileSync } from 'fs'; -import { workspaceRoot } from '../../utils/workspace-root'; +import { readFileSync, writeFileSync } from 'fs'; import { ensureDirSync } from 'fs-extra'; +import { dirname, join } from 'path'; import { performance } from 'perf_hooks'; +import { ProjectGraph } from '../../config/project-graph'; +import { projectGraphCacheDirectory } from '../../utils/cache-directory'; +import { combineGlobPatterns } from '../../utils/globs'; import { CreateDependencies, CreateDependenciesContext, + CreateNodes, } from '../../utils/nx-plugin'; +import { + getLockFileDependencies, + getLockFileName, + getLockFileNodes, + lockFileExists, + LOCKFILES, +} from './lock-file/lock-file'; +import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies'; +import { jsPluginConfig } from './utils/config'; +import { ProjectGraphDependencyWithFile } from '../../project-graph/project-graph-builder'; +import { hashArray } from '../../hasher/file-hasher'; +import { detectPackageManager } from '../../utils/package-manager'; +import { workspaceRoot } from '../../utils/workspace-root'; -const createDependencies: CreateDependencies = (context) => { - const pluginConfig = jsPluginConfig(context.nxJsonConfiguration); +export const name = 'nx-js-graph-plugin'; + +interface ParsedLockFile { + externalNodes?: ProjectGraph['externalNodes']; + dependencies?: ProjectGraphDependencyWithFile[]; +} +let parsedLockFile: ParsedLockFile = {}; + +export const createNodes: CreateNodes = [ + // Look for all lockfiles + combineGlobPatterns(LOCKFILES), + (lockFile, context) => { + const pluginConfig = jsPluginConfig(context.nxJsonConfiguration); + if (!pluginConfig.analyzePackageJson) { + return {}; + } + + const packageManager = detectPackageManager(workspaceRoot); + + // Only process the correct lockfile + if (lockFile !== getLockFileName(packageManager)) { + return {}; + } + + const lockFilePath = join(workspaceRoot, lockFile); + const lockFileContents = readFileSync(lockFilePath).toString(); + const lockFileHash = hashArray([lockFileContents]); + + if (!lockFileNeedsReprocessing(lockFileHash)) { + return { + externalNodes: readCachedParsedLockFile().externalNodes, + }; + } + + const externalNodes = getLockFileNodes( + packageManager, + lockFileContents, + lockFileHash + ); + parsedLockFile.externalNodes = externalNodes; + return { + externalNodes, + }; + }, +]; + +export const createDependencies: CreateDependencies = ( + ctx: CreateDependenciesContext +) => { + const pluginConfig = jsPluginConfig(ctx.nxJsonConfiguration); + + const packageManager = detectPackageManager(workspaceRoot); + + let lockfileDependencies: ProjectGraphDependencyWithFile[] = []; + // lockfile may not exist yet + if ( + pluginConfig.analyzePackageJson && + lockFileExists(packageManager) && + parsedLockFile + ) { + const lockFilePath = join(workspaceRoot, getLockFileName(packageManager)); + const lockFileContents = readFileSync(lockFilePath).toString(); + const lockFileHash = hashArray([lockFileContents]); + + if (!lockFileNeedsReprocessing(lockFileHash)) { + lockfileDependencies = readCachedParsedLockFile().dependencies ?? []; + } else { + lockfileDependencies = getLockFileDependencies( + packageManager, + lockFileContents, + lockFileHash, + ctx.graph + ); + + parsedLockFile.dependencies = lockfileDependencies; + + writeLastProcessedLockfileHash(lockFileHash, parsedLockFile); + } + } performance.mark('build typescript dependencies - start'); - const dependencies = buildExplicitDependencies(pluginConfig, context); + const explicitProjectDependencies = buildExplicitDependencies( + pluginConfig, + ctx + ); performance.mark('build typescript dependencies - end'); performance.measure( 'build typescript dependencies', 'build typescript dependencies - start', 'build typescript dependencies - end' ); - return dependencies; + return lockfileDependencies.concat(explicitProjectDependencies); }; -export const processProjectGraph: ProjectGraphProcessor = async ( - graph, - context -) => { - const builder = new ProjectGraphBuilder(graph, context.fileMap); - const pluginConfig = jsPluginConfig(readNxJson()); - - if (pluginConfig.analyzePackageJson) { - if ( - // during the create-nx-workspace lock file might not exists yet - lockFileExists() && - pluginConfig.analyzeLockfile - ) { - const lockHash = lockFileHash(); - let parsedLockFile: ProjectGraph; - if (lockFileNeedsReprocessing(lockHash)) { - parsedLockFile = parseLockFile(); - writeLastProcessedLockfileHash(lockHash, parsedLockFile); - } else { - parsedLockFile = readParsedLockFile(); - } - builder.mergeProjectGraph(parsedLockFile); - } - } - - const createDependenciesContext: CreateDependenciesContext = { - ...context, - graph, - }; - - const dependencies = createDependencies( - createDependenciesContext - ) as ProjectGraphDependencyWithFile[]; - - for (const dep of dependencies) { - builder.addDependency( - dep.source, - dep.target, - dep.dependencyType, - dep.sourceFile - ); - } - - return builder.getUpdatedProjectGraph(); -}; - -const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash'); -const parsedLockFile = join( - projectGraphCacheDirectory, - 'parsed-lock-file.json' -); - function lockFileNeedsReprocessing(lockHash: string) { try { return readFileSync(lockFileHashFile).toString() !== lockHash; @@ -101,82 +126,21 @@ function lockFileNeedsReprocessing(lockHash: string) { } } -function writeLastProcessedLockfileHash(hash: string, lockFile: ProjectGraph) { +function writeLastProcessedLockfileHash( + hash: string, + lockFile: ParsedLockFile +) { ensureDirSync(dirname(lockFileHashFile)); - writeFileSync(parsedLockFile, JSON.stringify(lockFile, null, 2)); + writeFileSync(cachedParsedLockFile, JSON.stringify(lockFile, null, 2)); writeFileSync(lockFileHashFile, hash); } -function readParsedLockFile(): ProjectGraph { - return JSON.parse(readFileSync(parsedLockFile).toString()); +function readCachedParsedLockFile(): ParsedLockFile { + return JSON.parse(readFileSync(cachedParsedLockFile).toString()); } -function jsPluginConfig( - nxJson: NxJsonConfiguration -): Required { - const nxJsonConfig: NrwlJsPluginConfig = - nxJson?.pluginsConfig?.['@nx/js'] ?? nxJson?.pluginsConfig?.['@nrwl/js']; - - // using lerna _before_ installing deps is causing an issue when parsing lockfile. - // See: https://github.com/lerna/lerna/issues/3807 - // Note that previous attempt to fix this caused issues with Nx itself, thus we're checking - // for Lerna explicitly. - // See: https://github.com/nrwl/nx/pull/18784/commits/5416138e1ddc1945d5b289672dfb468e8c544e14 - const analyzeLockfile = - !existsSync(join(workspaceRoot, 'lerna.json')) || - existsSync(join(workspaceRoot, 'nx.json')); - - if (nxJsonConfig) { - return { - analyzePackageJson: true, - analyzeSourceFiles: true, - analyzeLockfile, - ...nxJsonConfig, - }; - } - - if (!fileExists(join(workspaceRoot, 'package.json'))) { - return { - analyzeLockfile: false, - analyzePackageJson: false, - analyzeSourceFiles: false, - }; - } - - const packageJson = readJsonFile( - join(workspaceRoot, 'package.json') - ); - - const packageJsonDeps = { - ...packageJson.dependencies, - ...packageJson.devDependencies, - }; - if ( - packageJsonDeps['@nx/workspace'] || - packageJsonDeps['@nx/js'] || - packageJsonDeps['@nx/node'] || - packageJsonDeps['@nx/next'] || - packageJsonDeps['@nx/react'] || - packageJsonDeps['@nx/angular'] || - packageJsonDeps['@nx/web'] || - packageJsonDeps['@nrwl/workspace'] || - packageJsonDeps['@nrwl/js'] || - packageJsonDeps['@nrwl/node'] || - packageJsonDeps['@nrwl/next'] || - packageJsonDeps['@nrwl/react'] || - packageJsonDeps['@nrwl/angular'] || - packageJsonDeps['@nrwl/web'] - ) { - return { - analyzePackageJson: true, - analyzeLockfile, - analyzeSourceFiles: true, - }; - } else { - return { - analyzePackageJson: true, - analyzeLockfile, - analyzeSourceFiles: false, - }; - } -} +const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash'); +const cachedParsedLockFile = join( + projectGraphCacheDirectory, + 'parsed-lock-file.json' +); diff --git a/packages/nx/src/plugins/js/lock-file/lock-file.ts b/packages/nx/src/plugins/js/lock-file/lock-file.ts index 8dafa3d49f..513078ea91 100644 --- a/packages/nx/src/plugins/js/lock-file/lock-file.ts +++ b/packages/nx/src/plugins/js/lock-file/lock-file.ts @@ -11,15 +11,32 @@ import { PackageManager, } from '../../../utils/package-manager'; import { workspaceRoot } from '../../../utils/workspace-root'; -import { ProjectGraph } from '../../../config/project-graph'; -import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; +import { + ProjectGraph, + ProjectGraphExternalNode, +} from '../../../config/project-graph'; +import { + ProjectGraphBuilder, + ProjectGraphDependencyWithFile, +} from '../../../project-graph/project-graph-builder'; import { PackageJson } from '../../../utils/package-json'; -import { hashArray } from '../../../hasher/file-hasher'; import { output } from '../../../utils/output'; -import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser'; -import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser'; -import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser'; +import { + getNpmLockfileNodes, + stringifyNpmLockfile, + getNpmLockfileDependencies, +} from './npm-parser'; +import { + getPnpmLockfileDependencies, + getPnpmLockfileNodes, + stringifyPnpmLockfile, +} from './pnpm-parser'; +import { + getYarnLockfileDependencies, + getYarnLockfileNodes, + stringifyYarnLockfile, +} from './yarn-parser'; import { pruneProjectGraph } from './project-graph-pruning'; import { normalizePackageJson } from './utils/package-json'; import { readJsonFile } from '../../../utils/fileutils'; @@ -27,17 +44,75 @@ import { readJsonFile } from '../../../utils/fileutils'; const YARN_LOCK_FILE = 'yarn.lock'; const NPM_LOCK_FILE = 'package-lock.json'; const PNPM_LOCK_FILE = 'pnpm-lock.yaml'; +export const LOCKFILES = [YARN_LOCK_FILE, NPM_LOCK_FILE, PNPM_LOCK_FILE]; const YARN_LOCK_PATH = join(workspaceRoot, YARN_LOCK_FILE); const NPM_LOCK_PATH = join(workspaceRoot, NPM_LOCK_FILE); const PNPM_LOCK_PATH = join(workspaceRoot, PNPM_LOCK_FILE); /** - * Check if lock file exists + * Parses lock file and maps dependencies and metadata to {@link LockFileGraph} */ -export function lockFileExists( - packageManager: PackageManager = detectPackageManager(workspaceRoot) -): boolean { +export function getLockFileNodes( + packageManager: PackageManager, + contents: string, + lockFileHash: string +): Record { + try { + if (packageManager === 'yarn') { + const packageJson = readJsonFile('package.json'); + return getYarnLockfileNodes(contents, lockFileHash, packageJson); + } + if (packageManager === 'pnpm') { + return getPnpmLockfileNodes(contents, lockFileHash); + } + if (packageManager === 'npm') { + return getNpmLockfileNodes(contents, lockFileHash); + } + } catch (e) { + if (!isPostInstallProcess()) { + output.error({ + title: `Failed to parse ${packageManager} lockfile`, + bodyLines: errorBodyLines(e), + }); + } + return; + } + throw new Error(`Unknown package manager: ${packageManager}`); +} + +/** + * Parses lock file and maps dependencies and metadata to {@link LockFileGraph} + */ +export function getLockFileDependencies( + packageManager: PackageManager, + contents: string, + lockFileHash: string, + projectGraph: ProjectGraph +): ProjectGraphDependencyWithFile[] { + try { + if (packageManager === 'yarn') { + return getYarnLockfileDependencies(contents, lockFileHash, projectGraph); + } + if (packageManager === 'pnpm') { + return getPnpmLockfileDependencies(contents, lockFileHash, projectGraph); + } + if (packageManager === 'npm') { + return getNpmLockfileDependencies(contents, lockFileHash, projectGraph); + } + } catch (e) { + if (!isPostInstallProcess()) { + output.error({ + title: `Failed to parse ${packageManager} lockfile`, + bodyLines: errorBodyLines(e), + }); + } + return; + } + throw new Error(`Unknown package manager: ${packageManager}`); +} + +export function lockFileExists(packageManager: PackageManager): boolean { if (packageManager === 'yarn') { return existsSync(YARN_LOCK_PATH); } @@ -52,75 +127,12 @@ export function lockFileExists( ); } -/** - * Hashes lock file content - */ -export function lockFileHash( - packageManager: PackageManager = detectPackageManager(workspaceRoot) -): string { - let content: string; - if (packageManager === 'yarn') { - content = readFileSync(YARN_LOCK_PATH, 'utf8'); - } - if (packageManager === 'pnpm') { - content = readFileSync(PNPM_LOCK_PATH, 'utf8'); - } - if (packageManager === 'npm') { - content = readFileSync(NPM_LOCK_PATH, 'utf8'); - } - if (content) { - return hashArray([content]); - } else { - throw new Error( - `Unknown package manager ${packageManager} or lock file missing` - ); - } -} - -/** - * Parses lock file and maps dependencies and metadata to {@link LockFileGraph} - */ -export function parseLockFile( - packageManager: PackageManager = detectPackageManager(workspaceRoot) -): ProjectGraph { - const builder = new ProjectGraphBuilder(null, null); - try { - if (packageManager === 'yarn') { - const content = readFileSync(YARN_LOCK_PATH, 'utf8'); - const packageJson = readJsonFile('package.json'); - parseYarnLockfile(content, packageJson, builder); - return builder.getUpdatedProjectGraph(); - } - if (packageManager === 'pnpm') { - const content = readFileSync(PNPM_LOCK_PATH, 'utf8'); - parsePnpmLockfile(content, builder); - return builder.getUpdatedProjectGraph(); - } - if (packageManager === 'npm') { - const content = readFileSync(NPM_LOCK_PATH, 'utf8'); - parseNpmLockfile(content, builder); - return builder.getUpdatedProjectGraph(); - } - } catch (e) { - if (!isPostInstallProcess()) { - output.error({ - title: `Failed to parse ${packageManager} lockfile`, - bodyLines: errorBodyLines(e), - }); - } - return; - } - throw new Error(`Unknown package manager: ${packageManager}`); -} - /** * Returns lock file name based on the detected package manager in the root * @param packageManager * @returns */ -export function getLockFileName( - packageManager: PackageManager = detectPackageManager(workspaceRoot) -): string { +export function getLockFileName(packageManager: PackageManager): string { if (packageManager === 'yarn') { return YARN_LOCK_FILE; } @@ -143,31 +155,22 @@ export function getLockFileName( */ export function createLockFile( packageJson: PackageJson, + graph: ProjectGraph, packageManager: PackageManager = detectPackageManager(workspaceRoot) ): string { const normalizedPackageJson = normalizePackageJson(packageJson); const content = readFileSync(getLockFileName(packageManager), 'utf8'); - const rootPackageJson = readJsonFile('package.json'); - - const builder = new ProjectGraphBuilder(); try { if (packageManager === 'yarn') { - parseYarnLockfile(content, rootPackageJson, builder); - const graph = builder.getUpdatedProjectGraph(); const prunedGraph = pruneProjectGraph(graph, packageJson); return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson); } if (packageManager === 'pnpm') { - parsePnpmLockfile(content, builder); - const graph = builder.getUpdatedProjectGraph(); const prunedGraph = pruneProjectGraph(graph, packageJson); return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson); } if (packageManager === 'npm') { - parseNpmLockfile(content, builder); - - const graph = builder.getUpdatedProjectGraph(); const prunedGraph = pruneProjectGraph(graph, packageJson); return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson); } diff --git a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts index 2c10cf6aab..1de96292f5 100644 --- a/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts @@ -1,5 +1,9 @@ import { joinPathFragments } from '../../../utils/path'; -import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser'; +import { + getNpmLockfileDependencies, + getNpmLockfileNodes, + stringifyNpmLockfile, +} from './npm-parser'; import { pruneProjectGraph } from './project-graph-pruning'; import { vol } from 'memfs'; import { ProjectGraph } from '../../../config/project-graph'; @@ -27,8 +31,31 @@ describe('NPM lock file utility', () => { let graph: ProjectGraph; beforeEach(() => { - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootLockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } graph = builder.getUpdatedProjectGraph(); }); @@ -47,8 +74,31 @@ describe('NPM lock file utility', () => { )); // this is original generated lock file - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(appLockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(appLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(appLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const appGraph = builder.getUpdatedProjectGraph(); expect(Object.keys(appGraph.externalNodes).length).toEqual(984); @@ -95,8 +145,31 @@ describe('NPM lock file utility', () => { '__fixtures__/auxiliary-packages/package-lock.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootLockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202 @@ -154,9 +227,33 @@ describe('NPM lock file utility', () => { '__fixtures__/auxiliary-packages/package-lock-v2.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootV2LockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootV2LockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootV2LockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(212); expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` @@ -252,9 +349,33 @@ describe('NPM lock file utility', () => { cleanupTypes(prunedV2LockFile.packages); cleanupTypes(prunedV2LockFile.dependencies, true); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootV2LockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootV2LockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootV2LockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const result = stringifyNpmLockfile( prunedGraph, @@ -339,9 +460,33 @@ describe('NPM lock file utility', () => { '__fixtures__/duplicate-package/package-lock-v1.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootLockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(369); }); it('should parse v3', async () => { @@ -350,9 +495,33 @@ describe('NPM lock file utility', () => { '__fixtures__/duplicate-package/package-lock.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootLockFile), builder); + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(369); }); }); @@ -367,9 +536,31 @@ describe('NPM lock file utility', () => { __dirname, '__fixtures__/optional/package.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(lockFile), builder); + + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes(JSON.stringify(lockFile), hash); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(lockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(8); const prunedGraph = pruneProjectGraph(graph, packageJson); @@ -392,9 +583,34 @@ describe('NPM lock file utility', () => { __dirname, '__fixtures__/pruning/typescript/package.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootLockFile), builder); + + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson); const result = stringifyNpmLockfile( prunedGraph, @@ -419,9 +635,34 @@ describe('NPM lock file utility', () => { __dirname, '__fixtures__/pruning/devkit-yargs/package.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(rootLockFile), builder); + + const hash = uniq('mock-hash'); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(rootLockFile), + hash + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getNpmLockfileDependencies( + JSON.stringify(rootLockFile), + hash, + pg + ); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, multiPackageJson); const result = stringifyNpmLockfile( prunedGraph, @@ -450,10 +691,13 @@ describe('NPM lock file utility', () => { __dirname, '__fixtures__/workspaces/package-lock.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(lockFile), builder); - const result = builder.getUpdatedProjectGraph(); - expect(Object.keys(result.externalNodes).length).toEqual(5); + + const externalNodes = getNpmLockfileNodes( + JSON.stringify(lockFile), + uniq('mock-hash') + ); + + expect(Object.keys(externalNodes).length).toEqual(5); }); it('should parse v1 lock file', async () => { @@ -461,10 +705,15 @@ describe('NPM lock file utility', () => { __dirname, '__fixtures__/workspaces/package-lock.v1.json' )); - const builder = new ProjectGraphBuilder(); - parseNpmLockfile(JSON.stringify(lockFile), builder); - const result = builder.getUpdatedProjectGraph(); - expect(Object.keys(result.externalNodes).length).toEqual(5); + const externalNodes = getNpmLockfileNodes( + JSON.stringify(lockFile), + uniq('mock') + ); + expect(Object.keys(externalNodes).length).toEqual(5); }); }); }); + +function uniq(str: string) { + return `str-${(Math.random() * 10000).toFixed(0)}`; +} diff --git a/packages/nx/src/plugins/js/lock-file/npm-parser.ts b/packages/nx/src/plugins/js/lock-file/npm-parser.ts index afe64935b1..0ded7a8e21 100644 --- a/packages/nx/src/plugins/js/lock-file/npm-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/npm-parser.ts @@ -3,12 +3,16 @@ import { satisfies } from 'semver'; import { workspaceRoot } from '../../../utils/workspace-root'; import { reverse } from '../../../project-graph/operators'; import { NormalizedPackageJson } from './utils/package-json'; -import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; import { + ProjectGraphDependencyWithFile, + validateDependency, +} from '../../../project-graph/project-graph-builder'; +import { + DependencyType, ProjectGraph, ProjectGraphExternalNode, } from '../../../config/project-graph'; -import { fileHasher, hashArray } from '../../../hasher/file-hasher'; +import { hashArray } from '../../../hasher/file-hasher'; /** * NPM @@ -50,21 +54,51 @@ type NpmLockFile = { dependencies?: Record; }; -export function parseNpmLockfile( - lockFileContent: string, - builder: ProjectGraphBuilder -) { - const data = JSON.parse(lockFileContent) as NpmLockFile; +// we use key => node map to avoid duplicate work when parsing keys +let keyMap = new Map(); +let currentLockFileHash: string; - // we use key => node map to avoid duplicate work when parsing keys - const keyMap = new Map(); - addNodes(data, builder, keyMap); - addDependencies(data, builder, keyMap); +let parsedLockFile: NpmLockFile; +function parsePackageLockFile(lockFileContent: string, lockFileHash: string) { + if (lockFileHash === currentLockFileHash) { + return parsedLockFile; + } + + keyMap.clear(); + const results = JSON.parse(lockFileContent) as NpmLockFile; + parsedLockFile = results; + currentLockFileHash = lockFileHash; + return results; } -function addNodes( +export function getNpmLockfileNodes( + lockFileContent: string, + lockFileHash: string +) { + const data = parsePackageLockFile( + lockFileContent, + lockFileHash + ) as NpmLockFile; + + // we use key => node map to avoid duplicate work when parsing keys + return getNodes(data, keyMap); +} + +export function getNpmLockfileDependencies( + lockFileContent: string, + lockFileHash: string, + projectGraph: ProjectGraph +) { + const data = parsePackageLockFile( + lockFileContent, + lockFileHash + ) as NpmLockFile; + + return getDependencies(data, keyMap, projectGraph); +} + +function getNodes( data: NpmLockFile, - builder: ProjectGraphBuilder, keyMap: Map ) { const nodes: Map> = new Map(); @@ -92,8 +126,7 @@ function addNodes( depSnapshot, `${snapshot.version.slice(5)}/node_modules/${depName}`, nodes, - keyMap, - builder + keyMap ); } ); @@ -104,13 +137,14 @@ function addNodes( snapshot, `node_modules/${packageName}`, nodes, - keyMap, - builder + keyMap ); } }); } + const results: Record = {}; + // some packages can be both hoisted and nested // so we need to run this check once we have all the nodes and paths for (const [packageName, versionMap] of nodes.entries()) { @@ -120,9 +154,10 @@ function addNodes( } versionMap.forEach((node) => { - builder.addExternalNode(node); + results[node.name] = node; }); } + return results; } function addV1Node( @@ -130,8 +165,7 @@ function addV1Node( snapshot: NpmDependencyV1, path: string, nodes: Map>, - keyMap: Map, - builder: ProjectGraphBuilder + keyMap: Map ) { createNode(packageName, snapshot.version, path, nodes, keyMap, snapshot); @@ -143,8 +177,7 @@ function addV1Node( depSnapshot, `${path}/node_modules/${depName}`, nodes, - keyMap, - builder + keyMap ); }); } @@ -210,11 +243,12 @@ function findV3Version(snapshot: NpmDependencyV3, packageName: string): string { return version; } -function addDependencies( +function getDependencies( data: NpmLockFile, - builder: ProjectGraphBuilder, - keyMap: Map -) { + keyMap: Map, + projectGraph: ProjectGraph +): ProjectGraphDependencyWithFile[] { + const dependencies: ProjectGraphDependencyWithFile[] = []; if (data.lockfileVersion > 1) { Object.entries(data.packages).forEach(([path, snapshot]) => { // we are skipping workspaces packages @@ -231,7 +265,13 @@ function addDependencies( Object.entries(section).forEach(([name, versionRange]) => { const target = findTarget(path, keyMap, name, versionRange); if (target) { - builder.addStaticDependency(sourceName, target.name); + const dep = { + source: sourceName, + target: target.name, + dependencyType: DependencyType.static, + }; + validateDependency(projectGraph, dep); + dependencies.push(dep); } }); } @@ -242,11 +282,13 @@ function addDependencies( addV1NodeDependencies( `node_modules/${packageName}`, snapshot, - builder, - keyMap + dependencies, + keyMap, + projectGraph ); }); } + return dependencies; } function findTarget( @@ -284,15 +326,22 @@ function findTarget( function addV1NodeDependencies( path: string, snapshot: NpmDependencyV1, - builder: ProjectGraphBuilder, - keyMap: Map + dependencies: ProjectGraphDependencyWithFile[], + keyMap: Map, + projectGraph: ProjectGraph ) { if (keyMap.has(path) && snapshot.requires) { const source = keyMap.get(path).name; Object.entries(snapshot.requires).forEach(([name, versionRange]) => { const target = findTarget(path, keyMap, name, versionRange); if (target) { - builder.addStaticDependency(source, target.name); + const dep = { + source: source, + target: target.name, + dependencyType: DependencyType.static, + }; + validateDependency(projectGraph, dep); + dependencies.push(dep); } }); } @@ -302,8 +351,9 @@ function addV1NodeDependencies( addV1NodeDependencies( `${path}/node_modules/${depName}`, depSnapshot, - builder, - keyMap + dependencies, + keyMap, + projectGraph ); }); } @@ -311,15 +361,15 @@ function addV1NodeDependencies( if (peerDependencies) { const node = keyMap.get(path); Object.entries(peerDependencies).forEach(([depName, depSpec]) => { - if ( - !builder.graph.dependencies[node.name]?.find( - (d) => d.target === depName - ) - ) { - const target = findTarget(path, keyMap, depName, depSpec); - if (target) { - builder.addStaticDependency(node.name, target.name); - } + const target = findTarget(path, keyMap, depName, depSpec); + if (target) { + const dep = { + source: node.name, + target: target.name, + dependencyType: DependencyType.static, + }; + validateDependency(projectGraph, dep); + dependencies.push(dep); } }); } diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts index 440a62c5df..5443ae8a03 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts @@ -1,9 +1,16 @@ import { joinPathFragments } from '../../../utils/path'; -import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser'; +import { + getPnpmLockfileNodes, + getPnpmLockfileDependencies, + stringifyPnpmLockfile, +} from './pnpm-parser'; import { ProjectGraph } from '../../../config/project-graph'; import { vol } from 'memfs'; import { pruneProjectGraph } from './project-graph-pruning'; -import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; +import { + ProjectGraphBuilder, + ProjectGraphDependencyWithFile, +} from '../../../project-graph/project-graph-builder'; jest.mock('fs', () => { const memFs = require('memfs').fs; @@ -119,8 +126,12 @@ describe('pnpm LockFile utility', () => { vol.fromJSON(fileSys, '/root'); }); + let externalNodes: ProjectGraph['externalNodes']; + let dependencies: ProjectGraphDependencyWithFile[]; let graph: ProjectGraph; + let lockFile: string; + let lockFileHash: string; describe('v5.4', () => { beforeEach(() => { @@ -128,13 +139,34 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/nextjs/pnpm-lock.yaml' )).default; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); + lockFileHash = '__fixtures__/nextjs/pnpm-lock.yaml'; + + externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + graph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } graph = builder.getUpdatedProjectGraph(); }); it('should parse root lock file', async () => { - expect(Object.keys(graph.externalNodes).length).toEqual(1280); + expect(Object.keys(externalNodes).length).toEqual(1280); }); it('should prune lock file', async () => { @@ -165,8 +197,28 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/nextjs/pnpm-lock-v6.yaml' )).default; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); + lockFileHash = '__fixtures__/nextjs/pnpm-lock-v6.yaml'; + externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + graph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } graph = builder.getUpdatedProjectGraph(); }); @@ -184,10 +236,33 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/nextjs/app/pnpm-lock-v6.yaml' )).default; + const appLockFileHash = '__fixtures__/nextjs/app/pnpm-lock-v6.yaml'; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(appLockFile, builder); - const appGraph = builder.getUpdatedProjectGraph(); + const externalNodes = getPnpmLockfileNodes( + appLockFile, + appLockFileHash + ); + let appGraph: ProjectGraph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getPnpmLockfileDependencies( + appLockFile, + appLockFileHash, + appGraph + ); + + const builder = new ProjectGraphBuilder(appGraph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } + appGraph = builder.getUpdatedProjectGraph(); expect(Object.keys(appGraph.externalNodes).length).toEqual(864); // this is our pruned lock file structure @@ -234,9 +309,31 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/auxiliary-packages/pnpm-lock.yaml' )).default; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); - const graph = builder.getUpdatedProjectGraph(); + const lockFileHash = '__fixtures__/auxiliary-packages/pnpm-lock.yaml'; + + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + let graph: ProjectGraph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } + graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(213); expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` @@ -291,6 +388,7 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/auxiliary-packages/pnpm-lock.yaml' )).default; + const lockFileHash = '__fixtures__/auxiliary-packages/pnpm-lock.yaml'; const prunedLockFile: string = require(joinPathFragments( __dirname, '__fixtures__/auxiliary-packages/pnpm-lock.yaml.pruned' @@ -316,9 +414,29 @@ describe('pnpm LockFile utility', () => { }, }; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); - const graph = builder.getUpdatedProjectGraph(); + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + let graph: ProjectGraph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + + const dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } + graph = builder.getUpdatedProjectGraph(); const prunedGraph = pruneProjectGraph(graph, prunedPackageJson); const result = stringifyPnpmLockfile( prunedGraph, @@ -355,9 +473,31 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/duplicate-package/pnpm-lock.yaml' )).default; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); - const graph = builder.getUpdatedProjectGraph(); + const lockFileHash = '__fixtures__/duplicate-package/pnpm-lock.yaml'; + + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + let graph: ProjectGraph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } + graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(370); expect(Object.keys(graph.dependencies).length).toEqual(213); expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6); @@ -381,9 +521,29 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/optional/pnpm-lock.yaml' )).default; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); - const graph = builder.getUpdatedProjectGraph(); + const lockFileHash = '__fixtures__/optional/pnpm-lock.yaml'; + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + let graph: ProjectGraph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } + graph = builder.getUpdatedProjectGraph(); expect(Object.keys(graph.externalNodes).length).toEqual(8); const packageJson = require(joinPathFragments( @@ -396,7 +556,7 @@ describe('pnpm LockFile utility', () => { }); describe('pruning', () => { - let graph, lockFile; + let graph, lockFile, lockFileHash; beforeEach(() => { const fileSys = { @@ -420,9 +580,29 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/pruning/pnpm-lock.yaml' )).default; + lockFileHash = '__fixtures__/pruning/pnpm-lock.yaml'; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + graph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } graph = builder.getUpdatedProjectGraph(); }); @@ -471,9 +651,29 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/pruning/pnpm-lock-v6.yaml' )).default; + lockFileHash = '__fixtures__/pruning/pnpm-lock-v6.yaml'; - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + graph = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getPnpmLockfileDependencies( + lockFile, + lockFileHash, + graph + ); + + const builder = new ProjectGraphBuilder(graph); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } graph = builder.getUpdatedProjectGraph(); }); @@ -518,7 +718,7 @@ describe('pnpm LockFile utility', () => { }); describe('workspaces', () => { - let lockFile; + let lockFile, lockFileHash; beforeAll(() => { const fileSys = { @@ -534,13 +734,12 @@ describe('pnpm LockFile utility', () => { __dirname, '__fixtures__/workspaces/pnpm-lock.yaml' )).default; + lockFileHash = '__fixtures__/workspaces/pnpm-lock.yaml'; }); it('should parse lock file', async () => { - const builder = new ProjectGraphBuilder(); - parsePnpmLockfile(lockFile, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(5); + const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash); + expect(Object.keys(externalNodes).length).toEqual(5); }); }); }); diff --git a/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts b/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts index 8ccf8dda46..851f5f1241 100644 --- a/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/pnpm-parser.ts @@ -1,8 +1,8 @@ import type { - PackageSnapshot, Lockfile, - ProjectSnapshot, + PackageSnapshot, PackageSnapshots, + ProjectSnapshot, } from '@pnpm/lockfile-types'; import { isV6Lockfile, @@ -10,36 +10,65 @@ import { parseAndNormalizePnpmLockfile, stringifyToPnpmYaml, } from './utils/pnpm-normalizer'; -import { getHoistedPackageVersion } from './utils/package-json'; -import { NormalizedPackageJson } from './utils/package-json'; -import { sortObjectByKeys } from '../../../utils/object-sort'; -import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; import { + getHoistedPackageVersion, + NormalizedPackageJson, +} from './utils/package-json'; +import { sortObjectByKeys } from '../../../utils/object-sort'; +import { + ProjectGraphDependencyWithFile, + validateDependency, +} from '../../../project-graph/project-graph-builder'; +import { + DependencyType, ProjectGraph, ProjectGraphExternalNode, } from '../../../config/project-graph'; import { hashArray } from '../../../hasher/file-hasher'; -export function parsePnpmLockfile( - lockFileContent: string, - builder: ProjectGraphBuilder -): void { - const data = parseAndNormalizePnpmLockfile(lockFileContent); - const isV6 = isV6Lockfile(data); +// we use key => node map to avoid duplicate work when parsing keys +let keyMap = new Map(); +let currentLockFileHash: string; - // we use key => node map to avoid duplicate work when parsing keys - const keyMap = new Map(); +let parsedLockFile: Lockfile; +function parsePnpmLockFile(lockFileContent: string, lockFileHash: string) { + if (lockFileHash === currentLockFileHash) { + return parsedLockFile; + } - addNodes(data, builder, keyMap, isV6); - addDependencies(data, builder, keyMap, isV6); + keyMap.clear(); + const results = parseAndNormalizePnpmLockfile(lockFileContent); + parsedLockFile = results; + currentLockFileHash = lockFileHash; + return results; } -function addNodes( +export function getPnpmLockfileNodes( + lockFileContent: string, + lockFileHash: string +) { + const data = parsePnpmLockFile(lockFileContent, lockFileHash); + const isV6 = isV6Lockfile(data); + + return getNodes(data, keyMap, isV6); +} + +export function getPnpmLockfileDependencies( + lockFileContent: string, + lockFileHash: string, + projectGraph: ProjectGraph +) { + const data = parsePnpmLockFile(lockFileContent, lockFileHash); + const isV6 = isV6Lockfile(data); + + return getDependencies(data, keyMap, isV6, projectGraph); +} + +function getNodes( data: Lockfile, - builder: ProjectGraphBuilder, keyMap: Map, isV6: boolean -) { +): Record { const nodes: Map> = new Map(); Object.entries(data.packages).forEach(([key, snapshot]) => { @@ -80,6 +109,8 @@ function addNodes( }); const hoistedDeps = loadPnpmHoistedDepsDefinition(); + const results: Record = {}; + for (const [packageName, versionMap] of nodes.entries()) { let hoistedNode: ProjectGraphExternalNode; if (versionMap.size === 1) { @@ -93,9 +124,10 @@ function addNodes( } versionMap.forEach((node) => { - builder.addExternalNode(node); + results[node.name] = node; }); } + return results; } function getHoistedVersion( @@ -121,12 +153,13 @@ function getHoistedVersion( return version; } -function addDependencies( +function getDependencies( data: Lockfile, - builder: ProjectGraphBuilder, keyMap: Map, - isV6: boolean -) { + isV6: boolean, + projectGraph: ProjectGraph +): ProjectGraphDependencyWithFile[] { + const results: ProjectGraphDependencyWithFile[] = []; Object.entries(data.packages).forEach(([key, snapshot]) => { const node = keyMap.get(key); [snapshot.dependencies, snapshot.optionalDependencies].forEach( @@ -138,16 +171,24 @@ function addDependencies( isV6 ); const target = - builder.graph.externalNodes[`npm:${name}@${version}`] || - builder.graph.externalNodes[`npm:${name}`]; + projectGraph.externalNodes[`npm:${name}@${version}`] || + projectGraph.externalNodes[`npm:${name}`]; if (target) { - builder.addStaticDependency(node.name, target.name); + const dep = { + source: node.name, + target: target.name, + dependencyType: DependencyType.static, + }; + validateDependency(projectGraph, dep); + results.push(dep); } }); } } ); }); + + return results; } function parseBaseVersion(rawVersion: string, isV6: boolean): string { diff --git a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts index 9de08833ad..fd5c48faf9 100644 --- a/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts +++ b/packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts @@ -1,5 +1,9 @@ import { joinPathFragments } from '../../../utils/path'; -import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser'; +import { + getYarnLockfileNodes, + getYarnLockfileDependencies, + stringifyYarnLockfile, +} from './yarn-parser'; import { pruneProjectGraph } from './project-graph-pruning'; import { vol } from 'memfs'; import { ProjectGraph } from '../../../config/project-graph'; @@ -168,7 +172,6 @@ describe('yarn LockFile utility', () => { let graph: ProjectGraph; beforeEach(() => { - const builder = new ProjectGraphBuilder(); lockFile = require(joinPathFragments( __dirname, '__fixtures__/nextjs/yarn.lock' @@ -177,7 +180,25 @@ describe('yarn LockFile utility', () => { __dirname, '__fixtures__/nextjs/package.json' )); - parseYarnLockfile(lockFile, packageJson, builder); + + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } graph = builder.getUpdatedProjectGraph(); }); @@ -383,12 +404,17 @@ describe('yarn LockFile utility', () => { __dirname, '__fixtures__/auxiliary-packages/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(classicLockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(127); - expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes( + classicLockFile, + hash, + packageJson + ); + + expect(Object.keys(externalNodes).length).toEqual(127); + + expect(externalNodes['npm:minimatch']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", @@ -399,7 +425,7 @@ describe('yarn LockFile utility', () => { "type": "npm", } `); - expect(graph.externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(` + expect(externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g==", @@ -410,7 +436,7 @@ describe('yarn LockFile utility', () => { "type": "npm", } `); - expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` + expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(` { "data": { "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb", @@ -421,7 +447,7 @@ describe('yarn LockFile utility', () => { "type": "npm", } `); - expect(graph.externalNodes['npm:eslint-plugin-disable-autofix']) + expect(externalNodes['npm:eslint-plugin-disable-autofix']) .toMatchInlineSnapshot(` { "data": { @@ -465,9 +491,26 @@ describe('yarn LockFile utility', () => { '__fixtures__/auxiliary-packages/yarn.lock.pruned' )).default; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const result = stringifyYarnLockfile( prunedGraph, @@ -503,9 +546,30 @@ describe('yarn LockFile utility', () => { '__fixtures__/auxiliary-packages/yarn.lock.pruned' )).default; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, normalizedPackageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes( + lockFile, + hash, + normalizedPackageJson + ); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const result = stringifyYarnLockfile( prunedGraph, @@ -529,12 +593,17 @@ describe('yarn LockFile utility', () => { __dirname, '__fixtures__/auxiliary-packages/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(berryLockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(129); - expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes( + berryLockFile, + hash, + packageJson + ); + + expect(Object.keys(externalNodes).length).toEqual(129); + + expect(externalNodes['npm:minimatch']).toMatchInlineSnapshot(` { "data": { "hash": "c154e566406683e7bcb746e000b84d74465b3a832c45d59912b9b55cd50dee66e5c4b1e5566dba26154040e51672f9aa450a9aef0c97cfc7336b78b7afb9540a", @@ -545,7 +614,7 @@ describe('yarn LockFile utility', () => { "type": "npm", } `); - expect(graph.externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(` + expect(externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(` { "data": { "hash": "215edd0978320a3354188f84a537d45841f2449af4df4379f79b9b777e71aa4f5722cc9d1717eabd2a70d38ef76ab7b708d24d83ea6a6c909dfd8833de98b437", @@ -556,7 +625,7 @@ describe('yarn LockFile utility', () => { "type": "npm", } `); - expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` + expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(` { "data": { "hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a", @@ -567,7 +636,7 @@ describe('yarn LockFile utility', () => { "type": "npm", } `); - expect(graph.externalNodes['npm:eslint-plugin-disable-autofix']) + expect(externalNodes['npm:eslint-plugin-disable-autofix']) .toMatchInlineSnapshot(` { "data": { @@ -612,9 +681,26 @@ describe('yarn LockFile utility', () => { '__fixtures__/auxiliary-packages/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const result = stringifyYarnLockfile( prunedGraph, @@ -664,9 +750,26 @@ __metadata: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(graph.externalNodes).toMatchInlineSnapshot(` { "npm:@docusaurus/core": { @@ -728,10 +831,10 @@ __metadata: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(graph.externalNodes).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(externalNodes).toMatchInlineSnapshot(` { "npm:@docusaurus/core": { "data": { @@ -808,10 +911,10 @@ postgres@charsleysa/postgres#fix-errors-compiled: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==", @@ -822,7 +925,7 @@ postgres@charsleysa/postgres#fix-errors-compiled: "type": "npm", } `); - expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` + expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==", @@ -833,7 +936,7 @@ postgres@charsleysa/postgres#fix-errors-compiled: "type": "npm", } `); - expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` + expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(` { "data": { "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb", @@ -878,10 +981,10 @@ postgres@charsleysa/postgres#fix-errors-compiled: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==", @@ -892,7 +995,7 @@ postgres@charsleysa/postgres#fix-errors-compiled: "type": "npm", } `); - expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` + expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==", @@ -903,7 +1006,7 @@ postgres@charsleysa/postgres#fix-errors-compiled: "type": "npm", } `); - expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` + expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(` { "data": { "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb", @@ -934,10 +1037,10 @@ nx-cloud@latest: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` { "data": { "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==", @@ -961,12 +1064,17 @@ nx-cloud@latest: __dirname, '__fixtures__/auxiliary-packages/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(berryLockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(129); - expect(graph.externalNodes['npm:react']).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes( + berryLockFile, + hash, + packageJson + ); + + expect(Object.keys(externalNodes).length).toEqual(129); + + expect(externalNodes['npm:react']).toMatchInlineSnapshot(` { "data": { "hash": "88e38092da8839b830cda6feef2e8505dec8ace60579e46aa5490fc3dc9bba0bd50336507dc166f43e3afc1c42939c09fe33b25fae889d6f402721dcd78fca1b", @@ -978,7 +1086,7 @@ nx-cloud@latest: } `); - expect(graph.externalNodes['npm:typescript']).toMatchInlineSnapshot(` + expect(externalNodes['npm:typescript']).toMatchInlineSnapshot(` { "data": { "hash": "ee000bc26848147ad423b581bd250075662a354d84f0e06eb76d3b892328d8d4440b7487b5a83e851b12b255f55d71835b008a66cbf8f255a11e4400159237db", @@ -989,7 +1097,7 @@ nx-cloud@latest: "type": "npm", } `); - expect(graph.externalNodes['npm:@nrwl/devkit']).toMatchInlineSnapshot(` + expect(externalNodes['npm:@nrwl/devkit']).toMatchInlineSnapshot(` { "data": { "hash": "7dcc3600998448c496228e062d7edd8ecf959fa1ddb9721e91bb1f60f1a2284fd0e12e09edc022170988e2fb54acf101c79dc09fe9c54a21c9941e682eb73b92", @@ -1000,7 +1108,7 @@ nx-cloud@latest: "type": "npm", } `); - expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` + expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(` { "data": { "hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a", @@ -1011,7 +1119,7 @@ nx-cloud@latest: "type": "npm", } `); - expect(graph.externalNodes['npm:eslint-plugin-disable-autofix']) + expect(externalNodes['npm:eslint-plugin-disable-autofix']) .toMatchInlineSnapshot(` { "data": { @@ -1073,10 +1181,14 @@ nx-cloud@latest: __dirname, '__fixtures__/duplicate-package/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(classicLockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(371); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes( + classicLockFile, + hash, + packageJson + ); + + expect(Object.keys(externalNodes).length).toEqual(371); }); }); @@ -1103,9 +1215,27 @@ nx-cloud@latest: __dirname, '__fixtures__/optional/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(Object.keys(graph.externalNodes).length).toEqual(103); const prunedGraph = pruneProjectGraph(graph, packageJson); @@ -1286,9 +1416,27 @@ nx-cloud@latest: __dirname, '__fixtures__/pruning/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson); const result = stringifyYarnLockfile( prunedGraph, @@ -1317,9 +1465,27 @@ nx-cloud@latest: __dirname, '__fixtures__/pruning/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, multiPackageJson); const result = stringifyYarnLockfile( prunedGraph, @@ -1354,10 +1520,10 @@ nx-cloud@latest: __dirname, '__fixtures__/workspaces/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(5); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(Object.keys(externalNodes).length).toEqual(5); }); it('should parse berry lock file', async () => { @@ -1369,10 +1535,10 @@ nx-cloud@latest: __dirname, '__fixtures__/workspaces/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(Object.keys(graph.externalNodes).length).toEqual(5); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(Object.keys(externalNodes).length).toEqual(5); }); }); @@ -1435,9 +1601,24 @@ type-fest@^0.20.2: tslib: '^2.4.0', }, }; + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(` { @@ -1525,9 +1706,26 @@ __metadata: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(` { "data": { @@ -1623,9 +1821,26 @@ __metadata: '__fixtures__/mixed-keys/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(graph.externalNodes).toMatchInlineSnapshot(` { "npm:@isaacs/cliui": { @@ -1835,9 +2050,26 @@ __metadata: '__fixtures__/mixed-keys/package.json' )); - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + expect(graph.externalNodes).toMatchInlineSnapshot(` { "npm:@isaacs/cliui": { @@ -2099,10 +2331,10 @@ __metadata: }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); - const graph = builder.getUpdatedProjectGraph(); - expect(graph.externalNodes).toMatchInlineSnapshot(` + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + + expect(externalNodes).toMatchInlineSnapshot(` { "npm:@octokit/request-error": { "data": { @@ -2196,9 +2428,27 @@ __metadata: resolve: '^1.12.0', }, }; - const builder = new ProjectGraphBuilder(); - parseYarnLockfile(lockFile, packageJson, builder); + + const hash = uniq('mock-hash'); + const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson); + const pg = { + nodes: {}, + dependencies: {}, + externalNodes, + }; + const dependencies = getYarnLockfileDependencies(lockFile, hash, pg); + + const builder = new ProjectGraphBuilder(pg); + for (const dep of dependencies) { + builder.addDependency( + dep.source, + dep.target, + dep.dependencyType, + dep.sourceFile + ); + } const graph = builder.getUpdatedProjectGraph(); + const prunedGraph = pruneProjectGraph(graph, packageJson); const result = stringifyYarnLockfile(prunedGraph, lockFile, packageJson); expect(result).toMatchInlineSnapshot(` @@ -2247,3 +2497,7 @@ __metadata: }); }); }); + +function uniq(str: string) { + return `str-${(Math.random() * 10000).toFixed(0)}`; +} diff --git a/packages/nx/src/plugins/js/lock-file/yarn-parser.ts b/packages/nx/src/plugins/js/lock-file/yarn-parser.ts index 7756c5892c..0fc937c4f8 100644 --- a/packages/nx/src/plugins/js/lock-file/yarn-parser.ts +++ b/packages/nx/src/plugins/js/lock-file/yarn-parser.ts @@ -1,8 +1,14 @@ -import { getHoistedPackageVersion } from './utils/package-json'; -import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; -import { satisfies, Range, gt } from 'semver'; -import { NormalizedPackageJson } from './utils/package-json'; import { + getHoistedPackageVersion, + NormalizedPackageJson, +} from './utils/package-json'; +import { + ProjectGraphDependencyWithFile, + validateDependency, +} from '../../../project-graph/project-graph-builder'; +import { gt, Range, satisfies } from 'semver'; +import { + DependencyType, ProjectGraph, ProjectGraphExternalNode, } from '../../../config/project-graph'; @@ -31,23 +37,61 @@ type YarnDependency = { linkType?: 'soft' | 'hard'; }; -export function parseYarnLockfile( - lockFileContent: string, - packageJson: NormalizedPackageJson, - builder: ProjectGraphBuilder -) { - const { parseSyml } = require('@yarnpkg/parsers'); - const { __metadata, ...dependencies } = parseSyml(lockFileContent); - const isBerry = !!__metadata; +let currentLockFileHash: string; +let cachedParsedLockFile; - // we use key => node map to avoid duplicate work when parsing keys - const keyMap = new Map(); +// we use key => node map to avoid duplicate work when parsing keys +let keyMap = new Map(); + +function parseLockFile(lockFileContent: string, lockFileHash: string) { + if (currentLockFileHash === lockFileHash) { + return cachedParsedLockFile; + } + + const { parseSyml } = + require('@yarnpkg/parsers') as typeof import('@yarnpkg/parsers'); + + keyMap.clear(); + const result = parseSyml(lockFileContent); + cachedParsedLockFile = result; + currentLockFileHash = lockFileHash; + return result; +} + +export function getYarnLockfileNodes( + lockFileContent: string, + lockFileHash: string, + packageJson: NormalizedPackageJson +) { + const { __metadata, ...dependencies } = parseLockFile( + lockFileContent, + lockFileHash + ); + + const isBerry = !!__metadata; // yarn classic splits keys when parsing so we need to stich them back together const groupedDependencies = groupDependencies(dependencies, isBerry); - addNodes(groupedDependencies, packageJson, builder, keyMap, isBerry); - addDependencies(groupedDependencies, builder, keyMap); + return getNodes(groupedDependencies, packageJson, keyMap, isBerry); +} + +export function getYarnLockfileDependencies( + lockFileContent: string, + lockFileHash: string, + projectGraph: ProjectGraph +) { + const { __metadata, ...dependencies } = parseLockFile( + lockFileContent, + lockFileHash + ); + + const isBerry = !!__metadata; + + // yarn classic splits keys when parsing so we need to stich them back together + const groupedDependencies = groupDependencies(dependencies, isBerry); + + return getDependencies(groupedDependencies, keyMap, projectGraph); } function getPackageNameKeyPairs(keys: string): Map> { @@ -63,10 +107,9 @@ function getPackageNameKeyPairs(keys: string): Map> { return result; } -function addNodes( +function getNodes( dependencies: Record, packageJson: NormalizedPackageJson, - builder: ProjectGraphBuilder, keyMap: Map, isBerry: boolean ) { @@ -128,6 +171,7 @@ function addNodes( }); }); + const externalNodes: Record = {}; for (const [packageName, versionMap] of nodes.entries()) { const hoistedNode = findHoistedNode(packageName, versionMap, combinedDeps); if (hoistedNode) { @@ -135,9 +179,10 @@ function addNodes( } versionMap.forEach((node) => { - builder.addExternalNode(node); + externalNodes[node.name] = node; }); } + return externalNodes; } function findHoistedNode( @@ -241,11 +286,12 @@ function getHoistedVersion(packageName: string): string { } } -function addDependencies( +function getDependencies( dependencies: Record, - builder: ProjectGraphBuilder, - keyMap: Map + keyMap: Map, + projectGraph: ProjectGraph ) { + const projectGraphDependencies: ProjectGraphDependencyWithFile[] = []; Object.keys(dependencies).forEach((keys) => { const snapshot = dependencies[keys]; keys.split(', ').forEach((key) => { @@ -259,7 +305,13 @@ function addDependencies( keyMap.get(`${name}@npm:${versionRange}`) || keyMap.get(`${name}@${versionRange}`); if (target) { - builder.addStaticDependency(node.name, target.name); + const dep = { + source: node.name, + target: target.name, + dependencyType: DependencyType.static, + }; + validateDependency(projectGraph, dep); + projectGraphDependencies.push(dep); } }); } @@ -268,6 +320,8 @@ function addDependencies( } }); }); + + return projectGraphDependencies; } export function stringifyYarnLockfile( diff --git a/packages/nx/src/plugins/js/utils/config.ts b/packages/nx/src/plugins/js/utils/config.ts new file mode 100644 index 0000000000..baef4c7b48 --- /dev/null +++ b/packages/nx/src/plugins/js/utils/config.ts @@ -0,0 +1,80 @@ +import { join } from 'node:path'; + +import { + NrwlJsPluginConfig, + NxJsonConfiguration, +} from '../../../config/nx-json'; +import { fileExists, readJsonFile } from '../../../utils/fileutils'; +import { PackageJson } from '../../../utils/package-json'; +import { workspaceRoot } from '../../../utils/workspace-root'; +import { existsSync } from 'fs'; + +export function jsPluginConfig( + nxJson: NxJsonConfiguration +): Required { + const nxJsonConfig: NrwlJsPluginConfig = + nxJson?.pluginsConfig?.['@nx/js'] ?? nxJson?.pluginsConfig?.['@nrwl/js']; + + // using lerna _before_ installing deps is causing an issue when parsing lockfile. + // See: https://github.com/lerna/lerna/issues/3807 + // Note that previous attempt to fix this caused issues with Nx itself, thus we're checking + // for Lerna explicitly. + // See: https://github.com/nrwl/nx/pull/18784/commits/5416138e1ddc1945d5b289672dfb468e8c544e14 + const analyzeLockfile = + !existsSync(join(workspaceRoot, 'lerna.json')) || + existsSync(join(workspaceRoot, 'nx.json')); + + if (nxJsonConfig) { + return { + analyzePackageJson: true, + analyzeSourceFiles: true, + analyzeLockfile, + ...nxJsonConfig, + }; + } + + if (!fileExists(join(workspaceRoot, 'package.json'))) { + return { + analyzeLockfile: false, + analyzePackageJson: false, + analyzeSourceFiles: false, + }; + } + + const packageJson = readJsonFile( + join(workspaceRoot, 'package.json') + ); + + const packageJsonDeps = { + ...packageJson.dependencies, + ...packageJson.devDependencies, + }; + if ( + packageJsonDeps['@nx/workspace'] || + packageJsonDeps['@nx/js'] || + packageJsonDeps['@nx/node'] || + packageJsonDeps['@nx/next'] || + packageJsonDeps['@nx/react'] || + packageJsonDeps['@nx/angular'] || + packageJsonDeps['@nx/web'] || + packageJsonDeps['@nrwl/workspace'] || + packageJsonDeps['@nrwl/js'] || + packageJsonDeps['@nrwl/node'] || + packageJsonDeps['@nrwl/next'] || + packageJsonDeps['@nrwl/react'] || + packageJsonDeps['@nrwl/angular'] || + packageJsonDeps['@nrwl/web'] + ) { + return { + analyzePackageJson: true, + analyzeLockfile, + analyzeSourceFiles: true, + }; + } else { + return { + analyzePackageJson: true, + analyzeLockfile, + analyzeSourceFiles: false, + }; + } +} diff --git a/packages/nx/src/utils/nx-plugin.ts b/packages/nx/src/utils/nx-plugin.ts index a538c6bf88..b86d1c4147 100644 --- a/packages/nx/src/utils/nx-plugin.ts +++ b/packages/nx/src/utils/nx-plugin.ts @@ -67,7 +67,7 @@ export type CreateNodesFunction = ( /** * A pair of file patterns and {@link CreateNodesFunction} */ -export type CreateNodes = [ +export type CreateNodes = readonly [ projectFilePattern: string, createNodesFunction: CreateNodesFunction ]; @@ -192,7 +192,7 @@ export async function loadNxPluginAsync( let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root); const plugin = (await import(pluginPath)) as NxPlugin; - plugin.name = name; + plugin.name ??= name; nxPluginCache.set(moduleName, plugin); return plugin; } @@ -205,7 +205,7 @@ function loadNxPluginSync(moduleName: string, paths: string[], root: string) { let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root); const plugin = require(pluginPath) as NxPlugin; - plugin.name = name; + plugin.name ??= name; nxPluginCache.set(moduleName, plugin); return plugin; } @@ -218,14 +218,10 @@ export function loadNxPluginsSync( paths = getNxRequirePaths(), root = workspaceRoot ): (NxPluginV2 & Pick)[] { - const result: NxPlugin[] = []; - // TODO: This should be specified in nx.json // Temporarily load js as if it were a plugin which is built into nx // In the future, this will be optional and need to be specified in nx.json - const jsPlugin: any = require('../plugins/js'); - jsPlugin.name = 'nx-js-graph-plugin'; - result.push(jsPlugin as NxPlugin); + const result: NxPlugin[] = [...getDefaultPluginsSync(root)]; if (shouldMergeAngularProjects(root, false)) { result.push(NxAngularJsonPlugin); @@ -259,18 +255,12 @@ export async function loadNxPlugins( paths = getNxRequirePaths(), root = workspaceRoot ): Promise<(NxPluginV2 & Pick)[]> { - const result: NxPlugin[] = []; + const result: NxPlugin[] = [...(await getDefaultPlugins(root))]; - // TODO: This should be specified in nx.json + // TODO: These should be specified in nx.json // Temporarily load js as if it were a plugin which is built into nx // In the future, this will be optional and need to be specified in nx.json - const jsPlugin: any = await import('../plugins/js'); - jsPlugin.name = 'nx-js-graph-plugin'; - result.push(jsPlugin as NxPlugin); - - if (shouldMergeAngularProjects(root, false)) { - result.push(NxAngularJsonPlugin); - } + result.push(); plugins ??= []; for (const plugin of plugins) { @@ -484,3 +474,23 @@ function readPluginMainFromProjectConfiguration( {}; return main; } + +async function getDefaultPlugins(root: string) { + const plugins: NxPlugin[] = [await import('../plugins/js')]; + + if (shouldMergeAngularProjects(root, false)) { + plugins.push( + await import('../adapter/angular-json').then((m) => m.NxAngularJsonPlugin) + ); + } + return plugins; +} + +function getDefaultPluginsSync(root: string) { + const plugins: NxPlugin[] = [require('../plugins/js')]; + + if (shouldMergeAngularProjects(root, false)) { + plugins.push(require('../adapter/angular-json').NxAngularJsonPlugin); + } + return plugins; +} diff --git a/packages/vite/src/executors/build/build.impl.ts b/packages/vite/src/executors/build/build.impl.ts index b534b7885d..099a7daea6 100644 --- a/packages/vite/src/executors/build/build.impl.ts +++ b/packages/vite/src/executors/build/build.impl.ts @@ -1,4 +1,5 @@ import { + detectPackageManager, ExecutorContext, logger, stripIndents, @@ -79,11 +80,20 @@ export async function* viteBuildExecutor( builtPackageJson.type = 'module'; writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson); + const packageManager = detectPackageManager(context.root); - const lockFile = createLockFile(builtPackageJson); - writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, { - encoding: 'utf-8', - }); + const lockFile = createLockFile( + builtPackageJson, + context.projectGraph, + packageManager + ); + writeFileSync( + `${options.outputPath}/${getLockFileName(packageManager)}`, + lockFile, + { + encoding: 'utf-8', + } + ); } // For buildable libs, copy package.json if it exists. else if ( diff --git a/packages/webpack/src/plugins/generate-package-json-plugin.ts b/packages/webpack/src/plugins/generate-package-json-plugin.ts index 35ecd4d58f..47aa1ad940 100644 --- a/packages/webpack/src/plugins/generate-package-json-plugin.ts +++ b/packages/webpack/src/plugins/generate-package-json-plugin.ts @@ -1,6 +1,11 @@ import { type Compiler, sources, type WebpackPluginInstance } from 'webpack'; import { createLockFile, createPackageJson } from '@nx/js'; -import { ExecutorContext, type ProjectGraph, serializeJson } from '@nx/devkit'; +import { + detectPackageManager, + ExecutorContext, + type ProjectGraph, + serializeJson, +} from '@nx/devkit'; import { getHelperDependenciesFromProjectGraph, getLockFileName, @@ -66,9 +71,12 @@ export class GeneratePackageJsonPlugin implements WebpackPluginInstance { 'package.json', new sources.RawSource(serializeJson(packageJson)) ); + const packageManager = detectPackageManager(this.context.root); compilation.emitAsset( - getLockFileName(), - new sources.RawSource(createLockFile(packageJson)) + getLockFileName(packageManager), + new sources.RawSource( + createLockFile(packageJson, this.projectGraph, packageManager) + ) ); } );