From 68e974c848b5f1a56228039d87218ed3544f1e8f Mon Sep 17 00:00:00 2001 From: Jason Jean Date: Mon, 27 Nov 2023 09:45:48 -0500 Subject: [PATCH] =?UTF-8?q?Reapply=20"cleanup(core):=20separate=20retrieve?= =?UTF-8?q?=20project=20configurations=20and=20=E2=80=A6=20(#20398)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devkit/src/utils/convert-nx-executor.ts | 2 +- packages/nx/src/config/workspaces.spec.ts | 4 +- ...project-graph-incremental-recomputation.ts | 63 ++++---- .../executors/utils/convert-nx-executor.ts | 2 +- .../hasher/native-task-hasher-impl.spec.ts | 49 +++--- packages/nx/src/native/index.d.ts | 3 +- .../src/native/tests/workspace_files.spec.ts | 16 +- .../nx/src/native/workspace/config_files.rs | 17 --- packages/nx/src/native/workspace/context.rs | 38 +---- .../src/native/workspace/workspace_files.rs | 128 +++++++--------- .../explicit-project-dependencies.spec.ts | 15 +- .../nx/src/project-graph/project-graph.ts | 20 +-- .../utils/retrieve-workspace-files.ts | 139 +++++------------- packages/nx/src/utils/workspace-context.ts | 14 +- 14 files changed, 202 insertions(+), 308 deletions(-) diff --git a/packages/devkit/src/utils/convert-nx-executor.ts b/packages/devkit/src/utils/convert-nx-executor.ts index 8bae918420..fee060ba3c 100644 --- a/packages/devkit/src/utils/convert-nx-executor.ts +++ b/packages/devkit/src/utils/convert-nx-executor.ts @@ -32,7 +32,7 @@ export function convertNxExecutor(executor: Executor) { projects: await retrieveProjectConfigurationsWithAngularProjects( builderContext.workspaceRoot, nxJsonConfiguration - ).then((p) => p.projectNodes), + ).then((p) => (p as any).projectNodes ?? p.projects), } : // TODO(v18): remove retrieveProjectConfigurations. This is to be backwards compatible with Nx 16.5 and below. (workspaces as any).readProjectsConfigurations({ diff --git a/packages/nx/src/config/workspaces.spec.ts b/packages/nx/src/config/workspaces.spec.ts index edef764021..ae401485c4 100644 --- a/packages/nx/src/config/workspaces.spec.ts +++ b/packages/nx/src/config/workspaces.spec.ts @@ -60,13 +60,13 @@ describe('Workspaces', () => { }), }); - const { projectNodes } = await withEnvironmentVariables( + const { projects } = await withEnvironmentVariables( { NX_WORKSPACE_ROOT: fs.tempDir, }, () => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir)) ); - expect(projectNodes['my-package']).toEqual({ + expect(projects['my-package']).toEqual({ name: 'my-package', root: 'packages/my-package', sourceRoot: 'packages/my-package', diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index 081a8b67ad..8393848c1e 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -16,6 +16,7 @@ import { readFileMapCache, } from '../../project-graph/nx-deps-cache'; import { + RetrievedGraphNodes, retrieveProjectConfigurations, retrieveWorkspaceFiles, } from '../../project-graph/utils/retrieve-workspace-files'; @@ -144,34 +145,23 @@ function computeWorkspaceConfigHash( } async function processCollectedUpdatedAndDeletedFiles( - projects: Record, - nxJson: NxJsonConfiguration + { projects, externalNodes, projectRootMap }: RetrievedGraphNodes, + updatedFileHashes: Record, + deletedFiles: string[] ) { try { - performance.mark('hash-watched-changes-start'); - const updatedFiles = [...collectedUpdatedFiles.values()]; - const deletedFiles = [...collectedDeletedFiles.values()]; - let updatedFileHashes = updateFilesInContext(updatedFiles, deletedFiles); - performance.mark('hash-watched-changes-end'); - performance.measure( - 'hash changed files from watcher', - 'hash-watched-changes-start', - 'hash-watched-changes-end' - ); - const workspaceConfigHash = computeWorkspaceConfigHash(projects); - serverLogger.requestLog( - `Updated file-hasher based on watched changes, recomputing project graph...` - ); - serverLogger.requestLog([...updatedFiles.values()]); - serverLogger.requestLog([...deletedFiles]); // when workspace config changes we cannot incrementally update project file map if (workspaceConfigHash !== storedWorkspaceConfigHash) { storedWorkspaceConfigHash = workspaceConfigHash; - ({ externalNodes: knownExternalNodes, ...fileMapWithFiles } = - await retrieveWorkspaceFiles(workspaceRoot, nxJson)); + ({ ...fileMapWithFiles } = await retrieveWorkspaceFiles( + workspaceRoot, + projectRootMap + )); + + knownExternalNodes = externalNodes; } else { if (fileMapWithFiles) { fileMapWithFiles = updateFileMap( @@ -181,7 +171,10 @@ async function processCollectedUpdatedAndDeletedFiles( deletedFiles ); } else { - fileMapWithFiles = await retrieveWorkspaceFiles(workspaceRoot, nxJson); + fileMapWithFiles = await retrieveWorkspaceFiles( + workspaceRoot, + projectRootMap + ); } } @@ -205,17 +198,33 @@ async function processCollectedUpdatedAndDeletedFiles( async function processFilesAndCreateAndSerializeProjectGraph() { try { + performance.mark('hash-watched-changes-start'); + const updatedFiles = [...collectedUpdatedFiles.values()]; + const deletedFiles = [...collectedDeletedFiles.values()]; + let updatedFileHashes = updateFilesInContext(updatedFiles, deletedFiles); + performance.mark('hash-watched-changes-end'); + performance.measure( + 'hash changed files from watcher', + 'hash-watched-changes-start', + 'hash-watched-changes-end' + ); + serverLogger.requestLog( + `Updated workspace context based on watched changes, recomputing project graph...` + ); + serverLogger.requestLog([...updatedFiles.values()]); + serverLogger.requestLog([...deletedFiles]); const nxJson = readNxJson(workspaceRoot); const configResult = await retrieveProjectConfigurations( workspaceRoot, nxJson ); await processCollectedUpdatedAndDeletedFiles( - configResult.projectNodes, - nxJson + configResult, + updatedFileHashes, + deletedFiles ); writeSourceMaps(configResult.sourceMaps); - return createAndSerializeProjectGraph(configResult.projectNodes); + return createAndSerializeProjectGraph(configResult); } catch (err) { return Promise.resolve({ error: err, @@ -243,9 +252,9 @@ function copyFileMap(m: FileMap) { return c; } -async function createAndSerializeProjectGraph( - projects: Record -): Promise<{ +async function createAndSerializeProjectGraph({ + projects, +}: RetrievedGraphNodes): Promise<{ error: string | null; projectGraph: ProjectGraph | null; fileMap: FileMap | null; diff --git a/packages/nx/src/executors/utils/convert-nx-executor.ts b/packages/nx/src/executors/utils/convert-nx-executor.ts index bc618dcc4d..40b2566d91 100644 --- a/packages/nx/src/executors/utils/convert-nx-executor.ts +++ b/packages/nx/src/executors/utils/convert-nx-executor.ts @@ -24,7 +24,7 @@ export function convertNxExecutor(executor: Executor) { builderContext.workspaceRoot, nxJsonConfiguration ) - ).projectNodes, + ).projects, }; const context: ExecutorContext = { root: builderContext.workspaceRoot, diff --git a/packages/nx/src/hasher/native-task-hasher-impl.spec.ts b/packages/nx/src/hasher/native-task-hasher-impl.spec.ts index 43f34cf2c2..99e79cd512 100644 --- a/packages/nx/src/hasher/native-task-hasher-impl.spec.ts +++ b/packages/nx/src/hasher/native-task-hasher-impl.spec.ts @@ -77,16 +77,21 @@ describe('native task hasher', () => { }); it('should create a task hash', async () => { - const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); + const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + 'libs/unrelated': 'unrelated', + 'libs/tagged': 'tagged', + }); const builder = new ProjectGraphBuilder( undefined, workspaceFiles.fileMap.projectFileMap ); + builder.addNode({ name: 'parent', type: 'lib', data: { - root: 'parent', + root: 'libs/parent', targets: { build: { executor: 'nx:run-commands', @@ -149,9 +154,9 @@ describe('native task hasher', () => { "AllExternalDependencies": "3244421341483603138", "env:NONEXISTENTENV": "3244421341483603138", "env:TESTENV": "11441948532827618368", - "parent:ProjectConfiguration": "15828052557461792163", + "parent:ProjectConfiguration": "4131510303084753861", "parent:TsConfig": "2264969541778889434", - "parent:{projectRoot}/**/*": "3244421341483603138", + "parent:{projectRoot}/**/*": "15295586939211629225", "runtime:echo runtime123": "29846575039086708", "tagged:ProjectConfiguration": "1604492097835699503", "tagged:TsConfig": "2264969541778889434", @@ -163,17 +168,17 @@ describe('native task hasher', () => { "{workspaceRoot}/.nxignore": "3244421341483603138", "{workspaceRoot}/nx.json": "5219582320960288192", }, - "value": "2902224107680327789", + "value": "6332317845632665670", }, ] `); }); it('should hash tasks where the project has dependencies', async () => { - console.log('read first', await tempFs.readFile('nx.json')); - const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); - console.dir(workspaceFiles.allWorkspaceFiles); - console.log('read second', await tempFs.readFile('nx.json')); + const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + 'libs/child': 'child', + }); const builder = new ProjectGraphBuilder( undefined, workspaceFiles.fileMap.projectFileMap @@ -243,10 +248,10 @@ describe('native task hasher', () => { } as any; tempFs.writeFile('nx.json', JSON.stringify(nxJsonModified)); - const workspaceFiles = await retrieveWorkspaceFiles( - tempFs.tempDir, - nxJsonModified - ); + const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + 'libs/child': 'child', + }); let builder = new ProjectGraphBuilder( undefined, @@ -323,7 +328,9 @@ describe('native task hasher', () => { }, }; tempFs.writeFile('nx.json', JSON.stringify(nxJson)); - const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); + const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + }); let builder = new ProjectGraphBuilder( undefined, workspaceFiles.fileMap.projectFileMap @@ -403,7 +410,10 @@ describe('native task hasher', () => { }, }; tempFs.writeFile('nx.json', JSON.stringify(nxJson)); - const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); + const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + 'libs/child': 'child', + }); const builder = new ProjectGraphBuilder( undefined, workspaceFiles.fileMap.projectFileMap @@ -486,7 +496,9 @@ describe('native task hasher', () => { }); it('should be able to include only a part of the base tsconfig', async () => { - let workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); + let workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + }); const builder = new ProjectGraphBuilder( undefined, workspaceFiles.fileMap.projectFileMap @@ -539,7 +551,10 @@ describe('native task hasher', () => { }); it('should hash tasks where the project graph has circular dependencies', async () => { - const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); + const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, { + 'libs/parent': 'parent', + 'libs/child': 'child', + }); const builder = new ProjectGraphBuilder( undefined, workspaceFiles.fileMap.projectFileMap diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index 201881840b..68e70c7b68 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -170,10 +170,9 @@ export class Watcher { export class WorkspaceContext { workspaceRoot: string constructor(workspaceRoot: string) - getWorkspaceFiles(globs: Array, parseConfigurations: (arg0: Array) => Promise>): Promise + getWorkspaceFiles(projectRootMap: Record): NxWorkspaceFiles glob(globs: Array, exclude?: Array | undefined | null): Array hashFilesMatchingGlob(globs: Array, exclude?: Array | undefined | null): string - getProjectConfigurations(globs: Array, parseConfigurations: (arg0: Array) => Promise>): Promise> incrementalUpdate(updatedFiles: Array, deletedFiles: Array): Record updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject, globalFiles: ExternalObject>, updatedFiles: Record, deletedFiles: Array): UpdatedWorkspaceFiles allFileData(): Array diff --git a/packages/nx/src/native/tests/workspace_files.spec.ts b/packages/nx/src/native/tests/workspace_files.spec.ts index 21d527546d..0d661e7f3e 100644 --- a/packages/nx/src/native/tests/workspace_files.spec.ts +++ b/packages/nx/src/native/tests/workspace_files.spec.ts @@ -48,12 +48,16 @@ describe('workspace files', () => { './libs/package-project/index.js': '', './nested/non-project/file.txt': '', }); - let globs = ['project.json', '**/project.json', 'libs/*/package.json']; const context = new WorkspaceContext(fs.tempDir); let { projectFileMap, globalFiles } = await context.getWorkspaceFiles( - globs, - createParseConfigurationsFunction(fs.tempDir) + { + 'libs/project1': 'project1', + 'libs/project2': 'project2', + 'libs/project3': 'project3', + 'libs/nested/project': 'nested-project', + 'libs/package-project': 'package-project' + } ); expect(projectFileMap).toMatchInlineSnapshot(` @@ -147,10 +151,10 @@ describe('workspace files', () => { const context = new WorkspaceContext(fs.tempDir); - const globs = ['project.json', '**/project.json', '**/package.json']; const { globalFiles, projectFileMap } = await context.getWorkspaceFiles( - globs, - createParseConfigurationsFunction(fs.tempDir) + { + '.': 'repo-name' + } ); expect(globalFiles).toEqual([]); diff --git a/packages/nx/src/native/workspace/config_files.rs b/packages/nx/src/native/workspace/config_files.rs index d38c2ec81c..a8657674e6 100644 --- a/packages/nx/src/native/workspace/config_files.rs +++ b/packages/nx/src/native/workspace/config_files.rs @@ -1,6 +1,3 @@ -use std::collections::HashMap; - -use napi::bindgen_prelude::Promise; use rayon::prelude::*; use crate::native::glob::build_glob_set; @@ -32,17 +29,3 @@ pub(super) fn glob_files( .unwrap_or(is_match) })) } - -/// Get workspace config files based on provided globs -pub(super) fn get_project_configurations( - globs: Vec, - files: &[FileData], - parse_configurations: ConfigurationParser, -) -> napi::Result>> -where - ConfigurationParser: Fn(Vec) -> napi::Result>>, -{ - let files = glob_files(files, globs, None).map_err(anyhow::Error::from)?; - - parse_configurations(files.map(|file| file.file.to_owned()).collect()) -} diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index 315d785b22..4816020b84 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use crate::native::hasher::{hash, hash_file_path}; use crate::native::utils::Normalize; -use napi::bindgen_prelude::*; use rayon::prelude::*; use std::ops::Deref; use std::path::{Path, PathBuf}; @@ -21,7 +20,7 @@ use crate::native::walker::nx_walker; use crate::native::workspace::types::{ FileMap, NxWorkspaceFilesExternals, ProjectFiles, UpdatedWorkspaceFiles, }; -use crate::native::workspace::{config_files, workspace_files}; +use crate::native::workspace::{config_files, workspace_files, types::NxWorkspaceFiles}; #[napi] pub struct WorkspaceContext { @@ -177,17 +176,13 @@ impl WorkspaceContext { } } - #[napi(ts_return_type = "Promise")] - pub fn get_workspace_files( + #[napi] + pub fn get_workspace_files( &self, - env: Env, - globs: Vec, - parse_configurations: ConfigurationParser, - ) -> anyhow::Result> - where - ConfigurationParser: Fn(Vec) -> napi::Result>>, + project_root_map: HashMap, + ) -> anyhow::Result { - workspace_files::get_files(env, globs, parse_configurations, self.all_file_data()) + workspace_files::get_files(project_root_map, self.all_file_data()) .map_err(anyhow::Error::from) } @@ -218,27 +213,6 @@ impl WorkspaceContext { )) } - #[napi(ts_return_type = "Promise>")] - pub fn get_project_configurations( - &self, - env: Env, - globs: Vec, - parse_configurations: ConfigurationParser, - ) -> napi::Result - where - ConfigurationParser: Fn(Vec) -> napi::Result>>, - { - let promise = config_files::get_project_configurations( - globs, - &self.all_file_data(), - parse_configurations, - )?; - env.spawn_future(async move { - let result = promise.await?; - Ok(result) - }) - } - #[napi] pub fn incremental_update( &self, diff --git a/packages/nx/src/native/workspace/workspace_files.rs b/packages/nx/src/native/workspace/workspace_files.rs index 2b255e4c28..fed0d4dcbc 100644 --- a/packages/nx/src/native/workspace/workspace_files.rs +++ b/packages/nx/src/native/workspace/workspace_files.rs @@ -1,97 +1,79 @@ -use napi::bindgen_prelude::{Object, Promise}; use std::collections::HashMap; use std::path::{Path, PathBuf}; use napi::bindgen_prelude::External; -use napi::Env; use rayon::prelude::*; use tracing::trace; use crate::native::types::FileData; -use crate::native::workspace::config_files; use crate::native::workspace::types::{FileLocation, NxWorkspaceFiles, NxWorkspaceFilesExternals}; -pub(super) fn get_files( - env: Env, - globs: Vec, - parse_configurations: ConfigurationParser, +pub(super) fn get_files( + project_root_map: HashMap, files: Vec, -) -> napi::Result> -where - ConfigurationParser: Fn(Vec) -> napi::Result>>, -{ +) -> napi::Result { if files.is_empty() { return Ok(Default::default()); }; - trace!("{globs:?}"); - let promise = config_files::get_project_configurations(globs, &files, parse_configurations)?; + let root_map = transform_root_map(project_root_map); - let result = env.spawn_future(async move { - let parsed_graph_nodes = promise.await?; + trace!(?root_map); - let root_map = transform_root_map(parsed_graph_nodes); + let file_locations = files + .par_iter() + .cloned() + .map(|file_data| { + let file_path = PathBuf::from(&file_data.file); + let mut parent = file_path.parent().unwrap_or_else(|| Path::new(".")); - trace!(?root_map); - - let file_locations = files - .par_iter() - .cloned() - .map(|file_data| { - let file_path = PathBuf::from(&file_data.file); - let mut parent = file_path.parent().unwrap_or_else(|| Path::new(".")); - - while root_map.get(parent).is_none() && parent != Path::new(".") { - parent = parent.parent().unwrap_or_else(|| Path::new(".")); - } - - match root_map.get(parent) { - Some(project_name) => (FileLocation::Project(project_name.into()), file_data), - None => (FileLocation::Global, file_data), - } - }) - .collect::>(); - - let mut project_file_map: HashMap> = HashMap::with_capacity( - file_locations - .iter() - .filter(|&f| f.0 != FileLocation::Global) - .count(), - ); - let mut global_files: Vec = Vec::with_capacity( - file_locations - .iter() - .filter(|&f| f.0 == FileLocation::Global) - .count(), - ); - for (file_location, file_data) in file_locations { - match file_location { - FileLocation::Global => global_files.push(file_data), - FileLocation::Project(project_name) => { - match project_file_map.get_mut(&project_name) { - None => { - project_file_map.insert(project_name.clone(), vec![file_data]); - } - Some(project_files) => project_files.push(file_data), - } - } + while root_map.get(parent).is_none() && parent != Path::new(".") { + parent = parent.parent().unwrap_or_else(|| Path::new(".")); } - } - let project_files_external = External::new(project_file_map.clone()); - let global_files_external = External::new(global_files.clone()); - let all_workspace_files = External::new(files); - Ok(NxWorkspaceFiles { - project_file_map, - global_files, - external_references: Some(NxWorkspaceFilesExternals { - project_files: project_files_external, - global_files: global_files_external, - all_workspace_files, - }), + match root_map.get(parent) { + Some(project_name) => (FileLocation::Project(project_name.into()), file_data), + None => (FileLocation::Global, file_data), + } }) - })?; - Ok(Some(result)) + .collect::>(); + + let mut project_file_map: HashMap> = HashMap::with_capacity( + file_locations + .iter() + .filter(|&f| f.0 != FileLocation::Global) + .count(), + ); + let mut global_files: Vec = Vec::with_capacity( + file_locations + .iter() + .filter(|&f| f.0 == FileLocation::Global) + .count(), + ); + for (file_location, file_data) in file_locations { + match file_location { + FileLocation::Global => global_files.push(file_data), + FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) { + None => { + project_file_map.insert(project_name.clone(), vec![file_data]); + } + Some(project_files) => project_files.push(file_data), + }, + } + } + + let project_files_external = External::new(project_file_map.clone()); + let global_files_external = External::new(global_files.clone()); + let all_workspace_files = External::new(files); + Ok(NxWorkspaceFiles { + project_file_map, + global_files, + external_references: Some(NxWorkspaceFilesExternals { + project_files: project_files_external, + global_files: global_files_external, + all_workspace_files, + }), + }) } fn transform_root_map(root_map: HashMap) -> hashbrown::HashMap { diff --git a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts index b3207602f6..0cf2c22dd9 100644 --- a/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/build-dependencies/explicit-project-dependencies.spec.ts @@ -3,7 +3,11 @@ const tempFs = new TempFs('explicit-project-deps'); import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder'; import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies'; -import { retrieveWorkspaceFiles } from '../../../../project-graph/utils/retrieve-workspace-files'; +import { + retrieveProjectConfigurationPaths, + retrieveProjectConfigurations, + retrieveWorkspaceFiles, +} from '../../../../project-graph/utils/retrieve-workspace-files'; import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; import { setupWorkspaceContext } from '../../../../utils/workspace-context'; @@ -560,14 +564,19 @@ async function createContext( setupWorkspaceContext(tempFs.tempDir); - const { fileMap, projectConfigurations } = await retrieveWorkspaceFiles( + const { projects, projectRootMap } = await retrieveProjectConfigurations( tempFs.tempDir, nxJson ); + const { fileMap } = await retrieveWorkspaceFiles( + tempFs.tempDir, + projectRootMap + ); + return { externalNodes: builder.getUpdatedProjectGraph().externalNodes, - projects: projectConfigurations.projects, + projects: projects, nxJsonConfiguration: nxJson, filesToProcess: fileMap, fileMap: fileMap, diff --git a/packages/nx/src/project-graph/project-graph.ts b/packages/nx/src/project-graph/project-graph.ts index 1a119d97d0..0cb551344e 100644 --- a/packages/nx/src/project-graph/project-graph.ts +++ b/packages/nx/src/project-graph/project-graph.ts @@ -12,7 +12,10 @@ import { daemonClient } from '../daemon/client/client'; import { fileExists } from '../utils/fileutils'; import { workspaceRoot } from '../utils/workspace-root'; import { performance } from 'perf_hooks'; -import { retrieveWorkspaceFiles } from './utils/retrieve-workspace-files'; +import { + retrieveProjectConfigurations, + retrieveWorkspaceFiles, +} from './utils/retrieve-workspace-files'; import { readNxJson } from '../config/nx-json'; import { unregisterPluginTSTranspiler } from '../utils/nx-plugin'; import { writeSourceMaps } from '../utils/source-maps'; @@ -78,19 +81,16 @@ export function readProjectsConfigurationFromProjectGraph( export async function buildProjectGraphWithoutDaemon() { const nxJson = readNxJson(); - const { - allWorkspaceFiles, - fileMap, - projectConfigurations, - externalNodes, - sourceMaps, - rustReferences, - } = await retrieveWorkspaceFiles(workspaceRoot, nxJson); + const { projects, externalNodes, sourceMaps, projectRootMap } = + await retrieveProjectConfigurations(workspaceRoot, nxJson); + + const { allWorkspaceFiles, fileMap, rustReferences } = + await retrieveWorkspaceFiles(workspaceRoot, projectRootMap); const cacheEnabled = process.env.NX_CACHE_PROJECT_GRAPH !== 'false'; const projectGraph = ( await buildProjectGraphUsingProjectFileMap( - projectConfigurations.projects, + projects, externalNodes, fileMap, allWorkspaceFiles, diff --git a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts index 6caac126d4..bfe67207d5 100644 --- a/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts +++ b/packages/nx/src/project-graph/utils/retrieve-workspace-files.ts @@ -24,9 +24,8 @@ import { } from '../../utils/nx-plugin'; import { CreateProjectJsonProjectsPlugin } from '../../plugins/project-json/build-nodes/project-json'; import { - getNxWorkspaceFilesFromContext, - getProjectConfigurationsFromContext, globWithWorkspaceContext, + getNxWorkspaceFilesFromContext, } from '../../utils/workspace-context'; import { buildAllWorkspaceFiles } from './build-all-workspace-files'; @@ -38,15 +37,9 @@ import { buildAllWorkspaceFiles } from './build-all-workspace-files'; */ export async function retrieveWorkspaceFiles( workspaceRoot: string, - nxJson: NxJsonConfiguration + projectRootMap: Record ) { performance.mark('native-file-deps:start'); - const plugins = await loadNxPlugins( - nxJson?.plugins ?? [], - getNxRequirePaths(workspaceRoot), - workspaceRoot - ); - let globs = configurationGlobs(plugins); performance.mark('native-file-deps:end'); performance.measure( 'native-file-deps', @@ -55,29 +48,9 @@ export async function retrieveWorkspaceFiles( ); performance.mark('get-workspace-files:start'); - let projects: Record; - let externalNodes: Record; - let sourceMaps: ConfigurationSourceMaps; const { projectFileMap, globalFiles, externalReferences } = - (await getNxWorkspaceFilesFromContext( - workspaceRoot, - globs, - async (configs: string[]) => { - const projectConfigurations = await createProjectConfigurations( - workspaceRoot, - nxJson, - configs, - plugins - ); - - projects = projectConfigurations.projects; - sourceMaps = projectConfigurations.sourceMaps; - - externalNodes = projectConfigurations.externalNodes; - return projectConfigurations.rootMap; - } - )) as NxWorkspaceFiles; + getNxWorkspaceFilesFromContext(workspaceRoot, projectRootMap); performance.mark('get-workspace-files:end'); performance.measure( 'get-workspace-files', @@ -91,12 +64,6 @@ export async function retrieveWorkspaceFiles( projectFileMap, nonProjectFiles: globalFiles, }, - projectConfigurations: { - version: 2, - projects, - } as ProjectsConfigurations, - externalNodes, - sourceMaps, rustReferences: externalReferences, }; } @@ -110,29 +77,20 @@ export async function retrieveWorkspaceFiles( export async function retrieveProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration -): Promise<{ - externalNodes: Record; - projectNodes: Record; - sourceMaps: ConfigurationSourceMaps; -}> { +): Promise { const plugins = await loadNxPlugins( nxJson?.plugins ?? [], getNxRequirePaths(workspaceRoot), workspaceRoot ); - const globs = configurationGlobs(plugins); - return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins, globs); + return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins); } export async function retrieveProjectConfigurationsWithAngularProjects( workspaceRoot: string, nxJson: NxJsonConfiguration -): Promise<{ - externalNodes: Record; - projectNodes: Record; - sourceMaps: ConfigurationSourceMaps; -}> { +): Promise { const plugins = await loadNxPlugins( nxJson?.plugins ?? [], getNxRequirePaths(workspaceRoot), @@ -146,46 +104,30 @@ export async function retrieveProjectConfigurationsWithAngularProjects( plugins.push({ plugin: NxAngularJsonPlugin }); } - const globs = configurationGlobs(plugins); - return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins, globs); + return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins); } +export type RetrievedGraphNodes = { + externalNodes: Record; + projects: Record; + sourceMaps: ConfigurationSourceMaps; + projectRootMap: Record; +}; + function _retrieveProjectConfigurations( workspaceRoot: string, nxJson: NxJsonConfiguration, - plugins: LoadedNxPlugin[], - globs: string[] -): Promise<{ - externalNodes: Record; - projectNodes: Record; - sourceMaps: ConfigurationSourceMaps; -}> { - let result: { - externalNodes: Record; - projectNodes: Record; - sourceMaps: ConfigurationSourceMaps; - }; - return getProjectConfigurationsFromContext( + plugins: LoadedNxPlugin[] +): Promise { + const globPatterns = configurationGlobs(plugins); + const projectFiles = globWithWorkspaceContext(workspaceRoot, globPatterns); + + return createProjectConfigurations( workspaceRoot, - globs, - async (configs: string[]) => { - const { projects, externalNodes, rootMap, sourceMaps } = - await createProjectConfigurations( - workspaceRoot, - nxJson, - configs, - plugins - ); - - result = { - projectNodes: projects, - externalNodes: externalNodes, - sourceMaps, - }; - - return rootMap; - } - ).then(() => result); + nxJson, + projectFiles, + plugins + ); } export function retrieveProjectConfigurationPaths( @@ -214,23 +156,15 @@ export async function retrieveProjectConfigurationsWithoutPluginInference( return projectsWithoutPluginCache.get(cacheKey); } - let projects: Record; - await getProjectConfigurationsFromContext( + const projectFiles = globWithWorkspaceContext(root, projectGlobPatterns); + const { projects } = await createProjectConfigurations( root, - projectGlobPatterns, - async (configs: string[]) => { - const projectConfigurations = await createProjectConfigurations( - root, - nxJson, - configs, - [ - { plugin: getNxPackageJsonWorkspacesPlugin(root) }, - { plugin: CreateProjectJsonProjectsPlugin }, - ] - ); - projects = projectConfigurations.projects; - return projectConfigurations.rootMap; - } + nxJson, + projectFiles, + [ + { plugin: getNxPackageJsonWorkspacesPlugin(root) }, + { plugin: CreateProjectJsonProjectsPlugin }, + ] ); projectsWithoutPluginCache.set(cacheKey, projects); @@ -243,12 +177,7 @@ export async function createProjectConfigurations( nxJson: NxJsonConfiguration, configFiles: string[], plugins: LoadedNxPlugin[] -): Promise<{ - projects: Record; - externalNodes: Record; - rootMap: Record; - sourceMaps: ConfigurationSourceMaps; -}> { +): Promise { performance.mark('build-project-configs:start'); const { projects, externalNodes, rootMap, sourceMaps } = @@ -269,7 +198,7 @@ export async function createProjectConfigurations( return { projects, externalNodes, - rootMap, + projectRootMap: rootMap, sourceMaps, }; } diff --git a/packages/nx/src/utils/workspace-context.ts b/packages/nx/src/utils/workspace-context.ts index 14e86e9f6d..39ec8106fa 100644 --- a/packages/nx/src/utils/workspace-context.ts +++ b/packages/nx/src/utils/workspace-context.ts @@ -19,11 +19,10 @@ export function setupWorkspaceContext(workspaceRoot: string) { export function getNxWorkspaceFilesFromContext( workspaceRoot: string, - globs: string[], - parseConfigurations: (files: string[]) => Promise> + projectRootMap: Record ) { ensureContextAvailable(workspaceRoot); - return workspaceContext.getWorkspaceFiles(globs, parseConfigurations); + return workspaceContext.getWorkspaceFiles(projectRootMap); } export function globWithWorkspaceContext( @@ -44,15 +43,6 @@ export function hashWithWorkspaceContext( return workspaceContext.hashFilesMatchingGlob(globs, exclude); } -export function getProjectConfigurationsFromContext( - workspaceRoot: string, - globs: string[], - parseConfigurations: (files: string[]) => Promise> -) { - ensureContextAvailable(workspaceRoot); - return workspaceContext.getProjectConfigurations(globs, parseConfigurations); -} - export function updateFilesInContext( updatedFiles: string[], deletedFiles: string[]