Reapply "cleanup(core): separate retrieve project configurations and … (#20398)

This commit is contained in:
Jason Jean 2023-11-27 09:45:48 -05:00 committed by GitHub
parent eaa1a4a113
commit 68e974c848
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 202 additions and 308 deletions

View File

@ -32,7 +32,7 @@ export function convertNxExecutor(executor: Executor) {
projects: await retrieveProjectConfigurationsWithAngularProjects( projects: await retrieveProjectConfigurationsWithAngularProjects(
builderContext.workspaceRoot, builderContext.workspaceRoot,
nxJsonConfiguration nxJsonConfiguration
).then((p) => p.projectNodes), ).then((p) => (p as any).projectNodes ?? p.projects),
} }
: // TODO(v18): remove retrieveProjectConfigurations. This is to be backwards compatible with Nx 16.5 and below. : // TODO(v18): remove retrieveProjectConfigurations. This is to be backwards compatible with Nx 16.5 and below.
(workspaces as any).readProjectsConfigurations({ (workspaces as any).readProjectsConfigurations({

View File

@ -60,13 +60,13 @@ describe('Workspaces', () => {
}), }),
}); });
const { projectNodes } = await withEnvironmentVariables( const { projects } = await withEnvironmentVariables(
{ {
NX_WORKSPACE_ROOT: fs.tempDir, NX_WORKSPACE_ROOT: fs.tempDir,
}, },
() => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir)) () => retrieveProjectConfigurations(fs.tempDir, readNxJson(fs.tempDir))
); );
expect(projectNodes['my-package']).toEqual({ expect(projects['my-package']).toEqual({
name: 'my-package', name: 'my-package',
root: 'packages/my-package', root: 'packages/my-package',
sourceRoot: 'packages/my-package', sourceRoot: 'packages/my-package',

View File

@ -16,6 +16,7 @@ import {
readFileMapCache, readFileMapCache,
} from '../../project-graph/nx-deps-cache'; } from '../../project-graph/nx-deps-cache';
import { import {
RetrievedGraphNodes,
retrieveProjectConfigurations, retrieveProjectConfigurations,
retrieveWorkspaceFiles, retrieveWorkspaceFiles,
} from '../../project-graph/utils/retrieve-workspace-files'; } from '../../project-graph/utils/retrieve-workspace-files';
@ -144,34 +145,23 @@ function computeWorkspaceConfigHash(
} }
async function processCollectedUpdatedAndDeletedFiles( async function processCollectedUpdatedAndDeletedFiles(
projects: Record<string, ProjectConfiguration>, { projects, externalNodes, projectRootMap }: RetrievedGraphNodes,
nxJson: NxJsonConfiguration updatedFileHashes: Record<string, string>,
deletedFiles: string[]
) { ) {
try { try {
performance.mark('hash-watched-changes-start');
const updatedFiles = [...collectedUpdatedFiles.values()];
const deletedFiles = [...collectedDeletedFiles.values()];
let updatedFileHashes = updateFilesInContext(updatedFiles, deletedFiles);
performance.mark('hash-watched-changes-end');
performance.measure(
'hash changed files from watcher',
'hash-watched-changes-start',
'hash-watched-changes-end'
);
const workspaceConfigHash = computeWorkspaceConfigHash(projects); const workspaceConfigHash = computeWorkspaceConfigHash(projects);
serverLogger.requestLog(
`Updated file-hasher based on watched changes, recomputing project graph...`
);
serverLogger.requestLog([...updatedFiles.values()]);
serverLogger.requestLog([...deletedFiles]);
// when workspace config changes we cannot incrementally update project file map // when workspace config changes we cannot incrementally update project file map
if (workspaceConfigHash !== storedWorkspaceConfigHash) { if (workspaceConfigHash !== storedWorkspaceConfigHash) {
storedWorkspaceConfigHash = workspaceConfigHash; storedWorkspaceConfigHash = workspaceConfigHash;
({ externalNodes: knownExternalNodes, ...fileMapWithFiles } = ({ ...fileMapWithFiles } = await retrieveWorkspaceFiles(
await retrieveWorkspaceFiles(workspaceRoot, nxJson)); workspaceRoot,
projectRootMap
));
knownExternalNodes = externalNodes;
} else { } else {
if (fileMapWithFiles) { if (fileMapWithFiles) {
fileMapWithFiles = updateFileMap( fileMapWithFiles = updateFileMap(
@ -181,7 +171,10 @@ async function processCollectedUpdatedAndDeletedFiles(
deletedFiles deletedFiles
); );
} else { } else {
fileMapWithFiles = await retrieveWorkspaceFiles(workspaceRoot, nxJson); fileMapWithFiles = await retrieveWorkspaceFiles(
workspaceRoot,
projectRootMap
);
} }
} }
@ -205,17 +198,33 @@ async function processCollectedUpdatedAndDeletedFiles(
async function processFilesAndCreateAndSerializeProjectGraph() { async function processFilesAndCreateAndSerializeProjectGraph() {
try { try {
performance.mark('hash-watched-changes-start');
const updatedFiles = [...collectedUpdatedFiles.values()];
const deletedFiles = [...collectedDeletedFiles.values()];
let updatedFileHashes = updateFilesInContext(updatedFiles, deletedFiles);
performance.mark('hash-watched-changes-end');
performance.measure(
'hash changed files from watcher',
'hash-watched-changes-start',
'hash-watched-changes-end'
);
serverLogger.requestLog(
`Updated workspace context based on watched changes, recomputing project graph...`
);
serverLogger.requestLog([...updatedFiles.values()]);
serverLogger.requestLog([...deletedFiles]);
const nxJson = readNxJson(workspaceRoot); const nxJson = readNxJson(workspaceRoot);
const configResult = await retrieveProjectConfigurations( const configResult = await retrieveProjectConfigurations(
workspaceRoot, workspaceRoot,
nxJson nxJson
); );
await processCollectedUpdatedAndDeletedFiles( await processCollectedUpdatedAndDeletedFiles(
configResult.projectNodes, configResult,
nxJson updatedFileHashes,
deletedFiles
); );
writeSourceMaps(configResult.sourceMaps); writeSourceMaps(configResult.sourceMaps);
return createAndSerializeProjectGraph(configResult.projectNodes); return createAndSerializeProjectGraph(configResult);
} catch (err) { } catch (err) {
return Promise.resolve({ return Promise.resolve({
error: err, error: err,
@ -243,9 +252,9 @@ function copyFileMap(m: FileMap) {
return c; return c;
} }
async function createAndSerializeProjectGraph( async function createAndSerializeProjectGraph({
projects: Record<string, ProjectConfiguration> projects,
): Promise<{ }: RetrievedGraphNodes): Promise<{
error: string | null; error: string | null;
projectGraph: ProjectGraph | null; projectGraph: ProjectGraph | null;
fileMap: FileMap | null; fileMap: FileMap | null;

View File

@ -24,7 +24,7 @@ export function convertNxExecutor(executor: Executor) {
builderContext.workspaceRoot, builderContext.workspaceRoot,
nxJsonConfiguration nxJsonConfiguration
) )
).projectNodes, ).projects,
}; };
const context: ExecutorContext = { const context: ExecutorContext = {
root: builderContext.workspaceRoot, root: builderContext.workspaceRoot,

View File

@ -77,16 +77,21 @@ describe('native task hasher', () => {
}); });
it('should create a task hash', async () => { it('should create a task hash', async () => {
const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
'libs/parent': 'parent',
'libs/unrelated': 'unrelated',
'libs/tagged': 'tagged',
});
const builder = new ProjectGraphBuilder( const builder = new ProjectGraphBuilder(
undefined, undefined,
workspaceFiles.fileMap.projectFileMap workspaceFiles.fileMap.projectFileMap
); );
builder.addNode({ builder.addNode({
name: 'parent', name: 'parent',
type: 'lib', type: 'lib',
data: { data: {
root: 'parent', root: 'libs/parent',
targets: { targets: {
build: { build: {
executor: 'nx:run-commands', executor: 'nx:run-commands',
@ -149,9 +154,9 @@ describe('native task hasher', () => {
"AllExternalDependencies": "3244421341483603138", "AllExternalDependencies": "3244421341483603138",
"env:NONEXISTENTENV": "3244421341483603138", "env:NONEXISTENTENV": "3244421341483603138",
"env:TESTENV": "11441948532827618368", "env:TESTENV": "11441948532827618368",
"parent:ProjectConfiguration": "15828052557461792163", "parent:ProjectConfiguration": "4131510303084753861",
"parent:TsConfig": "2264969541778889434", "parent:TsConfig": "2264969541778889434",
"parent:{projectRoot}/**/*": "3244421341483603138", "parent:{projectRoot}/**/*": "15295586939211629225",
"runtime:echo runtime123": "29846575039086708", "runtime:echo runtime123": "29846575039086708",
"tagged:ProjectConfiguration": "1604492097835699503", "tagged:ProjectConfiguration": "1604492097835699503",
"tagged:TsConfig": "2264969541778889434", "tagged:TsConfig": "2264969541778889434",
@ -163,17 +168,17 @@ describe('native task hasher', () => {
"{workspaceRoot}/.nxignore": "3244421341483603138", "{workspaceRoot}/.nxignore": "3244421341483603138",
"{workspaceRoot}/nx.json": "5219582320960288192", "{workspaceRoot}/nx.json": "5219582320960288192",
}, },
"value": "2902224107680327789", "value": "6332317845632665670",
}, },
] ]
`); `);
}); });
it('should hash tasks where the project has dependencies', async () => { it('should hash tasks where the project has dependencies', async () => {
console.log('read first', await tempFs.readFile('nx.json')); const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); 'libs/parent': 'parent',
console.dir(workspaceFiles.allWorkspaceFiles); 'libs/child': 'child',
console.log('read second', await tempFs.readFile('nx.json')); });
const builder = new ProjectGraphBuilder( const builder = new ProjectGraphBuilder(
undefined, undefined,
workspaceFiles.fileMap.projectFileMap workspaceFiles.fileMap.projectFileMap
@ -243,10 +248,10 @@ describe('native task hasher', () => {
} as any; } as any;
tempFs.writeFile('nx.json', JSON.stringify(nxJsonModified)); tempFs.writeFile('nx.json', JSON.stringify(nxJsonModified));
const workspaceFiles = await retrieveWorkspaceFiles( const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
tempFs.tempDir, 'libs/parent': 'parent',
nxJsonModified 'libs/child': 'child',
); });
let builder = new ProjectGraphBuilder( let builder = new ProjectGraphBuilder(
undefined, undefined,
@ -323,7 +328,9 @@ describe('native task hasher', () => {
}, },
}; };
tempFs.writeFile('nx.json', JSON.stringify(nxJson)); tempFs.writeFile('nx.json', JSON.stringify(nxJson));
const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
'libs/parent': 'parent',
});
let builder = new ProjectGraphBuilder( let builder = new ProjectGraphBuilder(
undefined, undefined,
workspaceFiles.fileMap.projectFileMap workspaceFiles.fileMap.projectFileMap
@ -403,7 +410,10 @@ describe('native task hasher', () => {
}, },
}; };
tempFs.writeFile('nx.json', JSON.stringify(nxJson)); tempFs.writeFile('nx.json', JSON.stringify(nxJson));
const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
'libs/parent': 'parent',
'libs/child': 'child',
});
const builder = new ProjectGraphBuilder( const builder = new ProjectGraphBuilder(
undefined, undefined,
workspaceFiles.fileMap.projectFileMap workspaceFiles.fileMap.projectFileMap
@ -486,7 +496,9 @@ describe('native task hasher', () => {
}); });
it('should be able to include only a part of the base tsconfig', async () => { it('should be able to include only a part of the base tsconfig', async () => {
let workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); let workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
'libs/parent': 'parent',
});
const builder = new ProjectGraphBuilder( const builder = new ProjectGraphBuilder(
undefined, undefined,
workspaceFiles.fileMap.projectFileMap workspaceFiles.fileMap.projectFileMap
@ -539,7 +551,10 @@ describe('native task hasher', () => {
}); });
it('should hash tasks where the project graph has circular dependencies', async () => { it('should hash tasks where the project graph has circular dependencies', async () => {
const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, nxJson); const workspaceFiles = await retrieveWorkspaceFiles(tempFs.tempDir, {
'libs/parent': 'parent',
'libs/child': 'child',
});
const builder = new ProjectGraphBuilder( const builder = new ProjectGraphBuilder(
undefined, undefined,
workspaceFiles.fileMap.projectFileMap workspaceFiles.fileMap.projectFileMap

View File

@ -170,10 +170,9 @@ export class Watcher {
export class WorkspaceContext { export class WorkspaceContext {
workspaceRoot: string workspaceRoot: string
constructor(workspaceRoot: string) constructor(workspaceRoot: string)
getWorkspaceFiles(globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Promise<Record<string, string>>): Promise<NxWorkspaceFiles> getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string> glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string
getProjectConfigurations(globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Promise<Record<string, string>>): Promise<Record<string, string>>
incrementalUpdate(updatedFiles: Array<string>, deletedFiles: Array<string>): Record<string, string> incrementalUpdate(updatedFiles: Array<string>, deletedFiles: Array<string>): Record<string, string>
updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject<ProjectFiles>, globalFiles: ExternalObject<Array<FileData>>, updatedFiles: Record<string, string>, deletedFiles: Array<string>): UpdatedWorkspaceFiles updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject<ProjectFiles>, globalFiles: ExternalObject<Array<FileData>>, updatedFiles: Record<string, string>, deletedFiles: Array<string>): UpdatedWorkspaceFiles
allFileData(): Array<FileData> allFileData(): Array<FileData>

View File

@ -48,12 +48,16 @@ describe('workspace files', () => {
'./libs/package-project/index.js': '', './libs/package-project/index.js': '',
'./nested/non-project/file.txt': '', './nested/non-project/file.txt': '',
}); });
let globs = ['project.json', '**/project.json', 'libs/*/package.json'];
const context = new WorkspaceContext(fs.tempDir); const context = new WorkspaceContext(fs.tempDir);
let { projectFileMap, globalFiles } = await context.getWorkspaceFiles( let { projectFileMap, globalFiles } = await context.getWorkspaceFiles(
globs, {
createParseConfigurationsFunction(fs.tempDir) 'libs/project1': 'project1',
'libs/project2': 'project2',
'libs/project3': 'project3',
'libs/nested/project': 'nested-project',
'libs/package-project': 'package-project'
}
); );
expect(projectFileMap).toMatchInlineSnapshot(` expect(projectFileMap).toMatchInlineSnapshot(`
@ -147,10 +151,10 @@ describe('workspace files', () => {
const context = new WorkspaceContext(fs.tempDir); const context = new WorkspaceContext(fs.tempDir);
const globs = ['project.json', '**/project.json', '**/package.json'];
const { globalFiles, projectFileMap } = await context.getWorkspaceFiles( const { globalFiles, projectFileMap } = await context.getWorkspaceFiles(
globs, {
createParseConfigurationsFunction(fs.tempDir) '.': 'repo-name'
}
); );
expect(globalFiles).toEqual([]); expect(globalFiles).toEqual([]);

View File

@ -1,6 +1,3 @@
use std::collections::HashMap;
use napi::bindgen_prelude::Promise;
use rayon::prelude::*; use rayon::prelude::*;
use crate::native::glob::build_glob_set; use crate::native::glob::build_glob_set;
@ -32,17 +29,3 @@ pub(super) fn glob_files(
.unwrap_or(is_match) .unwrap_or(is_match)
})) }))
} }
/// Get workspace config files based on provided globs
pub(super) fn get_project_configurations<ConfigurationParser>(
globs: Vec<String>,
files: &[FileData],
parse_configurations: ConfigurationParser,
) -> napi::Result<Promise<HashMap<String, String>>>
where
ConfigurationParser: Fn(Vec<String>) -> napi::Result<Promise<HashMap<String, String>>>,
{
let files = glob_files(files, globs, None).map_err(anyhow::Error::from)?;
parse_configurations(files.map(|file| file.file.to_owned()).collect())
}

View File

@ -3,7 +3,6 @@ use std::collections::HashMap;
use crate::native::hasher::{hash, hash_file_path}; use crate::native::hasher::{hash, hash_file_path};
use crate::native::utils::Normalize; use crate::native::utils::Normalize;
use napi::bindgen_prelude::*;
use rayon::prelude::*; use rayon::prelude::*;
use std::ops::Deref; use std::ops::Deref;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
@ -21,7 +20,7 @@ use crate::native::walker::nx_walker;
use crate::native::workspace::types::{ use crate::native::workspace::types::{
FileMap, NxWorkspaceFilesExternals, ProjectFiles, UpdatedWorkspaceFiles, FileMap, NxWorkspaceFilesExternals, ProjectFiles, UpdatedWorkspaceFiles,
}; };
use crate::native::workspace::{config_files, workspace_files}; use crate::native::workspace::{config_files, workspace_files, types::NxWorkspaceFiles};
#[napi] #[napi]
pub struct WorkspaceContext { pub struct WorkspaceContext {
@ -177,17 +176,13 @@ impl WorkspaceContext {
} }
} }
#[napi(ts_return_type = "Promise<NxWorkspaceFiles>")] #[napi]
pub fn get_workspace_files<ConfigurationParser>( pub fn get_workspace_files(
&self, &self,
env: Env, project_root_map: HashMap<String, String>,
globs: Vec<String>, ) -> anyhow::Result<NxWorkspaceFiles>
parse_configurations: ConfigurationParser,
) -> anyhow::Result<Option<Object>>
where
ConfigurationParser: Fn(Vec<String>) -> napi::Result<Promise<HashMap<String, String>>>,
{ {
workspace_files::get_files(env, globs, parse_configurations, self.all_file_data()) workspace_files::get_files(project_root_map, self.all_file_data())
.map_err(anyhow::Error::from) .map_err(anyhow::Error::from)
} }
@ -218,27 +213,6 @@ impl WorkspaceContext {
)) ))
} }
#[napi(ts_return_type = "Promise<Record<string, string>>")]
pub fn get_project_configurations<ConfigurationParser>(
&self,
env: Env,
globs: Vec<String>,
parse_configurations: ConfigurationParser,
) -> napi::Result<Object>
where
ConfigurationParser: Fn(Vec<String>) -> napi::Result<Promise<HashMap<String, String>>>,
{
let promise = config_files::get_project_configurations(
globs,
&self.all_file_data(),
parse_configurations,
)?;
env.spawn_future(async move {
let result = promise.await?;
Ok(result)
})
}
#[napi] #[napi]
pub fn incremental_update( pub fn incremental_update(
&self, &self,

View File

@ -1,36 +1,22 @@
use napi::bindgen_prelude::{Object, Promise};
use std::collections::HashMap; use std::collections::HashMap;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use napi::bindgen_prelude::External; use napi::bindgen_prelude::External;
use napi::Env;
use rayon::prelude::*; use rayon::prelude::*;
use tracing::trace; use tracing::trace;
use crate::native::types::FileData; use crate::native::types::FileData;
use crate::native::workspace::config_files;
use crate::native::workspace::types::{FileLocation, NxWorkspaceFiles, NxWorkspaceFilesExternals}; use crate::native::workspace::types::{FileLocation, NxWorkspaceFiles, NxWorkspaceFilesExternals};
pub(super) fn get_files<ConfigurationParser>( pub(super) fn get_files(
env: Env, project_root_map: HashMap<String, String>,
globs: Vec<String>,
parse_configurations: ConfigurationParser,
files: Vec<FileData>, files: Vec<FileData>,
) -> napi::Result<Option<Object>> ) -> napi::Result<NxWorkspaceFiles> {
where
ConfigurationParser: Fn(Vec<String>) -> napi::Result<Promise<HashMap<String, String>>>,
{
if files.is_empty() { if files.is_empty() {
return Ok(Default::default()); return Ok(Default::default());
}; };
trace!("{globs:?}"); let root_map = transform_root_map(project_root_map);
let promise = config_files::get_project_configurations(globs, &files, parse_configurations)?;
let result = env.spawn_future(async move {
let parsed_graph_nodes = promise.await?;
let root_map = transform_root_map(parsed_graph_nodes);
trace!(?root_map); trace!(?root_map);
@ -67,14 +53,12 @@ where
for (file_location, file_data) in file_locations { for (file_location, file_data) in file_locations {
match file_location { match file_location {
FileLocation::Global => global_files.push(file_data), FileLocation::Global => global_files.push(file_data),
FileLocation::Project(project_name) => { FileLocation::Project(project_name) => match project_file_map.get_mut(&project_name) {
match project_file_map.get_mut(&project_name) {
None => { None => {
project_file_map.insert(project_name.clone(), vec![file_data]); project_file_map.insert(project_name.clone(), vec![file_data]);
} }
Some(project_files) => project_files.push(file_data), Some(project_files) => project_files.push(file_data),
} },
}
} }
} }
@ -90,8 +74,6 @@ where
all_workspace_files, all_workspace_files,
}), }),
}) })
})?;
Ok(Some(result))
} }
fn transform_root_map(root_map: HashMap<String, String>) -> hashbrown::HashMap<PathBuf, String> { fn transform_root_map(root_map: HashMap<String, String>) -> hashbrown::HashMap<PathBuf, String> {

View File

@ -3,7 +3,11 @@ const tempFs = new TempFs('explicit-project-deps');
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder'; import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies'; import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies';
import { retrieveWorkspaceFiles } from '../../../../project-graph/utils/retrieve-workspace-files'; import {
retrieveProjectConfigurationPaths,
retrieveProjectConfigurations,
retrieveWorkspaceFiles,
} from '../../../../project-graph/utils/retrieve-workspace-files';
import { CreateDependenciesContext } from '../../../../utils/nx-plugin'; import { CreateDependenciesContext } from '../../../../utils/nx-plugin';
import { setupWorkspaceContext } from '../../../../utils/workspace-context'; import { setupWorkspaceContext } from '../../../../utils/workspace-context';
@ -560,14 +564,19 @@ async function createContext(
setupWorkspaceContext(tempFs.tempDir); setupWorkspaceContext(tempFs.tempDir);
const { fileMap, projectConfigurations } = await retrieveWorkspaceFiles( const { projects, projectRootMap } = await retrieveProjectConfigurations(
tempFs.tempDir, tempFs.tempDir,
nxJson nxJson
); );
const { fileMap } = await retrieveWorkspaceFiles(
tempFs.tempDir,
projectRootMap
);
return { return {
externalNodes: builder.getUpdatedProjectGraph().externalNodes, externalNodes: builder.getUpdatedProjectGraph().externalNodes,
projects: projectConfigurations.projects, projects: projects,
nxJsonConfiguration: nxJson, nxJsonConfiguration: nxJson,
filesToProcess: fileMap, filesToProcess: fileMap,
fileMap: fileMap, fileMap: fileMap,

View File

@ -12,7 +12,10 @@ import { daemonClient } from '../daemon/client/client';
import { fileExists } from '../utils/fileutils'; import { fileExists } from '../utils/fileutils';
import { workspaceRoot } from '../utils/workspace-root'; import { workspaceRoot } from '../utils/workspace-root';
import { performance } from 'perf_hooks'; import { performance } from 'perf_hooks';
import { retrieveWorkspaceFiles } from './utils/retrieve-workspace-files'; import {
retrieveProjectConfigurations,
retrieveWorkspaceFiles,
} from './utils/retrieve-workspace-files';
import { readNxJson } from '../config/nx-json'; import { readNxJson } from '../config/nx-json';
import { unregisterPluginTSTranspiler } from '../utils/nx-plugin'; import { unregisterPluginTSTranspiler } from '../utils/nx-plugin';
import { writeSourceMaps } from '../utils/source-maps'; import { writeSourceMaps } from '../utils/source-maps';
@ -78,19 +81,16 @@ export function readProjectsConfigurationFromProjectGraph(
export async function buildProjectGraphWithoutDaemon() { export async function buildProjectGraphWithoutDaemon() {
const nxJson = readNxJson(); const nxJson = readNxJson();
const { const { projects, externalNodes, sourceMaps, projectRootMap } =
allWorkspaceFiles, await retrieveProjectConfigurations(workspaceRoot, nxJson);
fileMap,
projectConfigurations, const { allWorkspaceFiles, fileMap, rustReferences } =
externalNodes, await retrieveWorkspaceFiles(workspaceRoot, projectRootMap);
sourceMaps,
rustReferences,
} = await retrieveWorkspaceFiles(workspaceRoot, nxJson);
const cacheEnabled = process.env.NX_CACHE_PROJECT_GRAPH !== 'false'; const cacheEnabled = process.env.NX_CACHE_PROJECT_GRAPH !== 'false';
const projectGraph = ( const projectGraph = (
await buildProjectGraphUsingProjectFileMap( await buildProjectGraphUsingProjectFileMap(
projectConfigurations.projects, projects,
externalNodes, externalNodes,
fileMap, fileMap,
allWorkspaceFiles, allWorkspaceFiles,

View File

@ -24,9 +24,8 @@ import {
} from '../../utils/nx-plugin'; } from '../../utils/nx-plugin';
import { CreateProjectJsonProjectsPlugin } from '../../plugins/project-json/build-nodes/project-json'; import { CreateProjectJsonProjectsPlugin } from '../../plugins/project-json/build-nodes/project-json';
import { import {
getNxWorkspaceFilesFromContext,
getProjectConfigurationsFromContext,
globWithWorkspaceContext, globWithWorkspaceContext,
getNxWorkspaceFilesFromContext,
} from '../../utils/workspace-context'; } from '../../utils/workspace-context';
import { buildAllWorkspaceFiles } from './build-all-workspace-files'; import { buildAllWorkspaceFiles } from './build-all-workspace-files';
@ -38,15 +37,9 @@ import { buildAllWorkspaceFiles } from './build-all-workspace-files';
*/ */
export async function retrieveWorkspaceFiles( export async function retrieveWorkspaceFiles(
workspaceRoot: string, workspaceRoot: string,
nxJson: NxJsonConfiguration projectRootMap: Record<string, string>
) { ) {
performance.mark('native-file-deps:start'); performance.mark('native-file-deps:start');
const plugins = await loadNxPlugins(
nxJson?.plugins ?? [],
getNxRequirePaths(workspaceRoot),
workspaceRoot
);
let globs = configurationGlobs(plugins);
performance.mark('native-file-deps:end'); performance.mark('native-file-deps:end');
performance.measure( performance.measure(
'native-file-deps', 'native-file-deps',
@ -55,29 +48,9 @@ export async function retrieveWorkspaceFiles(
); );
performance.mark('get-workspace-files:start'); performance.mark('get-workspace-files:start');
let projects: Record<string, ProjectConfiguration>;
let externalNodes: Record<string, ProjectGraphExternalNode>;
let sourceMaps: ConfigurationSourceMaps;
const { projectFileMap, globalFiles, externalReferences } = const { projectFileMap, globalFiles, externalReferences } =
(await getNxWorkspaceFilesFromContext( getNxWorkspaceFilesFromContext(workspaceRoot, projectRootMap);
workspaceRoot,
globs,
async (configs: string[]) => {
const projectConfigurations = await createProjectConfigurations(
workspaceRoot,
nxJson,
configs,
plugins
);
projects = projectConfigurations.projects;
sourceMaps = projectConfigurations.sourceMaps;
externalNodes = projectConfigurations.externalNodes;
return projectConfigurations.rootMap;
}
)) as NxWorkspaceFiles;
performance.mark('get-workspace-files:end'); performance.mark('get-workspace-files:end');
performance.measure( performance.measure(
'get-workspace-files', 'get-workspace-files',
@ -91,12 +64,6 @@ export async function retrieveWorkspaceFiles(
projectFileMap, projectFileMap,
nonProjectFiles: globalFiles, nonProjectFiles: globalFiles,
}, },
projectConfigurations: {
version: 2,
projects,
} as ProjectsConfigurations,
externalNodes,
sourceMaps,
rustReferences: externalReferences, rustReferences: externalReferences,
}; };
} }
@ -110,29 +77,20 @@ export async function retrieveWorkspaceFiles(
export async function retrieveProjectConfigurations( export async function retrieveProjectConfigurations(
workspaceRoot: string, workspaceRoot: string,
nxJson: NxJsonConfiguration nxJson: NxJsonConfiguration
): Promise<{ ): Promise<RetrievedGraphNodes> {
externalNodes: Record<string, ProjectGraphExternalNode>;
projectNodes: Record<string, ProjectConfiguration>;
sourceMaps: ConfigurationSourceMaps;
}> {
const plugins = await loadNxPlugins( const plugins = await loadNxPlugins(
nxJson?.plugins ?? [], nxJson?.plugins ?? [],
getNxRequirePaths(workspaceRoot), getNxRequirePaths(workspaceRoot),
workspaceRoot workspaceRoot
); );
const globs = configurationGlobs(plugins); return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins);
return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins, globs);
} }
export async function retrieveProjectConfigurationsWithAngularProjects( export async function retrieveProjectConfigurationsWithAngularProjects(
workspaceRoot: string, workspaceRoot: string,
nxJson: NxJsonConfiguration nxJson: NxJsonConfiguration
): Promise<{ ): Promise<RetrievedGraphNodes> {
externalNodes: Record<string, ProjectGraphExternalNode>;
projectNodes: Record<string, ProjectConfiguration>;
sourceMaps: ConfigurationSourceMaps;
}> {
const plugins = await loadNxPlugins( const plugins = await loadNxPlugins(
nxJson?.plugins ?? [], nxJson?.plugins ?? [],
getNxRequirePaths(workspaceRoot), getNxRequirePaths(workspaceRoot),
@ -146,46 +104,30 @@ export async function retrieveProjectConfigurationsWithAngularProjects(
plugins.push({ plugin: NxAngularJsonPlugin }); plugins.push({ plugin: NxAngularJsonPlugin });
} }
const globs = configurationGlobs(plugins); return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins);
return _retrieveProjectConfigurations(workspaceRoot, nxJson, plugins, globs);
} }
export type RetrievedGraphNodes = {
externalNodes: Record<string, ProjectGraphExternalNode>;
projects: Record<string, ProjectConfiguration>;
sourceMaps: ConfigurationSourceMaps;
projectRootMap: Record<string, string>;
};
function _retrieveProjectConfigurations( function _retrieveProjectConfigurations(
workspaceRoot: string, workspaceRoot: string,
nxJson: NxJsonConfiguration, nxJson: NxJsonConfiguration,
plugins: LoadedNxPlugin[], plugins: LoadedNxPlugin[]
globs: string[] ): Promise<RetrievedGraphNodes> {
): Promise<{ const globPatterns = configurationGlobs(plugins);
externalNodes: Record<string, ProjectGraphExternalNode>; const projectFiles = globWithWorkspaceContext(workspaceRoot, globPatterns);
projectNodes: Record<string, ProjectConfiguration>;
sourceMaps: ConfigurationSourceMaps; return createProjectConfigurations(
}> {
let result: {
externalNodes: Record<string, ProjectGraphExternalNode>;
projectNodes: Record<string, ProjectConfiguration>;
sourceMaps: ConfigurationSourceMaps;
};
return getProjectConfigurationsFromContext(
workspaceRoot,
globs,
async (configs: string[]) => {
const { projects, externalNodes, rootMap, sourceMaps } =
await createProjectConfigurations(
workspaceRoot, workspaceRoot,
nxJson, nxJson,
configs, projectFiles,
plugins plugins
); );
result = {
projectNodes: projects,
externalNodes: externalNodes,
sourceMaps,
};
return rootMap;
}
).then(() => result);
} }
export function retrieveProjectConfigurationPaths( export function retrieveProjectConfigurationPaths(
@ -214,24 +156,16 @@ export async function retrieveProjectConfigurationsWithoutPluginInference(
return projectsWithoutPluginCache.get(cacheKey); return projectsWithoutPluginCache.get(cacheKey);
} }
let projects: Record<string, ProjectConfiguration>; const projectFiles = globWithWorkspaceContext(root, projectGlobPatterns);
await getProjectConfigurationsFromContext( const { projects } = await createProjectConfigurations(
root,
projectGlobPatterns,
async (configs: string[]) => {
const projectConfigurations = await createProjectConfigurations(
root, root,
nxJson, nxJson,
configs, projectFiles,
[ [
{ plugin: getNxPackageJsonWorkspacesPlugin(root) }, { plugin: getNxPackageJsonWorkspacesPlugin(root) },
{ plugin: CreateProjectJsonProjectsPlugin }, { plugin: CreateProjectJsonProjectsPlugin },
] ]
); );
projects = projectConfigurations.projects;
return projectConfigurations.rootMap;
}
);
projectsWithoutPluginCache.set(cacheKey, projects); projectsWithoutPluginCache.set(cacheKey, projects);
@ -243,12 +177,7 @@ export async function createProjectConfigurations(
nxJson: NxJsonConfiguration, nxJson: NxJsonConfiguration,
configFiles: string[], configFiles: string[],
plugins: LoadedNxPlugin[] plugins: LoadedNxPlugin[]
): Promise<{ ): Promise<RetrievedGraphNodes> {
projects: Record<string, ProjectConfiguration>;
externalNodes: Record<string, ProjectGraphExternalNode>;
rootMap: Record<string, string>;
sourceMaps: ConfigurationSourceMaps;
}> {
performance.mark('build-project-configs:start'); performance.mark('build-project-configs:start');
const { projects, externalNodes, rootMap, sourceMaps } = const { projects, externalNodes, rootMap, sourceMaps } =
@ -269,7 +198,7 @@ export async function createProjectConfigurations(
return { return {
projects, projects,
externalNodes, externalNodes,
rootMap, projectRootMap: rootMap,
sourceMaps, sourceMaps,
}; };
} }

View File

@ -19,11 +19,10 @@ export function setupWorkspaceContext(workspaceRoot: string) {
export function getNxWorkspaceFilesFromContext( export function getNxWorkspaceFilesFromContext(
workspaceRoot: string, workspaceRoot: string,
globs: string[], projectRootMap: Record<string, string>
parseConfigurations: (files: string[]) => Promise<Record<string, string>>
) { ) {
ensureContextAvailable(workspaceRoot); ensureContextAvailable(workspaceRoot);
return workspaceContext.getWorkspaceFiles(globs, parseConfigurations); return workspaceContext.getWorkspaceFiles(projectRootMap);
} }
export function globWithWorkspaceContext( export function globWithWorkspaceContext(
@ -44,15 +43,6 @@ export function hashWithWorkspaceContext(
return workspaceContext.hashFilesMatchingGlob(globs, exclude); return workspaceContext.hashFilesMatchingGlob(globs, exclude);
} }
export function getProjectConfigurationsFromContext(
workspaceRoot: string,
globs: string[],
parseConfigurations: (files: string[]) => Promise<Record<string, string>>
) {
ensureContextAvailable(workspaceRoot);
return workspaceContext.getProjectConfigurations(globs, parseConfigurations);
}
export function updateFilesInContext( export function updateFilesInContext(
updatedFiles: string[], updatedFiles: string[],
deletedFiles: string[] deletedFiles: string[]