diff --git a/.nx/workflows/agents.yaml b/.nx/workflows/agents.yaml index 0fae68ced8..49f584c512 100644 --- a/.nx/workflows/agents.yaml +++ b/.nx/workflows/agents.yaml @@ -19,6 +19,26 @@ launch-templates: key: 'pnpm-lock.yaml' paths: .pnpm-store base-branch: 'master' + + - name: Install zip and unzip + script: sudo apt-get -yqq install zip unzip + + # TODO: Remove this once the PR to set it on agents by default is merged + - name: Set SHELL environment variable + script: | + # We need $SHELL to be set for the bun installation to correctly link `bunx` + echo "SHELL=/usr/bin/bash" >> $NX_CLOUD_ENV + + - name: Install bun + script: | + curl -fsSL https://bun.sh/install | bash + echo "BUN_INSTALL=$HOME/.bun" >> $NX_CLOUD_ENV + echo "PATH=$HOME/.bun/bin:$PATH" >> $NX_CLOUD_ENV + + - name: Check bun + script: | + bun --version + - name: Install e2e deps script: | sudo apt-get update @@ -50,8 +70,6 @@ launch-templates: - name: Load Cargo Env script: echo "PATH=$HOME/.cargo/bin:$PATH" >> $NX_CLOUD_ENV - - name: Install zip and unzip - script: sudo apt-get -yqq install zip unzip linux-extra-large: resource-class: 'docker_linux_amd64/extra_large' image: 'ubuntu22.04-node20.11-v10' @@ -72,6 +90,26 @@ launch-templates: key: 'pnpm-lock.yaml' paths: .pnpm-store base-branch: 'master' + + - name: Install zip and unzip + script: sudo apt-get -yqq install zip unzip + + # TODO: Remove this once the PR to set it on agents by default is merged + - name: Set SHELL environment variable + script: | + # We need $SHELL to be set for the bun installation to correctly link `bunx` + echo "SHELL=/usr/bin/bash" >> $NX_CLOUD_ENV + + - name: Install bun + script: | + curl -fsSL https://bun.sh/install | bash + echo "BUN_INSTALL=$HOME/.bun" >> $NX_CLOUD_ENV + echo "PATH=$HOME/.bun/bin:$PATH" >> $NX_CLOUD_ENV + + - name: Check bun + script: | + bun --version + - name: Install e2e deps script: | sudo apt-get update @@ -102,6 +140,3 @@ launch-templates: - name: Load Cargo Env script: echo "PATH=$HOME/.cargo/bin:$PATH" >> $NX_CLOUD_ENV - - - name: Install zip and unzip - script: sudo apt-get -yqq install zip unzip diff --git a/docs/shared/recipes/nx-release/updating-version-references.md b/docs/shared/recipes/nx-release/updating-version-references.md index 52422dd706..afdc9c2c9e 100644 --- a/docs/shared/recipes/nx-release/updating-version-references.md +++ b/docs/shared/recipes/nx-release/updating-version-references.md @@ -124,13 +124,13 @@ This package.json is now valid and ready to be published to the registry. ## Scenario 3: I want to update package versions directly in my source files, but use local dependency references via file/workspace -{% callout type="caution" title="This scenario is only supported when your package manager is pnpm" %} -pnpm is the only package manager that provides a publish command that supports dynamically swapping the `file:` and `workspace:*` references with the actual version number at publish time. +{% callout type="caution" title="This scenario is currently only supported when your package manager is pnpm or bun" %} +pnpm and bun are the only package managers that provide a publish command that both supports dynamically swapping the `file:` and `workspace:*` references with the actual version number at publish time, and provides the customization needed for us to wrap it. `yarn npm publish` does support the replacements but is very limited on customization options. {% /callout %} This is a more advanced scenario because it removes the clean separation of concerns between versioning and publishing. The reason for this is that the `file:` and `workspace:` references simply have to be replaced with actual version numbers before they are written to the registry, otherwise they will break when a user tries to install the package. If versioning does not replace them, publishing needs to. -As mentioned at the start of this recipe, Nx Release intentionally does not manipulate your packages in memory during publishing, so this scenario is only supported when your package manager provides publishing functionality which dynamically swaps the local references. **Currently this is only supported by pnpm.** +As mentioned at the start of this recipe, Nx Release intentionally does not manipulate your packages in memory during publishing, so this scenario is only supported when your package manager provides publishing functionality which dynamically swaps the local references. **Currently this is only supported by pnpm and bun.** Let's first look at the default behavior of Nx Release, which is to update the all version references in the source package.json files with the new version number. @@ -190,4 +190,4 @@ Now, that same patch release to the source package.json file will result in the } ``` -Again, this is not in a valid state to be published to the registry, and so the publishing step will need to handle this. **This is only supported by pnpm**, in which case Nx Release invokes `pnpm publish` instead of `npm publish` behind the scenes during publishing, and you will receive a clear error if you attempt to use such a package.json with another package manager. +Again, this is not in a valid state to be published to the registry, and so the publishing step will need to handle this. **This is only supported by pnpm and bun**, in which case Nx Release invokes `pnpm publish` or `bun publish` instead of `npm publish` behind the scenes during publishing, and you will receive a clear error if you attempt to use such a package.json with npm or yarn. diff --git a/e2e/release/src/preserve-local-dependency-protocols.test.ts b/e2e/release/src/preserve-local-dependency-protocols.test.ts new file mode 100644 index 0000000000..0c3b7b6a38 --- /dev/null +++ b/e2e/release/src/preserve-local-dependency-protocols.test.ts @@ -0,0 +1,352 @@ +import { NxJsonConfiguration } from '@nx/devkit'; +import { + cleanupProject, + getPackageManagerCommand, + newProject, + readJson, + runCLI, + runCommandAsync, + tmpProjPath, + uniq, + updateFile, + updateJson, +} from '@nx/e2e/utils'; +import { execSync } from 'node:child_process'; +import { join } from 'node:path'; + +expect.addSnapshotSerializer({ + serialize(str: string) { + return ( + str + // Remove all output unique to specific projects to ensure deterministic snapshots + .replaceAll(/my-pkg-\d+/g, '{project-name}') + .replaceAll( + /integrity:\s*.*/g, + 'integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + ) + .replaceAll(/\b[0-9a-f]{40}\b/g, '{SHASUM}') + .replaceAll(/\d*B index\.js/g, 'XXB index.js') + .replaceAll(/\d*B\s+project\.json/g, 'XXB project.json') + .replaceAll(/\d*B\s+package\.json/g, 'XXXB package.json') + .replaceAll(/size:\s*\d*\s?B/g, 'size: XXXB') + .replaceAll(/\d*\.\d*\s?kB/g, 'XXX.XXX kb') + .replaceAll(/\d*B\s+src\//g, 'XXB src/') + .replaceAll(/\d*B\s+index/g, 'XXB index') + .replaceAll(/total files:\s+\d*/g, 'total files: X') + .replaceAll(/\d*B\s+README.md/g, 'XXB README.md') + .replaceAll(/Test @[\w\d]+/g, 'Test @{COMMIT_AUTHOR}') + .replaceAll(/(\w+) lock file/g, 'PM lock file') + // Normalize the version title date. + .replaceAll(/\(\d{4}-\d{2}-\d{2}\)/g, '(YYYY-MM-DD)') + // We trim each line to reduce the chances of snapshot flakiness + + // Slightly different handling needed for bun (length can be 8) + .replaceAll(/[a-fA-F0-9]{7,8}/g, '{COMMIT_SHA}') + .replaceAll(/bun publish v\d+\.\d+\.\d+/g, 'bun publish vX.X.X') + .replaceAll( + /Integrity:\s*.*/g, + 'Integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + ) + + .split('\n') + .map((r) => r.trim()) + .filter(Boolean) + .join('\n') + ); + }, + test(val: string) { + return val != null && typeof val === 'string'; + }, +}); + +describe('nx release preserve local dependency protocols', () => { + let previousPackageManager: string; + let e2eRegistryUrl: string; + + beforeAll(() => { + previousPackageManager = process.env.SELECTED_PM; + // This is the verdaccio instance that the e2e tests themselves are working from + e2eRegistryUrl = execSync('npm config get registry').toString().trim(); + }); + + afterEach(() => cleanupProject()); + + afterAll(() => { + process.env.SELECTED_PM = previousPackageManager; + }); + + /** + * Initialize each test with a fresh workspace using the specified + * package manager. + */ + const initializeProject = async (packageManager: 'pnpm' | 'bun') => { + process.env.SELECTED_PM = packageManager; + + console.log(`Creating workspace with package manager: ${packageManager}`); + + newProject({ + packages: ['@nx/js'], + packageManager, + }); + + const pkg1 = uniq('my-pkg-1'); + runCLI(`generate @nx/workspace:npm-package ${pkg1}`); + const pkg2 = uniq('my-pkg-2'); + runCLI(`generate @nx/workspace:npm-package ${pkg2}`); + + // Set up a workspace dependency using the workspace protocol + updateJson(join(pkg1, 'package.json'), (packageJson) => { + packageJson.dependencies = { + [`@proj/${pkg2}`]: 'workspace:*', + }; + return packageJson; + }); + + // Add workspaces config + if (packageManager === 'pnpm') { + updateFile('pnpm-workspace.yaml', `packages:\n - ${pkg1}\n - ${pkg2}\n`); + } else { + updateJson('package.json', (packageJson) => { + packageJson.workspaces = [pkg1, pkg2]; + return packageJson; + }); + } + + // workaround for NXC-143 + runCLI('reset'); + + await runCommandAsync(getPackageManagerCommand({ packageManager }).install); + + return { workspacePath: tmpProjPath(), pkg1, pkg2 }; + }; + + it('should replace local dependency protocols with the actual version number when version.generatorOptions.preserveLocalDependencyProtocols is not set to true', async () => { + // The package manager currently does not matter for the versioning behavior, it's imperatively controlled by the user + const { workspacePath } = await initializeProject('pnpm'); + + updateJson('nx.json', (nxJson) => { + nxJson.release = {}; + return nxJson; + }); + + // Show the dependency being updated + expect(runCLI(`release version minor -d --verbose`, { cwd: workspacePath })) + .toMatchInlineSnapshot(` + NX Running release version for project: {project-name} + {project-name} 🔍 Reading data for package "@proj/{project-name}" from {project-name}/package.json + {project-name} 📄 Resolved the current version as 0.0.0 from {project-name}/package.json + {project-name} 📄 Using the provided version specifier "minor". + {project-name} ✍️ New version 0.1.0 written to {project-name}/package.json + NX Running release version for project: {project-name} + {project-name} 🔍 Reading data for package "@proj/{project-name}" from {project-name}/package.json + {project-name} 📄 Resolved the current version as 0.0.0 from {project-name}/package.json + {project-name} 📄 Using the provided version specifier "minor". + {project-name} ✍️ New version 0.1.0 written to {project-name}/package.json + {project-name} ✍️ Applying new version 0.1.0 to 1 package which depends on {project-name} + "name": "@proj/{project-name}", + - "version": "0.0.0", + + "version": "0.1.0", + "scripts": { + "dependencies": { + - "@proj/{project-name}": "workspace:*" + + "@proj/{project-name}": "0.1.0" + } + } + + + "name": "@proj/{project-name}", + - "version": "0.0.0", + + "version": "0.1.0", + "scripts": { + NX Updating PM lock file + Would update pnpm-lock.yaml with the following command, but --dry-run was set: + pnpm install --lockfile-only + NX Staging changed files with git + Would stage files in git with the following command, but --dry-run was set: + git add {project-name}/package.json {project-name}/package.json + `); + }); + + it('should preserve local dependency protocols when version.generatorOptions.preserveLocalDependencyProtocols is set to true', async () => { + // The package manager currently does not matter for the versioning behavior, it's imperatively controlled by the user + const { workspacePath } = await initializeProject('pnpm'); + + updateJson('nx.json', (nxJson) => { + nxJson.release = { + version: { + generatorOptions: { + preserveLocalDependencyProtocols: true, + }, + }, + }; + return nxJson; + }); + + // Show that the dependency has not been updated + expect(runCLI(`release version minor -d --verbose`, { cwd: workspacePath })) + .toMatchInlineSnapshot(` + NX Running release version for project: {project-name} + {project-name} 🔍 Reading data for package "@proj/{project-name}" from {project-name}/package.json + {project-name} 📄 Resolved the current version as 0.0.0 from {project-name}/package.json + {project-name} 📄 Using the provided version specifier "minor". + {project-name} ✍️ New version 0.1.0 written to {project-name}/package.json + NX Running release version for project: {project-name} + {project-name} 🔍 Reading data for package "@proj/{project-name}" from {project-name}/package.json + {project-name} 📄 Resolved the current version as 0.0.0 from {project-name}/package.json + {project-name} 📄 Using the provided version specifier "minor". + {project-name} ✍️ New version 0.1.0 written to {project-name}/package.json + {project-name} ✍️ Applying new version 0.1.0 to 1 package which depends on {project-name} + "name": "@proj/{project-name}", + - "version": "0.0.0", + + "version": "0.1.0", + "scripts": { + } + + + "name": "@proj/{project-name}", + - "version": "0.0.0", + + "version": "0.1.0", + "scripts": { + NX Updating PM lock file + Would update pnpm-lock.yaml with the following command, but --dry-run was set: + pnpm install --lockfile-only + NX Staging changed files with git + Would stage files in git with the following command, but --dry-run was set: + git add {project-name}/package.json {project-name}/package.json + `); + }); + + describe('pnpm publish', () => { + it('should replace local dependency protocols dynamically during publishing', async () => { + const { workspacePath, pkg1 } = await initializeProject('pnpm'); + + // Prove that the local dependency protocol is present in the pkg1 package.json + expect(readJson(join(workspacePath, pkg1, 'package.json'))) + .toMatchInlineSnapshot(` + { + dependencies: { + @proj/{project-name}: workspace:*, + }, + name: @proj/{project-name}, + scripts: { + test: node index.js, + }, + version: 0.0.0, + } + `); + + // Publish the packages + expect( + runCLI(`release publish`, { silenceError: true, cwd: workspacePath }) + ).toMatchInlineSnapshot(` + NX Running target nx-release-publish for 2 projects: + - {project-name} + - {project-name} + > nx run {project-name}:nx-release-publish + 📦 @proj/{project-name}@0.0.0 + === Tarball Contents === + XXXXB index.js + XXXB package.json + XXB project.json + === Tarball Details === + name: @proj/{project-name} + version: 0.0.0 + filename: proj-{project-name}-0.0.0.tgz + package size: XXXB + unpacked size: XXXB + shasum: {SHASUM} + integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + total files: X + Published to ${e2eRegistryUrl} with tag "latest" + > nx run {project-name}:nx-release-publish + 📦 @proj/{project-name}@0.0.0 + === Tarball Contents === + XXXXB index.js + XXXB package.json + XXB project.json + === Tarball Details === + name: @proj/{project-name} + version: 0.0.0 + filename: proj-{project-name}-0.0.0.tgz + package size: XXXB + unpacked size: XXXB + shasum: {SHASUM} + integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + total files: X + Published to ${e2eRegistryUrl} with tag "latest" + NX Successfully ran target nx-release-publish for 2 projects + `); + + // Ensure that the dependency on pkg2 specified on the registry was replaced with the actual version number during publishing + expect( + (await runCommandAsync(`npm view @proj/${pkg1} dependencies`)) + .combinedOutput + ).toMatchInlineSnapshot(`{ '@proj/{project-name}': '0.0.0' }`); + }); + }); + + describe('bun publish', () => { + it('should replace local dependency protocols dynamically during publishing', async () => { + const { workspacePath, pkg1 } = await initializeProject('bun'); + + // Prove that the local dependency protocol is present in the pkg1 package.json + expect(readJson(join(workspacePath, pkg1, 'package.json'))) + .toMatchInlineSnapshot(` + { + dependencies: { + @proj/{project-name}: workspace:*, + }, + name: @proj/{project-name}, + scripts: { + test: node index.js, + }, + version: 0.0.0, + } + `); + + // Publish the packages + expect( + runCLI(`release publish`, { silenceError: true, cwd: workspacePath }) + ).toMatchInlineSnapshot(` + NX Running target nx-release-publish for 2 projects: + - {project-name} + - {project-name} + > nx run {project-name}:nx-release-publish + bun publish vX.X.X ({COMMIT_SHA}) + packed XXXB package.json + packed XXB index.js + packed XXB project.json + Total files: 3 + Shasum: {SHASUM} + Integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + Unpacked size: XXXB + Packed size: XXXB + Tag: latest + Access: default + Registry: ${e2eRegistryUrl} + + @proj/{project-name}@0.0.0 + Published to ${e2eRegistryUrl} with tag "latest" + > nx run {project-name}:nx-release-publish + bun publish vX.X.X ({COMMIT_SHA}) + packed XXXB package.json + packed XXB index.js + packed XXB project.json + Total files: 3 + Shasum: {SHASUM} + Integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + Unpacked size: XXXB + Packed size: XXXB + Tag: latest + Access: default + Registry: ${e2eRegistryUrl} + + @proj/{project-name}@0.0.0 + Published to ${e2eRegistryUrl} with tag "latest" + NX Successfully ran target nx-release-publish for 2 projects + `); + + // Ensure that the dependency on pkg2 specified on the registry was replaced with the actual version number during publishing + expect( + (await runCommandAsync(`npm view @proj/${pkg1} dependencies`)) + .combinedOutput + ).toMatchInlineSnapshot(`{ '@proj/{project-name}': '0.0.0' }`); + }); + }); +}); diff --git a/e2e/utils/command-utils.ts b/e2e/utils/command-utils.ts index 58f2646cd2..dcf65f9650 100644 --- a/e2e/utils/command-utils.ts +++ b/e2e/utils/command-utils.ts @@ -170,7 +170,8 @@ export function getPackageManagerCommand({ runNx: `pnpm exec nx`, runNxSilent: `pnpm exec nx`, runUninstalledPackage: 'pnpm dlx', - install: 'pnpm i', + // We need to install with --no-frozen-lockfile when running e2e tests because pnpm will pick up the fact we are in CI and default to --frozen-lockfile + install: 'pnpm install --no-frozen-lockfile', ciInstall: 'pnpm install --frozen-lockfile', addProd: isPnpmWorkspace ? 'pnpm add -w' : 'pnpm add', addDev: isPnpmWorkspace ? 'pnpm add -Dw' : 'pnpm add -D', @@ -179,7 +180,8 @@ export function getPackageManagerCommand({ exec: pnpmVersion && gte(pnpmVersion, '6.13.0') ? 'pnpm exec' : 'pnpx', }, bun: { - createWorkspace: `bunx create-nx-workspace@${publishedVersion}`, + // See note in runCreateWorkspace in create-project-utils.ts for why we don't set @{version} for `bunx create-nx-workspace` right now + createWorkspace: `bunx create-nx-workspace`, run: (script: string, args: string) => `bun run ${script} -- ${args}`, runNx: `bunx nx`, runNxSilent: `bunx nx`, diff --git a/e2e/utils/create-project-utils.ts b/e2e/utils/create-project-utils.ts index 6c72b4210b..7fe587502a 100644 --- a/e2e/utils/create-project-utils.ts +++ b/e2e/utils/create-project-utils.ts @@ -1,4 +1,5 @@ import { copySync, ensureDirSync, moveSync, removeSync } from 'fs-extra'; +import * as isCI from 'is-ci'; import { createFile, directoryExists, @@ -15,24 +16,22 @@ import { isVerbose, isVerboseE2ERun, } from './get-env-info'; -import * as isCI from 'is-ci'; +import { output, readJsonFile } from '@nx/devkit'; import { angularCliVersion as defaultAngularCliVersion } from '@nx/workspace/src/utils/versions'; import { dump } from '@zkochan/js-yaml'; -import { execSync, ExecSyncOptions } from 'child_process'; - -import { performance, PerformanceMeasure } from 'perf_hooks'; -import { logError, logInfo } from './log-utils'; +import { execSync, ExecSyncOptions } from 'node:child_process'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { performance, PerformanceMeasure } from 'node:perf_hooks'; +import { resetWorkspaceContext } from 'nx/src/utils/workspace-context'; import { getPackageManagerCommand, runCLI, RunCmdOpts, runCommand, } from './command-utils'; -import { output, readJsonFile } from '@nx/devkit'; -import { readFileSync } from 'fs'; -import { join } from 'path'; -import { resetWorkspaceContext } from 'nx/src/utils/workspace-context'; +import { logError, logInfo } from './log-utils'; let projName: string; @@ -90,7 +89,10 @@ export function newProject({ let createNxWorkspaceMeasure: PerformanceMeasure; let packageInstallMeasure: PerformanceMeasure; - if (!directoryExists(tmpBackupProjPath())) { + // Namespace by package manager to avoid conflicts in test suites which include multiple package managers + const backupPath = tmpBackupProjPath(packageManager); + + if (!directoryExists(backupPath)) { const createNxWorkspaceStart = performance.mark( 'create-nx-workspace:start' ); @@ -132,12 +134,12 @@ export function newProject({ stdio: isVerbose() ? 'inherit' : 'pipe', }); - moveSync(`${e2eCwd}/proj`, `${tmpBackupProjPath()}`); + moveSync(`${e2eCwd}/proj`, backupPath); } projName = name; const projectDirectory = tmpProjPath(); - copySync(`${tmpBackupProjPath()}`, `${projectDirectory}`); + copySync(backupPath, projectDirectory); const dependencies = readJsonFile( `${projectDirectory}/package.json` @@ -257,6 +259,9 @@ export function runCreateWorkspace( const pm = getPackageManagerCommand({ packageManager }); + // Needed for bun workarounds, see below + const registry = execSync('npm config get registry').toString().trim(); + let command = `${pm.createWorkspace} ${name} --preset=${preset} --nxCloud=skip --no-interactive`; if (appName) { @@ -330,6 +335,51 @@ export function runCreateWorkspace( command += ` --prefix=${prefix}`; } + if (packageManager === 'bun') { + /** + * `bunx` does not seem to work well at all with custom registries, I tried many combinations of flags and config files. + * + * The only viable workaround currently seems to be to write a package.json and a bunfig.toml in the e2e directory, + * install create-nx-workspace using `bun install` (which _does_ seem to respect the registry settings), and _then_ + * run `bunx create-nx-workspace` (but without the version number added with @{version}). + */ + writeFileSync( + join(cwd, 'bunfig.toml'), + // Also set up a dedicated cache directory to hopefully avoid conflicts with the global cache + ` +[install] +cache = ".bun-cache" +registry = "${registry}" +`.trim() + ); + writeFileSync( + join(cwd, 'package.json'), + ` +{ + "private": true, + "name": "only-here-to-make-bunx-happy" +} + ` + ); + const output = execSync('bun install create-nx-workspace', { + cwd, + stdio: 'pipe', + env: { + CI: 'true', + ...process.env, + }, + encoding: 'utf-8', + }); + const publishedVersion = getPublishedVersion(); + // Ensure that it installed the version published for the e2e tests + if (!output.includes(publishedVersion)) { + console.error(output); + throw new Error( + `bunx create-nx-workspace did not install the version published for the e2e tests: ${publishedVersion}, in ${cwd}` + ); + } + } + try { const create = execSync(`${command}${isVerbose() ? ' --verbose' : ''}`, { cwd, @@ -350,10 +400,32 @@ export function runCreateWorkspace( }); } + if (packageManager === 'bun') { + // We also have to add an explicit bunfig.toml in the workspace itself as bun does not seem to use the setting applied by the local registry logic + // (via `npm set config registry`), unlike all other package managers. + updateFile( + 'bunfig.toml', + ` +[install] +registry = { url = "${registry}", token = "secretVerdaccioToken" } +`.trim() + ); + } + return create; } catch (e) { logError(`Original command: ${command}`, `${e.stdout}\n\n${e.stderr}`); throw e; + } finally { + // Clean up files related to bun workarounds + if (packageManager === 'bun') { + removeSync(join(cwd, 'bunfig.toml')); + removeSync(join(cwd, 'package.json')); + removeSync(join(cwd, '.bun-cache')); + removeSync(join(cwd, 'node_modules')); + removeSync(join(cwd, 'bun.lock')); + removeSync(join(cwd, 'bun.lockb')); + } } } diff --git a/e2e/utils/get-env-info.ts b/e2e/utils/get-env-info.ts index c2d757276d..c60fa41d9f 100644 --- a/e2e/utils/get-env-info.ts +++ b/e2e/utils/get-env-info.ts @@ -1,7 +1,8 @@ import { readJsonFile, workspaceRoot } from '@nx/devkit'; -import { execSync } from 'child_process'; import { existsSync } from 'fs-extra'; +import { execSync } from 'node:child_process'; import { join } from 'path'; +import { gte } from 'semver'; import { dirSync } from 'tmp'; import * as isCI from 'is-ci'; @@ -23,7 +24,7 @@ export function getPublishedVersion(): string { } export function detectPackageManager(dir: string = ''): PackageManager { - return existsSync(join(dir, 'bun.lockb')) + return existsSync(join(dir, 'bun.lockb')) || existsSync(join(dir, 'bun.lock')) ? 'bun' : existsSync(join(dir, 'yarn.lock')) ? 'yarn' @@ -121,7 +122,16 @@ export const packageManagerLockFile = { npm: 'package-lock.json', yarn: 'yarn.lock', pnpm: 'pnpm-lock.yaml', - bun: 'bun.lockb', + bun: (() => { + try { + // In version 1.2.0, bun switched to a text based lockfile format by default + return gte(execSync('bun --version').toString().trim(), '1.2.0') + ? 'bun.lock' + : 'bun.lockb'; + } catch { + return 'bun.lockb'; + } + })(), }; export function ensureCypressInstallation() { diff --git a/packages/create-nx-workspace/src/utils/package-manager.ts b/packages/create-nx-workspace/src/utils/package-manager.ts index e8c23674f8..52a3aa3c48 100644 --- a/packages/create-nx-workspace/src/utils/package-manager.ts +++ b/packages/create-nx-workspace/src/utils/package-manager.ts @@ -1,6 +1,6 @@ -import { execSync } from 'child_process'; -import { existsSync, writeFileSync } from 'fs'; -import { join } from 'path'; +import { execSync } from 'node:child_process'; +import { existsSync, writeFileSync } from 'node:fs'; +import { join, sep } from 'node:path'; /* * Because we don't want to depend on @nx/workspace (to speed up the workspace creation) @@ -12,7 +12,7 @@ export const packageManagerList = ['pnpm', 'yarn', 'npm', 'bun'] as const; export type PackageManager = (typeof packageManagerList)[number]; export function detectPackageManager(dir: string = ''): PackageManager { - return existsSync(join(dir, 'bun.lockb')) + return existsSync(join(dir, 'bun.lockb')) || existsSync(join(dir, 'bun.lock')) ? 'bun' : existsSync(join(dir, 'yarn.lock')) ? 'yarn' @@ -83,7 +83,7 @@ export function getPackageManagerCommand( getRegistryUrl: 'npm config get registry', }; case 'bun': - // bun doesn't current support programatically reading config https://github.com/oven-sh/bun/issues/7140 + // bun doesn't current support programmatically reading config https://github.com/oven-sh/bun/issues/7140 return { install: 'bun install --silent --ignore-scripts', exec: 'bunx', @@ -135,24 +135,25 @@ export function getPackageManagerVersion( * - npx returns 'npm' * - pnpx returns 'pnpm' * - yarn create returns 'yarn' + * - bunx returns 'bun' * * Default to 'npm' */ export function detectInvokedPackageManager(): PackageManager { - let detectedPackageManager: PackageManager = 'npm'; - // mainModule is deprecated since Node 14, fallback for older versions - const invoker = require.main || process['mainModule']; - - // default to `npm` - if (!invoker) { - return detectedPackageManager; - } - for (const pkgManager of packageManagerList) { - if (invoker.path.includes(pkgManager)) { - detectedPackageManager = pkgManager; - break; + if (process.env.npm_config_user_agent) { + for (const pm of packageManagerList) { + if (process.env.npm_config_user_agent.startsWith(`${pm}/`)) { + return pm; + } } } - return detectedPackageManager; + if (process.env.npm_execpath) { + for (const pm of packageManagerList) { + if (process.env.npm_execpath.split(sep).includes(pm)) { + return pm; + } + } + } + return 'npm'; } diff --git a/packages/devkit/src/utils/replace-package.ts b/packages/devkit/src/utils/replace-package.ts index 4e8f656193..9cad8c216a 100644 --- a/packages/devkit/src/utils/replace-package.ts +++ b/packages/devkit/src/utils/replace-package.ts @@ -162,6 +162,7 @@ function replaceMentions( 'package-lock.json', 'pnpm-lock.yaml', 'bun.lockb', + 'bun.lock', 'CHANGELOG.md', ]; if (ignoredFiles.includes(basename(path))) { diff --git a/packages/expo/src/generators/application/lib/create-application-files.ts b/packages/expo/src/generators/application/lib/create-application-files.ts index 39c8cf72d6..16105a036a 100644 --- a/packages/expo/src/generators/application/lib/create-application-files.ts +++ b/packages/expo/src/generators/application/lib/create-application-files.ts @@ -2,29 +2,21 @@ import { detectPackageManager, generateFiles, offsetFromRoot, - PackageManager, toJS, Tree, } from '@nx/devkit'; +import { + createNxCloudOnboardingURLForWelcomeApp, + getNxCloudAppOnBoardingUrl, +} from 'nx/src/nx-cloud/utilities/onboarding'; import { join } from 'path'; import { NormalizedSchema } from './normalize-options'; -import { - getNxCloudAppOnBoardingUrl, - createNxCloudOnboardingURLForWelcomeApp, -} from 'nx/src/nx-cloud/utilities/onboarding'; export async function createApplicationFiles( host: Tree, options: NormalizedSchema ) { - const packageManagerLockFile: Record = { - npm: 'package-lock.json', - yarn: 'yarn.lock', - pnpm: 'pnpm-lock.yaml', - bun: 'bun.lockb', - }; const packageManager = detectPackageManager(host.root); - const packageLockFile = packageManagerLockFile[packageManager]; const onBoardingStatus = await createNxCloudOnboardingURLForWelcomeApp( host, @@ -43,7 +35,6 @@ export async function createApplicationFiles( ...options, offsetFromRoot: offsetFromRoot(options.appProjectRoot), packageManager, - packageLockFile, } ); @@ -56,7 +47,6 @@ export async function createApplicationFiles( connectCloudUrl, offsetFromRoot: offsetFromRoot(options.appProjectRoot), packageManager, - packageLockFile, } ); diff --git a/packages/js/src/executors/release-publish/release-publish.impl.ts b/packages/js/src/executors/release-publish/release-publish.impl.ts index fdd0e004ee..b7fa121547 100644 --- a/packages/js/src/executors/release-publish/release-publish.impl.ts +++ b/packages/js/src/executors/release-publish/release-publish.impl.ts @@ -51,21 +51,40 @@ export default async function runExecutor( const packageName = packageJson.name; /** - * pnpm supports dynamically updating locally linked packages during its packing phase, but other package managers do not. - * Therefore, protect the user from publishing invalid packages by checking if it contains local dependency protocols. + * Whether or not dynamically replacing local dependency protocols (such as "workspace:*") is supported during `nx release publish` is + * dependent on the package manager the user is using. + * + * npm does not support the workspace protocol at all, and `npm publish` does not support dynamically updating locally linked packages + * during its packing phase, so we give the user a clear error message informing them of that. + * + * - `pnpm publish` provides ideal support, it has the possibility of providing JSON output consistent with npm + * - `bun publish`, provides very good support, including all the flags we need apart from the JSON output, so we just have to accept that + * it will look and feel different and print what it gives us and perform one bit of string manipulation for the dry-run case. + * - `yarn npm publish`, IS NOT YET SUPPORTED, and will be tricky because it does not support the majority of the flags we need. However, it + * does support replacing local dependency protocols with the correct version during its packing phase. */ - if (pm !== 'pnpm') { + if (pm === 'npm' || pm === 'yarn') { const depTypes = ['dependencies', 'devDependencies', 'peerDependencies']; for (const depType of depTypes) { const deps = packageJson[depType]; if (deps) { for (const depName in deps) { if (isLocallyLinkedPackageVersion(deps[depName])) { - console.error( - `Error: Cannot publish package "${packageName}" because it contains a local dependency protocol in its "${depType}", and your package manager is ${pm}. + if (pm === 'npm') { + console.error( + `Error: Cannot publish package "${packageName}" because it contains a local dependency protocol in its "${depType}", and your package manager is npm. -Please update the local dependency on "${depName}" to be a valid semantic version (e.g. using \`nx release\`) before publishing, or switch to pnpm as a package manager, which supports dynamically replacing these protocols during publishing.` - ); +Please update the local dependency on "${depName}" to be a valid semantic version (e.g. using \`nx release\`) before publishing, or switch to pnpm or bun as a package manager, which support dynamically replacing these protocols during publishing.` + ); + } else if (pm === 'yarn') { + console.error( + `Error: Cannot publish package "${packageName}" because it contains a local dependency protocol in its "${depType}", and your package manager is yarn. + +Currently, yarn is not supported for this use case because its \`yarn npm publish\` command does not support the customization needed. + +Please update the local dependency on "${depName}" to be a valid semantic version (e.g. using \`nx release\`) before publishing, or switch to pnpm or bun as a package manager, which support dynamically replacing these protocols during publishing.` + ); + } return { success: false, }; @@ -245,7 +264,10 @@ Please update the local dependency on "${depName}" to be a valid semantic versio * JSON output under the name of the package in that case (and it would need to be handled below). */ const publishCommandSegments = [ - pm === 'pnpm' + pm === 'bun' + ? // Unlike npm, bun publish does not support a custom registryConfigKey option + `bun publish --cwd="${packageRoot}" --json --registry="${registry}" --tag=${tag}` + : pm === 'pnpm' ? // Unlike npm, pnpm publish does not support a custom registryConfigKey option, and will error on uncommitted changes by default if --no-git-checks is not set `pnpm publish "${packageRoot}" --json --registry="${registry}" --tag=${tag} --no-git-checks` : `npm publish "${packageRoot}" --json --"${registryConfigKey}=${registry}" --tag=${tag}`, @@ -271,6 +293,30 @@ Please update the local dependency on "${depName}" to be a valid semantic versio stdio: ['ignore', 'pipe', 'pipe'], windowsHide: false, }); + // If in dry-run mode, the version on disk will not represent the version that would be published, so we scrub it from the output to avoid confusion. + const dryRunVersionPlaceholder = 'X.X.X-dry-run'; + + const publishSummaryMessage = isDryRun + ? `Would publish to ${registry} with tag "${tag}", but ${chalk.keyword( + 'orange' + )('[dry-run]')} was set` + : `Published to ${registry} with tag "${tag}"`; + + // bun publish does not support outputting JSON, so we need to modify and print the output string directly + if (pm === 'bun') { + let outputStr = output.toString(); + if (isDryRun) { + outputStr = outputStr.replace( + new RegExp(`${packageJson.name}@${packageJson.version}`, 'g'), + `${packageJson.name}@${dryRunVersionPlaceholder}` + ); + } + console.log(outputStr); + console.log(publishSummaryMessage); + return { + success: true, + }; + } /** * We cannot JSON.parse the output directly because if the user is using lifecycle scripts, npm/pnpm will mix its publish output with the JSON output all on stdout. @@ -287,8 +333,6 @@ Please update the local dependency on "${depName}" to be a valid semantic versio }; } - // If in dry-run mode, the version on disk will not represent the version that would be published, so we scrub it from the output to avoid confusion. - const dryRunVersionPlaceholder = 'X.X.X-dry-run'; if (isDryRun) { for (const [key, val] of Object.entries(jsonData)) { if (typeof val !== 'string') { @@ -314,21 +358,24 @@ Please update the local dependency on "${depName}" to be a valid semantic versio console.log(afterJsonData); } - if (isDryRun) { - console.log( - `Would publish to ${registry} with tag "${tag}", but ${chalk.keyword( - 'orange' - )('[dry-run]')} was set` - ); - } else { - console.log(`Published to ${registry} with tag "${tag}"`); - } + // Print the summary message after the JSON data has been printed + console.log(publishSummaryMessage); return { success: true, }; } catch (err) { try { + // bun publish does not support outputting JSON, so we cannot perform any further processing + if (pm === 'bun') { + console.error(`bun publish error:`); + console.error(err.stderr?.toString() || ''); + console.error(err.stdout?.toString() || ''); + return { + success: false, + }; + } + const stdoutData = JSON.parse(err.stdout?.toString() || '{}'); console.error(`${pm} publish error:`); diff --git a/packages/nx/src/daemon/server/server.ts b/packages/nx/src/daemon/server/server.ts index de0a5c9a32..128dfd6b48 100644 --- a/packages/nx/src/daemon/server/server.ts +++ b/packages/nx/src/daemon/server/server.ts @@ -419,6 +419,7 @@ function lockFileHashChanged(): boolean { join(workspaceRoot, 'yarn.lock'), join(workspaceRoot, 'pnpm-lock.yaml'), join(workspaceRoot, 'bun.lockb'), + join(workspaceRoot, 'bun.lock'), ] .filter((file) => existsSync(file)) .map((file) => hashFile(file)); diff --git a/packages/nx/src/plugins/js/lock-file/lock-file.ts b/packages/nx/src/plugins/js/lock-file/lock-file.ts index 4575cf83b4..4156ce44d1 100644 --- a/packages/nx/src/plugins/js/lock-file/lock-file.ts +++ b/packages/nx/src/plugins/js/lock-file/lock-file.ts @@ -3,8 +3,10 @@ * It encapsulates the package manager specific logic and implementation details. */ -import { readFileSync, existsSync } from 'fs'; -import { join } from 'path'; +import { execSync } from 'node:child_process'; +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { gte } from 'semver'; import { detectPackageManager, @@ -46,17 +48,20 @@ const YARN_LOCK_FILE = 'yarn.lock'; const NPM_LOCK_FILE = 'package-lock.json'; const PNPM_LOCK_FILE = 'pnpm-lock.yaml'; const BUN_LOCK_FILE = 'bun.lockb'; +const BUN_TEXT_LOCK_FILE = 'bun.lock'; export const LOCKFILES = [ YARN_LOCK_FILE, NPM_LOCK_FILE, PNPM_LOCK_FILE, BUN_LOCK_FILE, + BUN_TEXT_LOCK_FILE, ]; const YARN_LOCK_PATH = join(workspaceRoot, YARN_LOCK_FILE); const NPM_LOCK_PATH = join(workspaceRoot, NPM_LOCK_FILE); const PNPM_LOCK_PATH = join(workspaceRoot, PNPM_LOCK_FILE); const BUN_LOCK_PATH = join(workspaceRoot, BUN_LOCK_FILE); +const BUN_TEXT_LOCK_PATH = join(workspaceRoot, BUN_TEXT_LOCK_FILE); /** * Parses lock file and maps dependencies and metadata to {@link LockFileGraph} @@ -143,7 +148,7 @@ export function lockFileExists(packageManager: PackageManager): boolean { return existsSync(NPM_LOCK_PATH); } if (packageManager === 'bun') { - return existsSync(BUN_LOCK_PATH); + return existsSync(BUN_LOCK_PATH) || existsSync(BUN_TEXT_LOCK_PATH); } throw new Error( `Unknown package manager ${packageManager} or lock file missing` @@ -182,7 +187,16 @@ function getLockFilePath(packageManager: PackageManager): string { return NPM_LOCK_PATH; } if (packageManager === 'bun') { - return BUN_LOCK_PATH; + try { + const bunVersion = execSync('bun --version').toString().trim(); + // In version 1.2.0, bun switched to a text based lockfile format by default + if (gte(bunVersion, '1.2.0')) { + return BUN_TEXT_LOCK_FILE; + } + return BUN_LOCK_PATH; + } catch { + return BUN_LOCK_PATH; + } } throw new Error(`Unknown package manager: ${packageManager}`); } diff --git a/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.spec.ts b/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.spec.ts index 69ff822601..6de0781a51 100644 --- a/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.spec.ts +++ b/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.spec.ts @@ -42,6 +42,7 @@ describe('getTouchedProjectsFromLockFile', () => { 'pnpm-lock.yaml', 'pnpm-lock.yml', 'bun.lockb', + 'bun.lock', ].forEach((lockFile) => { describe(`"${lockFile}"`, () => { it(`should not return changes when "${lockFile}" is not touched`, () => { diff --git a/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.ts b/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.ts index 8f4c4e393a..c430cd7928 100644 --- a/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.ts +++ b/packages/nx/src/plugins/js/project-graph/affected/lock-file-changes.ts @@ -26,6 +26,7 @@ export const getTouchedProjectsFromLockFile: TouchedProjectLocator< 'pnpm-lock.yaml', 'pnpm-lock.yml', 'bun.lockb', + 'bun.lock', ]; if (fileChanges.some((f) => lockFiles.includes(f.file))) { diff --git a/packages/nx/src/utils/package-manager.spec.ts b/packages/nx/src/utils/package-manager.spec.ts index 454afbcd7a..7a98017186 100644 --- a/packages/nx/src/utils/package-manager.spec.ts +++ b/packages/nx/src/utils/package-manager.spec.ts @@ -29,8 +29,14 @@ describe('package-manager', () => { packageManager: 'pnpm', }, }); - const packageManager = detectPackageManager(); - expect(packageManager).toEqual('pnpm'); + expect(detectPackageManager()).toEqual('pnpm'); + + jest.spyOn(configModule, 'readNxJson').mockReturnValueOnce({ + cli: { + packageManager: 'yarn', + }, + }); + expect(detectPackageManager()).toEqual('yarn'); }); it('should detect yarn package manager from yarn.lock', () => { @@ -45,13 +51,15 @@ describe('package-manager', () => { return false; case 'bun.lockb': return false; + case 'bun.lock': + return false; default: return jest.requireActual('fs').existsSync(p); } }); const packageManager = detectPackageManager(); expect(packageManager).toEqual('yarn'); - expect(fs.existsSync).toHaveBeenNthCalledWith(2, 'yarn.lock'); + expect(fs.existsSync).toHaveBeenNthCalledWith(3, 'yarn.lock'); }); it('should detect pnpm package manager from pnpm-lock.yaml', () => { @@ -66,13 +74,15 @@ describe('package-manager', () => { return false; case 'bun.lockb': return false; + case 'bun.lock': + return false; default: return jest.requireActual('fs').existsSync(p); } }); const packageManager = detectPackageManager(); expect(packageManager).toEqual('pnpm'); - expect(fs.existsSync).toHaveBeenCalledTimes(3); + expect(fs.existsSync).toHaveBeenCalledTimes(4); }); it('should detect bun package manager from bun.lockb', () => { @@ -87,6 +97,8 @@ describe('package-manager', () => { return false; case 'bun.lockb': return true; + case 'bun.lock': + return false; default: return jest.requireActual('fs').existsSync(p); } @@ -96,6 +108,29 @@ describe('package-manager', () => { expect(fs.existsSync).toHaveBeenCalledTimes(1); }); + it('should detect bun package manager from bun.lock', () => { + jest.spyOn(configModule, 'readNxJson').mockReturnValueOnce({}); + jest.spyOn(fs, 'existsSync').mockImplementation((p) => { + switch (p) { + case 'yarn.lock': + return false; + case 'pnpm-lock.yaml': + return false; + case 'package-lock.json': + return false; + case 'bun.lock': + return true; + case 'bun.lockb': + return false; + default: + return jest.requireActual('fs').existsSync(p); + } + }); + const packageManager = detectPackageManager(); + expect(packageManager).toEqual('bun'); + expect(fs.existsSync).toHaveBeenCalledTimes(2); + }); + it('should use npm package manager as default', () => { jest.spyOn(configModule, 'readNxJson').mockReturnValueOnce({}); jest.spyOn(fs, 'existsSync').mockImplementation((p) => { @@ -108,13 +143,15 @@ describe('package-manager', () => { return false; case 'bun.lockb': return false; + case 'bun.lock': + return false; default: return jest.requireActual('fs').existsSync(p); } }); const packageManager = detectPackageManager(); expect(packageManager).toEqual('npm'); - expect(fs.existsSync).toHaveBeenCalledTimes(3); + expect(fs.existsSync).toHaveBeenCalledTimes(4); }); }); diff --git a/packages/nx/src/utils/package-manager.ts b/packages/nx/src/utils/package-manager.ts index da88ac1edb..4b251a834a 100644 --- a/packages/nx/src/utils/package-manager.ts +++ b/packages/nx/src/utils/package-manager.ts @@ -53,7 +53,7 @@ export function detectPackageManager(dir: string = ''): PackageManager { const nxJson = readNxJson(); return ( nxJson.cli?.packageManager ?? - (existsSync(join(dir, 'bun.lockb')) + (existsSync(join(dir, 'bun.lockb')) || existsSync(join(dir, 'bun.lock')) ? 'bun' : existsSync(join(dir, 'yarn.lock')) ? 'yarn' @@ -185,7 +185,7 @@ export function getPackageManagerCommand( }; }, bun: () => { - // bun doesn't current support programatically reading config https://github.com/oven-sh/bun/issues/7140 + // bun doesn't current support programmatically reading config https://github.com/oven-sh/bun/issues/7140 return { install: 'bun install', ciInstall: 'bun install --no-cache', diff --git a/packages/workspace/src/generators/ci-workflow/ci-workflow.spec.ts b/packages/workspace/src/generators/ci-workflow/ci-workflow.spec.ts index 8c8bbea665..fb2989d4b0 100644 --- a/packages/workspace/src/generators/ci-workflow/ci-workflow.spec.ts +++ b/packages/workspace/src/generators/ci-workflow/ci-workflow.spec.ts @@ -38,7 +38,8 @@ jest.mock('fs', () => { existsSync: (p) => p.endsWith('yarn.lock') || p.endsWith('pnpm-lock.yaml') || - p.endsWith('bun.lockb') + p.endsWith('bun.lockb') || + p.endsWith('bun.lock') ? memFs.existsSync(p) : actualFs.existsSync(p), }; @@ -87,7 +88,7 @@ describe('CI Workflow generator', () => { beforeEach(() => { let fileSys; if (packageManager === 'bun') { - fileSys = { 'bun.lockb': '' }; + fileSys = { 'bun.lock': '' }; } else if (packageManager === 'yarn') { fileSys = { 'yarn.lock': '' }; } else if (packageManager === 'pnpm') { diff --git a/scripts/check-lock-files.js b/scripts/check-lock-files.js index e4584d82d5..1802daaaed 100644 --- a/scripts/check-lock-files.js +++ b/scripts/check-lock-files.js @@ -12,6 +12,11 @@ function checkLockFiles() { 'Invalid occurence of "bun.lockb" file. Please remove it and use only "pnpm-lock.yaml"' ); } + if (fs.existsSync('bun.lock')) { + errors.push( + 'Invalid occurence of "bun.lockb" file. Please remove it and use only "pnpm-lock.yaml"' + ); + } if (fs.existsSync('yarn.lock')) { errors.push( 'Invalid occurence of "yarn.lock" file. Please remove it and use only "pnpm-lock.yaml"'