Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@microsoft/rush",
"comment": "rush-resolver-cache-plugin: add pnpm 10 / lockfile v9 compatibility",
"type": "none"
}
],
"packageName": "@microsoft/rush"
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
computeResolverCacheFromLockfileAsync,
type IPlatformInfo
} from './computeResolverCacheFromLockfileAsync';
import { type PnpmMajorVersion, type IPnpmVersionHelpers, getPnpmVersionHelpersAsync } from './pnpm';
import type { IResolverContext } from './types';

/**
Expand Down Expand Up @@ -79,10 +80,19 @@ export async function afterInstallAsync(

const lockFilePath: string = subspace.getCommittedShrinkwrapFilePath(variant);

const pnpmStoreDir: string = `${rushConfiguration.pnpmOptions.pnpmStorePath}/v3/files/`;
const pnpmStorePath: string = rushConfiguration.pnpmOptions.pnpmStorePath;

const pnpmMajorVersion: PnpmMajorVersion = (() => {
const major: number = parseInt(rushConfiguration.packageManagerToolVersion, 10);
if (major >= 10) return 10;
if (major >= 9) return 9;
return 8;
})() as PnpmMajorVersion;

const pnpmHelpers: IPnpmVersionHelpers = await getPnpmVersionHelpersAsync(pnpmMajorVersion);

terminal.writeLine(`Using pnpm-lock from: ${lockFilePath}`);
terminal.writeLine(`Using pnpm store folder: ${pnpmStoreDir}`);
terminal.writeLine(`Using pnpm ${pnpmMajorVersion} store at: ${pnpmStorePath}`);

const workspaceRoot: string = subspace.getSubspaceTempFolderPath();
const cacheFilePath: string = `${workspaceRoot}/resolver-cache.json`;
Expand Down Expand Up @@ -166,10 +176,7 @@ export async function afterInstallAsync(
const prefixIndex: number = descriptionFileHash.indexOf('-');
const hash: string = Buffer.from(descriptionFileHash.slice(prefixIndex + 1), 'base64').toString('hex');

// The pnpm store directory has index files of package contents at paths:
// <store>/v3/files/<hash (0-2)>/<hash (2-)>-index.json
// See https://github.com/pnpm/pnpm/blob/f394cfccda7bc519ceee8c33fc9b68a0f4235532/store/cafs/src/getFilePathInCafs.ts#L33
const indexPath: string = `${pnpmStoreDir}${hash.slice(0, 2)}/${hash.slice(2)}-index.json`;
const indexPath: string = pnpmHelpers.getStoreIndexPath(pnpmStorePath, context, hash);

try {
const indexContent: string = await FileSystem.readFileAsync(indexPath);
Expand Down Expand Up @@ -254,6 +261,7 @@ export async function afterInstallAsync(
platformInfo: getPlatformInfo(),
projectByImporterPath,
lockfile: lockFile,
pnpmVersion: pnpmMajorVersion,
afterExternalPackagesAsync
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,13 @@ import type {
} from '@rushstack/webpack-workspace-resolve-plugin';

import type { PnpmShrinkwrapFile } from './externals';
import { getDescriptionFileRootFromKey, resolveDependencies, createContextSerializer } from './helpers';
import {
getDescriptionFileRootFromKey,
resolveDependencies,
createContextSerializer,
extractNameAndVersionFromKey
} from './helpers';
import { type PnpmMajorVersion, type IPnpmVersionHelpers, getPnpmVersionHelpersAsync } from './pnpm';
import type { IResolverContext } from './types';

/**
Expand Down Expand Up @@ -105,6 +111,9 @@ function extractBundledDependencies(
}
}

// Re-export for downstream consumers
export type { PnpmMajorVersion, IPnpmVersionHelpers } from './pnpm';

/**
* Options for computing the resolver cache from a lockfile.
*/
Expand All @@ -129,6 +138,13 @@ export interface IComputeResolverCacheFromLockfileOptions {
* The lockfile to compute the cache from
*/
lockfile: PnpmShrinkwrapFile;
/**
* The major version of pnpm configured in rush.json (e.g. `"10.27.0"` → 10).
* Used to select the correct dep-path hashing algorithm and store layout.
* When omitted, the version is inferred from the lockfile format (v6 → pnpm 8,
* v9 → pnpm 9).
*/
pnpmVersion?: PnpmMajorVersion;
/**
* A callback to process external packages after they have been enumerated.
* Broken out as a separate function to facilitate testing without hitting the disk.
Expand All @@ -152,6 +168,44 @@ function convertToSlashes(path: string): string {
return path.replace(/\\/g, '/');
}

/**
* Detects the pnpm major version from the lockfile format and an optional
* caller-supplied version (derived from rush.json `pnpmVersion`).
*
* @param lockfile - The parsed shrinkwrap / lockfile
* @param configuredPnpmVersion - The pnpm major version from rush.json, if available.
* When provided this takes precedence, because the lockfile alone cannot distinguish
* pnpm 9 from pnpm 10 (both use lockfile v9).
*/
export function detectPnpmMajorVersion(
lockfile: PnpmShrinkwrapFile,
configuredPnpmVersion?: PnpmMajorVersion
): PnpmMajorVersion {
if (configuredPnpmVersion !== undefined) {
return configuredPnpmVersion;
}

// Detect from lockfile version
if (lockfile.shrinkwrapFileMajorVersion >= 9) {
// Lockfile v9 is shared by pnpm 9 and pnpm 10.
// Without the configured version we cannot tell them apart; default to 9
// (v8 dep-path algorithm, v3 store, v9 key format).
return 9;
}

if (lockfile.shrinkwrapFileMajorVersion > 0) {
return 8;
}

// Fallback for lockfiles where version parsing failed: inspect the first non-file package key.
for (const key of lockfile.packages.keys()) {
if (!key.startsWith('file:')) {
return key.startsWith('/') ? 8 : 9;
}
}
return 8;
}

/**
* Given a lockfile and information about the workspace and platform, computes the resolver cache file.
* @param params - The options for computing the resolver cache
Expand All @@ -169,10 +223,19 @@ export async function computeResolverCacheFromLockfileAsync(
const contexts: Map<string, IResolverContext> = new Map();
const missingOptionalDependencies: Set<string> = new Set();

const pnpmVersion: PnpmMajorVersion = detectPnpmMajorVersion(lockfile, params.pnpmVersion);

const helpers: IPnpmVersionHelpers = await getPnpmVersionHelpersAsync(pnpmVersion);

// Enumerate external dependencies first, to simplify looping over them for store data
for (const [key, pack] of lockfile.packages) {
let name: string | undefined = pack.name;
const descriptionFileRoot: string = getDescriptionFileRootFromKey(workspaceRoot, key, name);
const descriptionFileRoot: string = getDescriptionFileRootFromKey(
workspaceRoot,
key,
helpers.depPathToFilename,
name
);

// Skip optional dependencies that are incompatible with the current environment
if (pack.optional && !isPackageCompatible(pack, platformInfo)) {
Expand All @@ -182,9 +245,12 @@ export async function computeResolverCacheFromLockfileAsync(

const integrity: string | undefined = pack.resolution?.integrity;

if (!name && key.startsWith('/')) {
const versionIndex: number = key.indexOf('@', 2);
name = key.slice(1, versionIndex);
// Extract name and version from the key if not already provided
const parsed: { name: string; version: string } | undefined = extractNameAndVersionFromKey(key);
if (parsed) {
if (!name) {
name = parsed.name;
}
}

if (!name) {
Expand All @@ -196,6 +262,7 @@ export async function computeResolverCacheFromLockfileAsync(
descriptionFileHash: integrity,
isProject: false,
name,
version: parsed?.version,
deps: new Map(),
ordinal: -1,
optional: pack.optional
Expand All @@ -204,10 +271,10 @@ export async function computeResolverCacheFromLockfileAsync(
contexts.set(descriptionFileRoot, context);

if (pack.dependencies) {
resolveDependencies(workspaceRoot, pack.dependencies, context);
resolveDependencies(workspaceRoot, pack.dependencies, context, helpers, lockfile.packages);
}
if (pack.optionalDependencies) {
resolveDependencies(workspaceRoot, pack.optionalDependencies, context);
resolveDependencies(workspaceRoot, pack.optionalDependencies, context, helpers, lockfile.packages);
}
}

Expand Down Expand Up @@ -248,13 +315,13 @@ export async function computeResolverCacheFromLockfileAsync(
contexts.set(descriptionFileRoot, context);

if (importer.dependencies) {
resolveDependencies(workspaceRoot, importer.dependencies, context);
resolveDependencies(workspaceRoot, importer.dependencies, context, helpers, lockfile.packages);
}
if (importer.devDependencies) {
resolveDependencies(workspaceRoot, importer.devDependencies, context);
resolveDependencies(workspaceRoot, importer.devDependencies, context, helpers, lockfile.packages);
}
if (importer.optionalDependencies) {
resolveDependencies(workspaceRoot, importer.optionalDependencies, context);
resolveDependencies(workspaceRoot, importer.optionalDependencies, context, helpers, lockfile.packages);
}
}

Expand Down
117 changes: 49 additions & 68 deletions rush-plugins/rush-resolver-cache-plugin/src/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,98 +1,64 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import { createHash } from 'node:crypto';
import * as path from 'node:path';

import type { ISerializedResolveContext } from '@rushstack/webpack-workspace-resolve-plugin';

import type { IDependencyEntry, IResolverContext } from './types';

const MAX_LENGTH_WITHOUT_HASH: number = 120 - 26 - 1;
const BASE32: string[] = 'abcdefghijklmnopqrstuvwxyz234567'.split('');

// https://github.com/swansontec/rfc4648.js/blob/ead9c9b4b68e5d4a529f32925da02c02984e772c/src/codec.ts#L82-L118
export function createBase32Hash(input: string): string {
const data: Buffer = createHash('md5').update(input).digest();

const mask: 0x1f = 0x1f;
let out: string = '';

let bits: number = 0; // Number of bits currently in the buffer
let buffer: number = 0; // Bits waiting to be written out, MSB first
for (let i: number = 0; i < data.length; ++i) {
// eslint-disable-next-line no-bitwise
buffer = (buffer << 8) | (0xff & data[i]);
bits += 8;

// Write out as much as we can:
while (bits > 5) {
bits -= 5;
// eslint-disable-next-line no-bitwise
out += BASE32[mask & (buffer >> bits)];
}
}

// Partial character:
if (bits) {
// eslint-disable-next-line no-bitwise
out += BASE32[mask & (buffer << (5 - bits))];
}

return out;
}

// https://github.com/pnpm/pnpm/blob/f394cfccda7bc519ceee8c33fc9b68a0f4235532/packages/dependency-path/src/index.ts#L167-L189
export function depPathToFilename(depPath: string): string {
let filename: string = depPathToFilenameUnescaped(depPath).replace(/[\\/:*?"<>|]/g, '+');
if (filename.includes('(')) {
filename = filename.replace(/(\)\()|\(/g, '_').replace(/\)$/, '');
}
if (filename.length > 120 || (filename !== filename.toLowerCase() && !filename.startsWith('file+'))) {
return `${filename.substring(0, MAX_LENGTH_WITHOUT_HASH)}_${createBase32Hash(filename)}`;
}
return filename;
}
import type { IPnpmVersionHelpers } from './pnpm';

/**
* Computes the root folder for a dependency from a reference to it in another package
* @param lockfileFolder - The folder that contains the lockfile
* @param key - The key of the dependency
* @param specifier - The specifier in the lockfile for the dependency
* @param context - The owning package
* @param helpers - Version-specific pnpm helpers
* @returns The identifier for the dependency
*/
export function resolveDependencyKey(
lockfileFolder: string,
key: string,
specifier: string,
context: IResolverContext
context: IResolverContext,
helpers: IPnpmVersionHelpers,
packageKeys?: { has(key: string): boolean }
): string {
if (specifier.startsWith('/')) {
return getDescriptionFileRootFromKey(lockfileFolder, specifier);
} else if (specifier.startsWith('link:')) {
if (specifier.startsWith('link:')) {
if (context.isProject) {
return path.posix.join(context.descriptionFileRoot, specifier.slice(5));
} else {
return path.posix.join(lockfileFolder, specifier.slice(5));
}
} else if (specifier.startsWith('file:')) {
return getDescriptionFileRootFromKey(lockfileFolder, specifier, key);
return getDescriptionFileRootFromKey(lockfileFolder, specifier, helpers.depPathToFilename, key);
} else if (packageKeys?.has(specifier)) {
// The specifier is a full package key (v6: '/pkg@ver', v9: 'pkg@ver')
return getDescriptionFileRootFromKey(lockfileFolder, specifier, helpers.depPathToFilename);
} else {
return getDescriptionFileRootFromKey(lockfileFolder, `/${key}@${specifier}`);
const fullKey: string = helpers.buildDependencyKey(key, specifier);
return getDescriptionFileRootFromKey(lockfileFolder, fullKey, helpers.depPathToFilename);
}
}

/**
* Computes the physical path to a dependency based on its entry
* @param lockfileFolder - The folder that contains the lockfile during installation
* @param key - The key of the dependency
* @param depPathToFilename - Version-specific function to convert dep paths to filenames
* @param name - The name of the dependency, if provided
* @returns The physical path to the dependency
*/
export function getDescriptionFileRootFromKey(lockfileFolder: string, key: string, name?: string): string {
if (!key.startsWith('file:')) {
name = key.slice(1, key.indexOf('@', 2));
export function getDescriptionFileRootFromKey(
lockfileFolder: string,
key: string,
depPathToFilename: (depPath: string) => string,
name?: string
): string {
if (!key.startsWith('file:') && !name) {
const offset: number = key.startsWith('/') ? 1 : 0;
name = key.slice(offset, key.indexOf('@', offset + 1));
}
if (!name) {
throw new Error(`Missing package name for ${key}`);
Expand All @@ -106,29 +72,44 @@ export function getDescriptionFileRootFromKey(lockfileFolder: string, key: strin
export function resolveDependencies(
lockfileFolder: string,
collection: Record<string, IDependencyEntry>,
context: IResolverContext
context: IResolverContext,
helpers: IPnpmVersionHelpers,
packageKeys?: { has(key: string): boolean }
): void {
for (const [key, value] of Object.entries(collection)) {
const version: string = typeof value === 'string' ? value : value.version;
const resolved: string = resolveDependencyKey(lockfileFolder, key, version, context);
const resolved: string = resolveDependencyKey(
lockfileFolder,
key,
version,
context,
helpers,
packageKeys
);

context.deps.set(key, resolved);
}
}

/**
*
* @param depPath - The path to the dependency
* @returns The folder name for the dependency
* Extracts the package name and version from a lockfile package key.
* @param key - The lockfile package key (e.g. '/autoprefixer\@9.8.8', '\@scope/name\@1.0.0(peer\@2.0.0)')
* @returns The extracted name and version, or undefined for file: keys
*/
export function depPathToFilenameUnescaped(depPath: string): string {
if (depPath.indexOf('file:') !== 0) {
if (depPath.startsWith('/')) {
depPath = depPath.slice(1);
}
return depPath;
export function extractNameAndVersionFromKey(key: string): { name: string; version: string } | undefined {
if (key.startsWith('file:')) {
return undefined;
}
const offset: number = key.startsWith('/') ? 1 : 0;
const versionAtIndex: number = key.indexOf('@', offset + 1);
if (versionAtIndex === -1) {
return undefined;
}
return depPath.replace(':', '+');
const name: string = key.slice(offset, versionAtIndex);
const parenIndex: number = key.indexOf('(', versionAtIndex);
const version: string =
parenIndex !== -1 ? key.slice(versionAtIndex + 1, parenIndex) : key.slice(versionAtIndex + 1);
return { name, version };
}

/**
Expand Down
Loading
Loading