From 6dcf02c69617438382251f6b5f1c740ab2402262 Mon Sep 17 00:00:00 2001 From: Alex Hunt Date: Mon, 12 Feb 2024 08:04:11 -0800 Subject: [PATCH 1/2] [skip ci] Use npm as source of truth for updated packages (make publish script rerunnable) (#42944) Summary: Updates `find-and-publish-all-bumped-packages` to use the npm registry as the source of truth, similar to tools like Lerna (`lerna publish from-package`). **This enables safe reruns of the publish script**, and replaces the previous Git-diff-detection implementation. Changelog: [Internal] Reviewed By: lunaleaps Differential Revision: D53607807 --- ...nd-and-publish-all-bumped-packages-test.js | 49 +++++++---- .../find-and-publish-all-bumped-packages.js | 88 ++++++++----------- 2 files changed, 68 insertions(+), 69 deletions(-) diff --git a/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js b/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js index 488e00c50eeaec..e30468a1c0d713 100644 --- a/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js +++ b/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js @@ -16,14 +16,16 @@ const { const getPackagesMock = jest.fn(); const execSync = jest.fn(); -const spawnSync = jest.fn(); const execMock = jest.fn(); +const fetchMock = jest.fn(); -jest.mock('child_process', () => ({execSync, spawnSync})); +jest.mock('child_process', () => ({execSync})); jest.mock('shelljs', () => ({exec: execMock})); jest.mock('../../releases/utils/monorepo', () => ({ getPackages: getPackagesMock, })); +// $FlowIgnore[cannot-write] +global.fetch = fetchMock; const BUMP_COMMIT_MESSAGE = 'bumped packages versions\n\n#publish-packages-to-npm'; @@ -76,7 +78,7 @@ describe('findAndPublishAllBumpedPackages', () => { `); }); - test('should throw an error if updated version is not 0.x.y', async () => { + test('should throw an error if updated version is not 0.x.x', async () => { execSync.mockImplementation((command: string) => { switch (command) { case 'git log -1 --pretty=%B': @@ -93,16 +95,16 @@ describe('findAndPublishAllBumpedPackages', () => { }, }); - spawnSync.mockImplementationOnce(() => ({ - stdout: `- "version": "0.72.0"\n+ "version": "${mockedPackageNewVersion}"\n`, - })); + fetchMock.mockResolvedValueOnce({ + json: () => Promise.resolve({versions: {}}), + }); await expect(findAndPublishAllBumpedPackages()).rejects.toThrow( - `Package version expected to be 0.x.y, but received ${mockedPackageNewVersion}`, + `Package version expected to be 0.x.x, but received ${mockedPackageNewVersion}`, ); }); - test('should publish all changed packages', async () => { + test('should publish all updated packages', async () => { execSync.mockImplementation((command: string) => { switch (command) { case 'git log -1 --pretty=%B': @@ -111,39 +113,48 @@ describe('findAndPublishAllBumpedPackages', () => { }); getPackagesMock.mockResolvedValue({ '@react-native/package-a': { + name: '@react-native/package-a', path: 'absolute/path/to/package-a', packageJson: { version: '0.72.1', }, }, '@react-native/package-b': { + name: '@react-native/package-b', path: 'absolute/path/to/package-b', packageJson: { version: '0.72.1', }, }, '@react-native/package-c': { + name: '@react-native/package-c', path: 'absolute/path/to/package-c', packageJson: { version: '0.72.0', }, }, }); - - spawnSync.mockImplementationOnce(() => ({ - stdout: `- "version": "0.72.0"\n+ "version": "0.72.1"\n`, - })); - spawnSync.mockImplementationOnce(() => ({ - stdout: `- "version": "0.72.0"\n+ "version": "0.72.1"\n`, - })); - spawnSync.mockImplementationOnce(() => ({ - stdout: '\n', - })); - + fetchMock.mockResolvedValue({ + json: () => + Promise.resolve({ + versions: {'0.72.0': {}}, + }), + }); execMock.mockImplementation(() => ({code: 0})); + const consoleLog = jest.spyOn(console, 'log').mockImplementation(() => {}); + await findAndPublishAllBumpedPackages(); + expect(consoleLog.mock.calls.flat().join('\n')).toMatchInlineSnapshot(` + "Discovering updated packages + - Skipping @react-native/package-c (0.72.0 already present on npm) + Done ✅ + Publishing updated packages to npm + - Publishing @react-native/package-a (0.72.1) + - Publishing @react-native/package-b (0.72.1) + Done ✅" + `); expect(execMock.mock.calls).toMatchInlineSnapshot(` Array [ Array [ diff --git a/scripts/monorepo/find-and-publish-all-bumped-packages.js b/scripts/monorepo/find-and-publish-all-bumped-packages.js index 8353e6683645f3..b96647b03a56ac 100644 --- a/scripts/monorepo/find-and-publish-all-bumped-packages.js +++ b/scripts/monorepo/find-and-publish-all-bumped-packages.js @@ -12,10 +12,8 @@ const {publishPackage} = require('../npm-utils'); const {getPackages} = require('../releases/utils/monorepo'); const {PUBLISH_PACKAGES_TAG} = require('./constants'); -const {execSync, spawnSync} = require('child_process'); -const path = require('path'); +const {execSync} = require('child_process'); -const ROOT_LOCATION = path.join(__dirname, '..', '..'); const NPM_CONFIG_OTP = process.env.NPM_CONFIG_OTP; async function findAndPublishAllBumpedPackages() { @@ -36,76 +34,66 @@ async function findAndPublishAllBumpedPackages() { return; } - const tags = getTagsFromCommitMessage(commitMessage); - - console.log('Traversing all packages inside /packages...'); + console.log('Discovering updated packages'); const packages = await getPackages({ includeReactNative: false, }); + const packagesToUpdate = []; - for (const package of Object.values(packages)) { - const {stdout: diff, stderr: commitDiffStderr} = spawnSync( - 'git', - [ - 'log', - '-p', - '--format=""', - 'HEAD~1..HEAD', - `${package.path}/package.json`, - ], - {cwd: ROOT_LOCATION, shell: true, stdio: 'pipe', encoding: 'utf-8'}, - ); + await Promise.all( + Object.values(packages).map(async package => { + const version = package.packageJson.version; - if (commitDiffStderr) { - console.log( - `\u274c Failed to get latest committed changes for ${package.name}:`, - ); - console.log(commitDiffStderr); + if (!version.startsWith('0.')) { + throw new Error( + `Package version expected to be 0.x.x, but received ${version}`, + ); + } - process.exit(1); - } + const response = await fetch( + 'https://registry.npmjs.org/' + package.name, + ); + const {versions: versionsInRegistry} = await response.json(); - const previousVersionPatternMatches = diff - .toString() - .match(/- {2}"version": "([0-9]+.[0-9]+.[0-9]+)"/); + if (version in versionsInRegistry) { + console.log( + `- Skipping ${package.name} (${version} already present on npm)`, + ); + return; + } - if (!previousVersionPatternMatches) { - console.log(`\uD83D\uDD0E No version bump for ${package.name}`); + packagesToUpdate.push(package.name); + }), + ); - return; - } + console.log('Done ✅'); + console.log('Publishing updated packages to npm'); - const [, previousVersion] = previousVersionPatternMatches; - const nextVersion = package.packageJson.version; + const tags = getTagsFromCommitMessage(commitMessage); + for (const packageName of packagesToUpdate) { + const package = packages[packageName]; console.log( - `\uD83D\uDCA1 ${package.name} was updated: ${previousVersion} -> ${nextVersion}`, + `- Publishing ${package.name} (${package.packageJson.version})`, ); - if (!nextVersion.startsWith('0.')) { - throw new Error( - `Package version expected to be 0.x.y, but received ${nextVersion}`, - ); - } - const result = publishPackage(package.path, { tags, otp: NPM_CONFIG_OTP, }); - if (result.code !== 0) { - console.log( - `\u274c Failed to publish version ${nextVersion} of ${package.name}. npm publish exited with code ${result.code}:`, - ); - console.log(result.stderr); - process.exit(1); - } else { - console.log( - `\u2705 Successfully published new version of ${package.name}`, + if (result.code !== 0) { + console.error( + `Failed to publish ${package.name}. npm publish exited with code ${result.code}:`, ); + console.error(result.stderr); + process.exitCode = 1; + return; } } + + console.log('Done ✅'); } function getTagsFromCommitMessage(msg /*: string */) /*: Array */ { From 53c01a4ec93fc74e179e7ee17379d5a1b02f428d Mon Sep 17 00:00:00 2001 From: Alex Hunt Date: Mon, 12 Feb 2024 08:04:11 -0800 Subject: [PATCH 2/2] Add retry to monorepo publish script (#42964) Summary: We've seen npm publishes fail occasionally in CI as part of this script, most recently in S391653. This change adds a single retry, per package, during the execution of this script, in an attempt to reduce the chance of manual interventions after a broken pipeline. Changelog: [Internal] Reviewed By: cipolleschi Differential Revision: D53607808 --- ...nd-and-publish-all-bumped-packages-test.js | 92 +++++++++++++++++++ .../find-and-publish-all-bumped-packages.js | 46 +++++++--- 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js b/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js index e30468a1c0d713..ab961bde30d736 100644 --- a/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js +++ b/scripts/monorepo/__tests__/find-and-publish-all-bumped-packages-test.js @@ -172,6 +172,98 @@ describe('findAndPublishAllBumpedPackages', () => { ] `); }); + + describe('retry behaviour', () => { + beforeEach(() => { + execSync.mockImplementation((command: string) => { + switch (command) { + case 'git log -1 --pretty=%B': + return BUMP_COMMIT_MESSAGE; + } + }); + getPackagesMock.mockResolvedValue({ + '@react-native/package-a': { + name: '@react-native/package-a', + path: 'absolute/path/to/package-a', + packageJson: { + version: '0.72.1', + }, + }, + '@react-native/package-b': { + name: '@react-native/package-b', + path: 'absolute/path/to/package-b', + packageJson: { + version: '0.72.1', + }, + }, + }); + fetchMock.mockResolvedValue({ + json: () => + Promise.resolve({ + versions: {'0.72.0': {}}, + }), + }); + }); + + test('should retry once if `npm publish` fails', async () => { + execMock.mockImplementationOnce(() => ({code: 0})); + execMock.mockImplementationOnce(() => ({ + code: 1, + stderr: '503 Service Unavailable', + })); + execMock.mockImplementationOnce(() => ({code: 0})); + + const consoleError = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + await findAndPublishAllBumpedPackages(); + + expect(consoleError.mock.calls.flat().join('\n')).toMatchInlineSnapshot(` + "Failed to publish @react-native/package-b. npm publish exited with code 1: + 503 Service Unavailable" + `); + expect(execMock.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "npm publish", + Object { + "cwd": "absolute/path/to/package-a", + }, + ], + Array [ + "npm publish", + Object { + "cwd": "absolute/path/to/package-b", + }, + ], + Array [ + "npm publish", + Object { + "cwd": "absolute/path/to/package-b", + }, + ], + ] + `); + }); + + test('should exit with error if one or more packages fail after retry', async () => { + execMock.mockImplementationOnce(() => ({code: 0})); + execMock.mockImplementation(() => ({ + code: 1, + stderr: '503 Service Unavailable', + })); + + const consoleLog = jest + .spyOn(console, 'log') + .mockImplementation(() => {}); + + await findAndPublishAllBumpedPackages(); + + expect(consoleLog).toHaveBeenLastCalledWith('--- Retrying once! ---'); + expect(process.exitCode).toBe(1); + }); + }); }); describe('getTagsFromCommitMessage', () => { diff --git a/scripts/monorepo/find-and-publish-all-bumped-packages.js b/scripts/monorepo/find-and-publish-all-bumped-packages.js index b96647b03a56ac..55230030c5f108 100644 --- a/scripts/monorepo/find-and-publish-all-bumped-packages.js +++ b/scripts/monorepo/find-and-publish-all-bumped-packages.js @@ -71,6 +71,7 @@ async function findAndPublishAllBumpedPackages() { console.log('Publishing updated packages to npm'); const tags = getTagsFromCommitMessage(commitMessage); + const failedPackages = []; for (const packageName of packagesToUpdate) { const package = packages[packageName]; @@ -78,21 +79,23 @@ async function findAndPublishAllBumpedPackages() { `- Publishing ${package.name} (${package.packageJson.version})`, ); - const result = publishPackage(package.path, { - tags, - otp: NPM_CONFIG_OTP, - }); - - if (result.code !== 0) { - console.error( - `Failed to publish ${package.name}. npm publish exited with code ${result.code}:`, - ); - console.error(result.stderr); - process.exitCode = 1; - return; + try { + runPublish(package.name, package.path, tags); + } catch { + console.log('--- Retrying once! ---'); + try { + runPublish(package.name, package.path, tags); + } catch (e) { + failedPackages.push(package.name); + } } } + if (failedPackages.length) { + process.exitCode = 1; + return; + } + console.log('Done ✅'); } @@ -106,6 +109,25 @@ function getTagsFromCommitMessage(msg /*: string */) /*: Array */ { .slice(1); } +function runPublish( + packageName /*: string */, + packagePath /*: string */, + tags /*: Array */, +) { + const result = publishPackage(packagePath, { + tags, + otp: NPM_CONFIG_OTP, + }); + + if (result.code !== 0) { + console.error( + `Failed to publish ${packageName}. npm publish exited with code ${result.code}:`, + ); + console.error(result.stderr); + throw new Error(result.stderr); + } +} + if (require.main === module) { // eslint-disable-next-line no-void void findAndPublishAllBumpedPackages();