Skip to content

Ability to store a built package in offline mirror #5314

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 22, 2018
Merged

Ability to store a built package in offline mirror #5314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7e5d6fb
WIP: ability to store a built package in offline mirror
bestander Feb 3, 2018
5d4428e
added tests
bestander Feb 7, 2018
61dd958
more tests
bestander Feb 8, 2018
8436160
flow fixes and tests
bestander Feb 8, 2018
0af2a4d
lockfile updates
bestander Feb 8, 2018
1c92e61
fixed more tests
bestander Feb 9, 2018
c6fc640
feedback
bestander Feb 19, 2018
b8f8a81
fixed test
bestander Feb 19, 2018
11234fd
feedback
bestander Feb 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 0 additions & 166 deletions __tests__/commands/install/integration.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {run as cache} from '../../../src/cli/commands/cache.js';
import {run as check} from '../../../src/cli/commands/check.js';
import * as constants from '../../../src/constants.js';
import * as reporters from '../../../src/reporters/index.js';
import {parse} from '../../../src/lockfile';
import {Install, run as install} from '../../../src/cli/commands/install.js';
import Lockfile from '../../../src/lockfile';
import * as fs from '../../../src/util/fs.js';
Expand All @@ -16,7 +15,6 @@ import {getPackageVersion, explodeLockfile, runInstall, runLink, createLockfile,
jasmine.DEFAULT_TIMEOUT_INTERVAL = 150000;

let request = require('request');
const semver = require('semver');
const path = require('path');
const stream = require('stream');

Expand Down Expand Up @@ -145,22 +143,6 @@ test('reading a lockfile should not optimize it', async () => {
});
});

Copy link
Member Author

@bestander bestander Feb 9, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved offline mirror tests out

test('creates the file in the mirror when fetching a git repository', async () => {
await runInstall({}, 'install-git', async (config, reporter): Promise<void> => {
const lockfile = await Lockfile.fromDirectory(config.cwd);

expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1);

await fs.unlink(path.join(config.cwd, 'offline-mirror'));
await fs.unlink(path.join(config.cwd, 'node_modules'));

const firstReinstall = new Install({}, config, reporter, lockfile);
await firstReinstall.init();

expect(await fs.glob('example-yarn-package.git-*', {cwd: `${config.cwd}/offline-mirror`})).toHaveLength(1);
});
});

test.concurrent('creates a symlink to a directory when using the link: protocol', async () => {
await runInstall({}, 'install-link', async (config): Promise<void> => {
const expectPath = path.join(config.cwd, 'node_modules', 'test-absolute');
Expand Down Expand Up @@ -570,24 +552,6 @@ test.concurrent('install renamed packages', (): Promise<void> => {
});
});

test.concurrent('install from offline mirror', (): Promise<void> => {
return runInstall({}, 'install-from-offline-mirror', async (config): Promise<void> => {
const allFiles = await fs.walk(config.cwd);

expect(
allFiles.findIndex((file): boolean => {
return file.relative === path.join('node_modules', 'fake-dependency', 'package.json');
}),
).toBeGreaterThanOrEqual(0);

expect(
allFiles.findIndex((file): boolean => {
return file.relative === path.join('node_modules', '@fakescope', 'fake-dependency', 'package.json');
}),
).toBeGreaterThanOrEqual(0);
});
});

test.concurrent('install from git cache', (): Promise<void> => {
return runInstall({}, 'install-from-git-cache', async (config): Promise<void> => {
expect(await getPackageVersion(config, 'dep-a')).toEqual('0.0.1');
Expand Down Expand Up @@ -712,26 +676,6 @@ test.concurrent('install should be idempotent', (): Promise<void> => {
});
});

test.concurrent('install should add missing deps to yarn and mirror (PR import scenario)', (): Promise<void> => {
return runInstall({}, 'install-import-pr', async config => {
expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0');
expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true);
expect(await getPackageVersion(config, 'fake-yarn-dependency')).toEqual('1.0.1');

const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline'));
expect(mirror).toHaveLength(3);
expect(mirror[0].relative).toEqual('fake-yarn-dependency-1.0.1.tgz');
expect(mirror[1].relative.indexOf('mime-db-1.0.')).toEqual(0);
expect(mirror[2].relative).toEqual('mime-types-2.0.0.tgz');

const lockFileContent = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const lockFileLines = explodeLockfile(lockFileContent);
expect(lockFileLines).toHaveLength(11);
expect(lockFileLines[3].indexOf('mime-db@')).toEqual(0);
expect(lockFileLines[6].indexOf('[email protected]')).toEqual(0);
});
});

test.concurrent('install should update checksums in yarn.lock (--update-checksums)', (): Promise<void> => {
const packageRealHash = '5faad9c2c07f60dd76770f71cf025b62a63cfd4e';
const packageCacheName = `npm-abab-1.0.4-${packageRealHash}`;
Expand All @@ -747,47 +691,6 @@ test.concurrent('install should update checksums in yarn.lock (--update-checksum
});
});

test.concurrent('install should update a dependency to yarn and mirror (PR import scenario 2)', (): Promise<void> => {
// [email protected] is gets updated to [email protected] via
// a change in package.json,
// files in mirror, yarn.lock, package.json and node_modules should reflect that

return runInstall({}, 'install-import-pr-2', async (config, reporter): Promise<void> => {
expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.0.1')).toEqual(true);

expect(await getPackageVersion(config, 'mime-types')).toEqual('2.0.0');

await fs.copy(path.join(config.cwd, 'package.json.after'), path.join(config.cwd, 'package.json'), reporter);

const reinstall = new Install({}, config, reporter, await Lockfile.fromDirectory(config.cwd));
await reinstall.init();

expect(semver.satisfies(await getPackageVersion(config, 'mime-db'), '~1.23.0')).toEqual(true);

expect(await getPackageVersion(config, 'mime-types')).toEqual('2.1.11');

const lockFileWritten = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const lockFileLines = explodeLockfile(lockFileWritten);

expect(lockFileLines[0]).toEqual('mime-db@~1.23.0:');
expect(lockFileLines[2]).toMatch(/resolved "https:\/\/registry\.yarnpkg\.com\/mime-db\/-\/mime-db-/);

expect(lockFileLines[3]).toEqual('[email protected]:');
expect(lockFileLines[5]).toMatch(
/resolved "https:\/\/registry\.yarnpkg\.com\/mime-types\/-\/mime-types-2\.1\.11\.tgz#[a-f0-9]+"/,
);

const mirror = await fs.walk(path.join(config.cwd, 'mirror-for-offline'));
expect(mirror).toHaveLength(4);

const newFilesInMirror = mirror.filter((elem): boolean => {
return elem.relative !== 'mime-db-1.0.3.tgz' && elem.relative !== 'mime-types-2.0.0.tgz';
});

expect(newFilesInMirror).toHaveLength(2);
});
});

if (process.platform !== 'win32') {
// TODO: This seems like a real issue, not just a config issue
test.concurrent('install cache symlinks properly', (): Promise<void> => {
Expand All @@ -805,51 +708,6 @@ if (process.platform !== 'win32') {
});
}

test.concurrent('offline mirror can be enabled from parent dir', (): Promise<void> => {
const fixture = {
source: 'offline-mirror-configuration',
cwd: 'enabled-from-parent',
};
return runInstall({}, fixture, async (config, reporter) => {
const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const {object: lockfile} = parse(rawLockfile);
expect(lockfile['[email protected]'].resolved).toEqual(
'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee',
);
expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true);
});
});

test.concurrent('offline mirror can be enabled from parent dir, with merging of own .yarnrc', (): Promise<void> => {
const fixture = {
source: 'offline-mirror-configuration',
cwd: 'enabled-from-parent-merge',
};
return runInstall({}, fixture, async (config, reporter) => {
const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const {object: lockfile} = parse(rawLockfile);
expect(lockfile['[email protected]'].resolved).toEqual(
'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee',
);
expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(true);
});
});

test.concurrent('offline mirror can be disabled locally', (): Promise<void> => {
const fixture = {
source: 'offline-mirror-configuration',
cwd: 'disabled-locally',
};
return runInstall({}, fixture, async (config, reporter) => {
const rawLockfile = await fs.readFile(path.join(config.cwd, 'yarn.lock'));
const {object: lockfile} = parse(rawLockfile);
expect(lockfile['[email protected]'].resolved).toEqual(
'https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.14.tgz#f7ef7d97583fcaf3b7d282b6f8b5679dab1e94ee',
);
expect(await fs.exists(path.join(config.cwd, '../offline-mirror/mime-types-2.1.14.tgz'))).toBe(false);
});
});

// sync test because we need to get all the requests to confirm their validity
test('install a scoped module from authed private registry', (): Promise<void> => {
return runInstall({}, 'install-from-authed-private-registry', async config => {
Expand Down Expand Up @@ -1022,30 +880,6 @@ test.concurrent('should install if symlink source does not exist', async (): Pro
await runInstall({}, 'relative-symlinks-work', () => {});
});

test.concurrent('prunes the offline mirror tarballs after pruning is enabled', (): Promise<void> => {
return runInstall({}, 'prune-offline-mirror', async (config): Promise<void> => {
const mirrorPath = 'mirror-for-offline';
// Scenario:
// dep-a 1.0.0 was originally installed, and it depends on dep-b 1.0.0, so
// both of these were added to the offline mirror. Then dep-a was upgraded
// to 1.1.0 which doesn't depend on dep-b. After this, pruning was enabled,
// so the next install should remove dep-a-1.0.0.tgz and dep-b-1.0.0.tgz.
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-a-1.0.0.tgz`))).toEqual(false);
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dep-b-1.0.0.tgz`))).toEqual(false);
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/dummy.txt`))).toEqual(true);
});
});

test.concurrent('scoped packages remain in offline mirror after pruning is enabled', (): Promise<void> => {
return runInstall({}, 'prune-offline-mirror-scoped', async (config): Promise<void> => {
const mirrorPath = 'mirror-for-offline';
// scoped package exists
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/@fakescope-fake-dependency-1.0.1.tgz`))).toEqual(true);
// unscoped package exists
expect(await fs.exists(path.join(config.cwd, `${mirrorPath}/fake-dependency-1.0.1.tgz`))).toEqual(true);
});
});

test.concurrent('bailout should work with --production flag too', (): Promise<void> => {
return runInstall({production: true}, 'bailout-prod', async (config, reporter): Promise<void> => {
// remove file
Expand Down
Loading