WIP: ability to store a built package in offline mirror

Author: bestander

src/cli/commands/pack.js

@@ -148,7 +148,7 @@ export function hasWrapper(commander: Object, args: Array<string>): boolean {
   return true;
 }
 
-export async function run(config: Config, reporter: Reporter, flags: Object, args: Array<string>): Promise<void> {
+export async function run(config: Config, reporter: Reporter, flags: {filename?: string}, args?: Array<string>): Promise<void> {
   const pkg = await config.readRootManifest();
   if (!pkg.name) {
     throw new MessageError(reporter.lang('noName'));

src/lockfile/index.js

@@ -29,6 +29,7 @@ export type LockManifest = {
   permissions: ?{[key: string]: boolean},
   optionalDependencies: ?Dependencies,
   dependencies: ?Dependencies,
+  prebuiltVariants: ?{[key: string]: string},
 };
 
 type MinimalLockManifest = {
@@ -69,6 +70,7 @@ export function implodeEntry(pattern: string, obj: Object): MinimalLockManifest
     dependencies: blankObjectUndefined(obj.dependencies),
     optionalDependencies: blankObjectUndefined(obj.optionalDependencies),
     permissions: blankObjectUndefined(obj.permissions),
+    prebuiltVariants: blankObjectUndefined(obj.prebuiltVariants),
   };
 }
 
@@ -184,7 +186,6 @@ export default class Lockfile {
         }
         continue;
       }
-
       const obj = implodeEntry(pattern, {
         name: pkg.name,
         version: pkg.version,
@@ -195,6 +196,7 @@ export default class Lockfile {
         peerDependencies: pkg.peerDependencies,
         optionalDependencies: pkg.optionalDependencies,
         permissions: ref.permissions,
+        prebuiltVariants: pkg.prebuiltVariants,
       });
       lockfile[pattern] = obj;
 

src/package-install-scripts.js

@@ -3,12 +3,16 @@
 import type {Manifest} from './types.js';
 import type PackageResolver from './package-resolver.js';
 import type {Reporter} from './reporters/index.js';
-import type Config from './config.js';
+import Config from './config.js';
 import type {ReporterSetSpinner} from './reporters/types.js';
 import executeLifecycleScript from './util/execute-lifecycle-script.js';
+import * as crypto from './util/crypto.js';
 import * as fs from './util/fs.js';
+import {pack} from './cli/commands/pack.js';Config
 
+const fs2 = require('fs');
 const invariant = require('invariant');
+const path = require('path');
 
 const INSTALL_STAGES = ['preinstall', 'install', 'postinstall'];
 
@@ -137,6 +141,13 @@ export default class PackageInstallScripts {
       return false;
     }
     const ref = pkg._reference;
+    if (pkg.prebuiltVariants) {
+      for (let variant in pkg.prebuiltVariants) {
+        if (pkg._remote && pkg._remote.reference && pkg._remote.reference.includes(variant)) {
+          return false;
+        }
+      }
+    }
     invariant(ref, 'Missing package reference');
     if (!ref.fresh && !this.force) {
       // this package hasn't been touched
@@ -291,7 +302,53 @@ export default class PackageInstallScripts {
         await this.saveBuildArtifacts(loc, pkg, beforeFiles, set.spinners[0]);
       }
     }
-
     set.end();
+
+    // generate built package as prebuilt one for offline mirror
+    for (const pkg of pkgs) {
+      if (this.packageCanBeInstalled(pkg)) {
+        const filename = PackageInstallScripts.getPrebuiltName(pkg);
+        // TODO maybe generated prebuilt packages should be in a subfolder
+        const filePath = this.config.getOfflineMirrorPath(filename + '.tgz');
+        if (!filePath) {
+          break;
+        }
+        const ref = pkg._reference;
+        invariant(ref, 'expected reference');
+        const loc = this.config.generateHardModulePath(ref);
+        const pkgConfig = await Config.create(
+          {
+            cwd: loc,
+          },
+          this.reporter,
+        );
+        const stream = await pack(pkgConfig, loc);
+
+        const hash = await new Promise((resolve, reject) => {
+          console.log("building", pkg.name)
+          const validateStream = new crypto.HashStream();
+          stream
+          .pipe(validateStream)
+          .pipe(fs2.createWriteStream(filePath))
+          .on('error', reject)
+          .on('close', () => resolve(validateStream.getHash()));
+        });
+        // TODO ! don't run scripts/install/preinstall/postinstall if pkg comes from prebuilt
+        // TODO ! don't save artifacts in .yarn-integrity, it is part of the package now
+        // TODO ! .yarn-integrity should contain prebuiltPackages array now
+        // TODO how does it scale to npm storage?
+        pkg.prebuiltVariants = pkg.prebuiltVariants || {};
+        pkg.prebuiltVariants[filename] = hash;
+      }
+    }
+
+  }
+
+  static getPrebuiltName(pkg: Manifest): string {
+    // TODO support platform variant for linux
+    // TODO support hash for all subdependencies that have installs scripts
+    const normaliseScope = name => (name[0] === '@' ? name.substr(1).replace('/', '-') : name);
+    const suffix = `${process.platform}-${process.arch}-${process.versions.modules || ''}`;
+    return `${normaliseScope(pkg.name)}-v${pkg.version}-${suffix}`;
   }
 }

src/package-request.js

@@ -56,6 +56,7 @@ export default class PackageRequest {
   optional: boolean;
   foundInfo: ?Manifest;
 
+  // TODO is it Manifest?
   getLocked(remoteType: string): ?Object {
     // always prioritise root lockfile
     const shrunk = this.lockfile.getLocked(this.pattern);
@@ -79,6 +80,7 @@ export default class PackageRequest {
         },
         optionalDependencies: shrunk.optionalDependencies,
         dependencies: shrunk.dependencies,
+        prebuiltVariants: shrunk.prebuiltVariants,
       };
     } else {
       return null;

src/package-resolver.js

@@ -105,6 +105,7 @@ export default class PackageResolver {
     newPkg._remote = ref.remote;
     newPkg.name = oldPkg.name;
     newPkg.fresh = oldPkg.fresh;
+    newPkg.prebuiltVariants = oldPkg.prebuiltVariants;
 
     // update patterns
     for (const pattern of ref.patterns) {
@@ -118,6 +119,8 @@ export default class PackageResolver {
     for (const newPkg of newPkgs) {
       if (newPkg._reference) {
         for (const pattern of newPkg._reference.patterns) {
+          const oldPkg = this.patterns[pattern];
+          newPkg.prebuiltVariants = oldPkg.prebuiltVariants;
           this.patterns[pattern] = newPkg;
         }
       }

src/resolvers/registries/npm-resolver.js

@@ -3,6 +3,7 @@
 import type {Manifest} from '../../types.js';
 import type Config from '../../config.js';
 import type PackageRequest from '../../package-request.js';
+import PackageInstallScripts from '../../package-install-scripts.js';
 import {MessageError} from '../../errors.js';
 import RegistryResolver from './registry-resolver.js';
 import NpmRegistry, {SCOPE_SEPARATOR} from '../../registries/npm-registry.js';
@@ -176,6 +177,14 @@ export default class NpmResolver extends RegistryResolver {
     // lockfile
     const shrunk = this.request.getLocked('tarball');
     if (shrunk) {
+      if (shrunk.prebuiltVariants && shrunk._remote) {
+        const prebuiltName = PackageInstallScripts.getPrebuiltName(shrunk);
+        if (shrunk.prebuiltVariants[prebuiltName]) {
+          const filename = this.config.getOfflineMirrorPath(prebuiltName + '.tgz');
+          shrunk._remote.reference = `file:${filename || ''}`;
+          shrunk._remote.hash = shrunk.prebuiltVariants[prebuiltName];
+        }
+      }
       return shrunk;
     }
 

src/types.js

@@ -141,6 +141,8 @@ export type Manifest = {
   // We need to preserve the flag because we print a list of new packages in
   // the end of the add command
   fresh?: boolean,
+
+  prebuiltVariants?: {[filename: string]: string},
 };
 
 //