Adds whether there are types for a package to an index

README.md

@@ -39,94 +39,94 @@ For every single NPM package, we create a record in the Algolia index. The resul
 
 ```json5
 {
-  "name": "babel-core",
-  "concatenatedName": "babelcore",
-  "downloadsLast30Days": 10978749,
-  "downloadsRatio": 0.08310651682685861,
-  "humanDownloadsLast30Days": "11m",
-  "jsDelivrHits": 11684192,
-  "popular": true,
-  "version": "6.26.0",
-  "versions": {
+  name: 'babel-core',
+  concatenatedName: 'babelcore',
+  downloadsLast30Days: 10978749,
+  downloadsRatio: 0.08310651682685861,
+  humanDownloadsLast30Days: '11m',
+  jsDelivrHits: 11684192,
+  popular: true,
+  version: '6.26.0',
+  versions: {
     // [...]
-    "7.0.0-beta.3": "2017-10-15T13:12:35.166Z"
+    '7.0.0-beta.3': '2017-10-15T13:12:35.166Z',
   },
-  "tags": {
-    "latest": "6.26.0",
-    "old": "5.8.38",
-    "next": "7.0.0-beta.3"
+  tags: {
+    latest: '6.26.0',
+    old: '5.8.38',
+    next: '7.0.0-beta.3',
   },
-  "description": "Babel compiler core.",
-  "dependencies": {
-    "babel-code-frame": "^6.26.0"
+  description: 'Babel compiler core.',
+  dependencies: {
+    'babel-code-frame': '^6.26.0',
     // [...]
   },
-  "devDependencies": {
-    "babel-helper-fixtures": "^6.26.0"
+  devDependencies: {
+    'babel-helper-fixtures': '^6.26.0',
     // [...]
   },
-  "repository": {
-    "url": "https://github.com/babel/babel/tree/master/packages/babel-core",
-    "host": "github.com",
-    "user": "babel",
-    "project": "babel",
-    "path": "/tree/master/packages/babel-core",
-    "branch": "master"
+  repository: {
+    url: 'https://github.com/babel/babel/tree/master/packages/babel-core',
+    host: 'github.com',
+    user: 'babel',
+    project: 'babel',
+    path: '/tree/master/packages/babel-core',
+    branch: 'master',
   },
-  "readme":
-    "# babel-core\n\n> Babel compiler core.\n\n\n [... truncated at 200kb]",
-  "owner": {
+  readme: '# babel-core\n\n> Babel compiler core.\n\n\n [... truncated at 200kb]',
+  owner: {
     // either GitHub owner or npm owner
-    "name": "babel",
-    "avatar": "https://github.com/babel.png",
-    "link": "https://github.com/babel"
+    name: 'babel',
+    avatar: 'https://github.com/babel.png',
+    link: 'https://github.com/babel',
   },
-  "deprecated": false,
-  "badPackage": false,
-  "homepage": "https://babeljs.io/",
-  "license": "MIT",
-  "keywords": [
-    "6to5",
-    "babel",
-    "classes",
-    "const",
-    "es6",
-    "harmony",
-    "let",
-    "modules",
-    "transpile",
-    "transpiler",
-    "var",
-    "babel-core",
-    "compiler"
+  deprecated: false,
+  badPackage: false,
+  homepage: 'https://babeljs.io/',
+  license: 'MIT',
+  keywords: [
+    '6to5',
+    'babel',
+    'classes',
+    'const',
+    'es6',
+    'harmony',
+    'let',
+    'modules',
+    'transpile',
+    'transpiler',
+    'var',
+    'babel-core',
+    'compiler',
   ],
-  "created": 1424009748555,
-  "modified": 1508833762239,
-  "lastPublisher": {
-    "name": "hzoo",
-    "email": "[email protected]",
-    "avatar": "https://gravatar.com/avatar/851fb4fa7ca479bce1ae0cdf80d6e042",
-    "link": "https://www.npmjs.com/~hzoo"
+  created: 1424009748555,
+  modified: 1508833762239,
+  lastPublisher: {
+    name: 'hzoo',
+    email: '[email protected]',
+    avatar: 'https://gravatar.com/avatar/851fb4fa7ca479bce1ae0cdf80d6e042',
+    link: 'https://www.npmjs.com/~hzoo',
   },
-  "owners": [
+  owners: [
     {
-      "email": "[email protected]",
-      "name": "thejameskyle",
-      "avatar": "https://gravatar.com/avatar/8a00efb48d632ae449794c094f7d5c38",
-      "link": "https://www.npmjs.com/~thejameskyle"
-    }
+      email: '[email protected]',
+      name: 'thejameskyle',
+      avatar: 'https://gravatar.com/avatar/8a00efb48d632ae449794c094f7d5c38',
+      link: 'https://www.npmjs.com/~thejameskyle',
+    },
     // [...]
   ],
-  "lastCrawl": "2017-10-24T08:29:24.672Z",
-  "dependents": 3321,
-  "humanDependents": "3.3k",
-  "changelogFilename": null, // if babel-core had a changelog, it would be the raw GitHub url here
-  "objectID": "babel-core",
-  "_searchInternal": {
-    "popularName": "babel-core",
-    "downloadsMagnitude": 8,
-    "jsDelivrPopularity": 5
-  }
+  lastCrawl: '2017-10-24T08:29:24.672Z',
+  dependents: 3321,
+  types: { ts: '@types/babel-core' },
+  humanDependents: '3.3k',
+  changelogFilename: null, // if babel-core had a changelog, it would be the raw GitHub url here
+  objectID: 'babel-core',
+  _searchInternal: {
+    popularName: 'babel-core',
+    downloadsMagnitude: 8,
+    jsDelivrPopularity: 5,
+  },
 }
 ```
 
@@ -182,7 +182,7 @@ apiKey=... yarn start
 
 ### Restart
 
-To restart from a particular point (or from the begining):
+To restart from a particular point (or from the beginning):
 
 ```sh
 seq=0 apiKey=... yarn start

src/__tests__/__snapshots__/config.test.js.snap

@@ -179,9 +179,11 @@ Object {
   "maxObjSize": 450000,
   "npmDownloadsEndpoint": "https://api.npmjs.org/downloads",
   "npmRegistryEndpoint": "https://replicate.npmjs.com/registry",
+  "npmRootEndpoint": "https://api.npmjs.org/",
   "popularDownloadsRatio": 0.005,
   "replicateConcurrency": 10,
   "seq": null,
   "timeToRedoBootstrap": 604800000,
+  "unpkgRoot": "https://unpkg.com/",
 }
 `;

src/__tests__/typescript.test.js

@@ -0,0 +1,47 @@
+import { getTypeScriptSupport } from '../typescriptSupport';
+jest.mock('../npm');
+jest.mock('../unpkg');
+import { validatePackageExists } from '../npm';
+import { fileExistsInUnpkg } from '../unpkg.js';
+
+describe('getTypeScriptSupport()', () => {
+  it('If types or typings are present in pkg.json - return early', async () => {
+    let typesSupport = await getTypeScriptSupport({
+      name: 'Has Types',
+      types: './types',
+    });
+
+    expect(typesSupport).toEqual({ types: { ts: 'included' } });
+
+    typesSupport = await getTypeScriptSupport({
+      name: 'Has Types',
+      typings: './types',
+    });
+
+    expect(typesSupport).toEqual({ types: { ts: 'included' } });
+  });
+
+  describe('without types/typings', () => {
+    it('Checks for @types/[name]', async () => {
+      validatePackageExists.mockResolvedValue(true);
+      const atTypesSupport = await getTypeScriptSupport({ name: 'my-lib' });
+      expect(atTypesSupport).toEqual({ types: { ts: '@types/my-lib' } });
+    });
+
+    it('Checks for a d.ts resolved version of main ', async () => {
+      validatePackageExists.mockResolvedValue(false);
+      fileExistsInUnpkg.mockResolvedValue(true);
+
+      const typesSupport = await getTypeScriptSupport({ name: 'my-lib' });
+      expect(typesSupport).toEqual({ types: { ts: 'included' } });
+    });
+
+    it('Handles not having and TS types', async () => {
+      validatePackageExists.mockResolvedValue(false);
+      fileExistsInUnpkg.mockResolvedValue(false);
+
+      const typesSupport = await getTypeScriptSupport({ name: 'my-lib' });
+      expect(typesSupport).toEqual({ types: { ts: null } });
+    });
+  });
+});

src/config.js

@@ -5,7 +5,9 @@ import ms from 'ms';
 const defaultConfig = {
   npmRegistryEndpoint: 'https://replicate.npmjs.com/registry',
   npmDownloadsEndpoint: 'https://api.npmjs.org/downloads',
+  npmRootEndpoint: 'https://api.npmjs.org/',
   jsDelivrHitsEndpoint: 'https://data.jsdelivr.com/v1/stats/packages/month/all',
+  unpkgRoot: 'https://unpkg.com/',
   maxObjSize: 450000,
   popularDownloadsRatio: 0.005,
   appId: 'OFCNCOG2CU',

src/npm.js

@@ -20,6 +20,13 @@ const logWarning = ({ error, type, packagesStr }) => {
   );
 };
 
+export function validatePackageExists(pkgName) {
+  return got(`${c.npmRootEndpoint}/${pkgName}`, {
+    json: true,
+    method: 'HEAD',
+  }).then(response => response.statusCode === 200);
+}
+
 export async function getDownloads(pkgs) {
   // npm has a weird API to get downloads via GET params, so we split pkgs into chunks
   // and do multiple requests to avoid weird cases when concurrency is high

src/saveDocs.js

@@ -3,6 +3,7 @@ import log from './log.js';
 import { getDownloads, getDependents } from './npm.js';
 import { getChangelogs } from './changelog.js';
 import { getHits } from './jsDelivr';
+import { getTSSupport } from './typescriptSupport';
 
 export default function saveDocs({ docs, index }) {
   const rawPkgs = docs
@@ -26,13 +27,15 @@ function addMetaData(pkgs) {
     getDependents(pkgs),
     getChangelogs(pkgs),
     getHits(pkgs),
-  ]).then(([downloads, dependents, changelogs, hits]) =>
+    getTSSupport(pkgs),
+  ]).then(([downloads, dependents, changelogs, hits, ts]) =>
     pkgs.map((pkg, index) => ({
       ...pkg,
       ...downloads[index],
       ...dependents[index],
       ...changelogs[index],
       ...hits[index],
+      ...ts[index],
       _searchInternal: {
         ...pkg._searchInternal,
         ...downloads[index]._searchInternal,

src/typescriptSupport.js

@@ -0,0 +1,46 @@
+// @ts-check
+
+import { validatePackageExists } from './npm.js';
+import { fileExistsInUnpkg } from './unpkg.js';
+
+/**
+ * Basically either
+ *  - { types: { ts: null }}  for no existing TypeScript support
+ *  - { types: { ts: "@types/module" }} - for definitely typed support
+ *  - { types: { ts: "included" }} - for types shipped with the module
+ * */
+export async function getTypeScriptSupport(pkg) {
+  // The cheap and simple (+ recommended by TS) way
+  // of adding a types section to your package.json
+  if (pkg.types) {
+    return { types: { ts: 'included' } };
+  }
+
+  // Older, but still works way of defining your types
+  if (pkg.typings) {
+    return { types: { ts: 'included' } };
+  }
+
+  // The 2nd most likely is definitely typed
+  const defTypeName = `@types/${pkg.name}`;
+  const defTyped = await validatePackageExists(defTypeName);
+  if (defTyped) {
+    return { types: { ts: defTypeName } };
+  }
+
+  // Check if main's JS file can be resolved to a d.ts file instead
+  const main = pkg.main || 'index.js';
+  if (main.endsWith('.js')) {
+    const dtsMain = main.replace(/js$/, 'd.ts');
+    const resolved = await fileExistsInUnpkg(pkg.name, pkg.version, dtsMain);
+    if (resolved) {
+      return { types: { ts: 'included' } };
+    }
+  }
+
+  return { types: { ts: null } };
+}
+
+export function getTSSupport(pkgs) {
+  return Promise.all(pkgs.map(getTypeScriptSupport));
+}

src/unpkg.js

@@ -0,0 +1,14 @@
+// @ts-check
+import c from './config.js';
+import got from 'got';
+
+// make a head request to a route like:
+// https://unpkg.com/[email protected]/_LazyWrapper.js
+// to validate the existence of a particular file
+export function fileExistsInUnpkg(pkg, version, path) {
+  const uri = `${c.unpkgRoot}/${pkg}@${version}/${path}`;
+  return got(uri, {
+    json: true,
+    method: 'HEAD',
+  }).then(response => response.statusCode === 200);
+}