WIP: Begin reorganizing the kernel with kernel-space typescript fs

.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+build/
+dist/
+package-lock.json

example/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "webabi-example",
+  "version": "0.0.1",
+  "description": "",
+  "scripts": {
+    "build": "tsc && webpack",
+    "install": "npm run build",
+    "test": "npm run build && (cd ..; echo bar | node --experimental-worker example/build/main.js)"
+  },
+  "author": "Will Fancher",
+  "dependencies": {
+    "typescript": "^2.9.2",
+    "webabi-kernel": "file:../kernel"
+  },
+  "devDependencies": {
+    "webpack": "^4.16.3",
+    "webpack-cli": "^3.1.0"
+  }
+}

example/src/index.ts

@@ -0,0 +1,36 @@
+import { Device, configureKernel, BFSCallback,
+         Stats, File, FileFlag, FS, ApiError,
+         ErrorCode, asyncRead, asyncWrite } from "webabi-kernel";
+import { makeWorker, PostMessage, OnMessage } from "webabi-kernel/dist/worker";
+
+class JSaddleDevice implements Device {
+  async open(flag: FileFlag): Promise<File> {
+    throw new ApiError(ErrorCode.ENOTSUP);
+  }
+  async stat(isLstat: boolean | null): Promise<Stats> {
+    throw new ApiError(ErrorCode.ENOTSUP);
+  }
+}
+
+async function main() {
+  let kernel = await configureKernel({ "/jsaddle": new JSaddleDevice() });
+  let fs = kernel.fs;
+  let buf = Buffer.from("foo\n");
+  await asyncWrite(fs, 1, buf, 0, buf.length, null);
+  const { byteLength } = await asyncRead(fs, 0, buf, 0, buf.length, null);
+  console.log({ byteLength: byteLength, buffer: buf.toString() });
+
+  console.log("working");
+  let worker: PostMessage;
+  worker = await makeWorker("./exec/build/main.js", {
+    onMessage: msg => {
+      console.log(msg);
+      worker.close();
+    }
+  });
+  worker.postMessage("foo");
+}
+
+main().catch(e => {
+  console.error("Error: ", e);
+});

example/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "outDir": "dist",
+    "lib": ["dom", "es2015", "es2016", "es2017"],
+    "module": "commonjs",
+    "declaration": true
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}

example/webpack.config.js

@@ -0,0 +1,25 @@
+const path = require('path');
+
+module.exports = {
+  entry: './dist/index.js',
+  mode: "production",
+  externals: [
+    ((context, request, callback) => {
+      if (request == "worker_threads") {
+        return callback(null, "commonjs " + request);
+      }
+      callback();
+    })
+  ],
+  output: {
+    filename: 'main.js',
+    path: path.resolve(__dirname, 'build')
+  },
+  resolve: {
+    // Using file:../kernel in package.json requires this
+    symlinks: false
+  },
+  node: {
+    process: false
+  }
+};

exec/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "webabi-exec",
+  "version": "0.0.1",
+  "description": "",
+  "scripts": {
+    "build": "tsc && webpack",
+    "install": "npm run build"
+  },
+  "main": "dist/index.js",
+  "author": "Will Fancher",
+  "dependencies": {
+    "typescript": "^2.9.2"
+  },
+  "devDependencies": {
+    "webpack": "^4.16.3",
+    "webpack-cli": "^3.1.0"
+  }
+}

exec/src/index.ts

@@ -0,0 +1,14 @@
+import { connectParent, PostMessage } from "./worker";
+
+async function main() {
+  let parent: PostMessage;
+  parent = await connectParent({
+    onMessage: msg => {
+      console.log(msg);
+      parent.close();
+    }
+  });
+  parent.postMessage("bar");
+}
+
+main();

exec/src/node_worker.ts

@@ -0,0 +1,8 @@
+declare module "worker_threads" {
+  interface MessagePort {
+    postMessage(msg: any, transferList: [any]);
+    on(event: "message", handler: (msg: any) => void);
+    unref(): void;
+  }
+  export const parentPort: MessagePort;
+}

exec/src/worker.ts

@@ -0,0 +1,28 @@
+export interface OnMessage {
+  onMessage(msg: any): void;
+}
+
+export interface PostMessage {
+  postMessage(msg: any, transferList?: [any]): void;
+  close(): void;
+}
+
+export let connectParent: (receiver: OnMessage) => Promise<PostMessage>;
+if (typeof self !== "undefined") {
+  connectParent = async (receiver) => {
+    self.onmessage = msg => receiver.onMessage(msg.data);
+    return {
+      postMessage: (msg, transferList) => self.postMessage(msg, transferList as any /*tsc complains wrongly*/),
+      close: () => {}
+    };
+  };
+} else {
+  connectParent = async (receiver) => {
+    const nodeWorkers = await import("worker_threads");
+    nodeWorkers.parentPort.on("message", msg => receiver.onMessage(msg));
+    return {
+      postMessage: (msg, transferList) => nodeWorkers.parentPort.postMessage(msg, transferList),
+      close: () => nodeWorkers.parentPort.unref()
+    };
+  }
+}

exec/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "outDir": "dist",
+    "lib": ["dom", "es2015", "es2016", "es2017"],
+    "module": "commonjs",
+    "declaration": true
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}

exec/webpack.config.js

@@ -0,0 +1,18 @@
+const path = require('path');
+
+module.exports = {
+  entry: './dist/index.js',
+  mode: "production",
+  externals: [
+    ((context, request, callback) => {
+      if (request == "worker_threads") {
+        return callback(null, "commonjs " + request);
+      }
+      callback();
+    })
+  ],
+  output: {
+    filename: 'main.js',
+    path: path.resolve(__dirname, 'build')
+  }
+};

index.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Getting Started</title>
+  </head>
+  <body>
+    <script src="./example/build/main.js"></script>
+  </body>
+</html>

kernel/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "webabi-kernel",
+  "version": "0.0.1",
+  "description": "",
+  "scripts": {
+    "build": "tsc",
+    "install": "npm run build"
+  },
+  "main": "dist/index.js",
+  "author": "Will Fancher",
+  "dependencies": {
+    "@types/archiver": "^2.0.0",
+    "@types/async": "^2.0.49",
+    "@types/body-parser": "^1.16.4",
+    "@types/dropboxjs": "0.0.29",
+    "@types/express": "^4.0.36",
+    "@types/filesystem": "0.0.28",
+    "@types/isomorphic-fetch": "^0.0.34",
+    "@types/mocha": "^5.2.5",
+    "@types/node": "^7.0",
+    "@types/rimraf": "^2.0.2",
+    "browserfs": "^1.4.3",
+    "typescript": "^2.9.2"
+  }
+}

kernel/src/DeviceFileSystem.ts

@@ -0,0 +1,63 @@
+import { BaseFileSystem, BFSCallback, FileSystem, FileSystemOptions
+       } from "browserfs/dist/node/core/file_system";
+import Stats from 'browserfs/dist/node/core/node_fs_stats';
+import { File } from "browserfs/dist/node/core/file";
+import { FileFlag } from "browserfs/dist/node/core/file_flag";
+import { ApiError } from 'browserfs/dist/node/core/api_error';
+
+export interface Device {
+  open(flag: FileFlag): Promise<File>;
+  stat(isLstat: boolean | null): Promise<Stats>;
+}
+
+export interface DeviceFileSystemOptions {
+  devices: {[name: string]: Device};
+}
+
+export class DeviceFileSystem extends BaseFileSystem implements FileSystem {
+  public static readonly Name = "DeviceFileSystem";
+  public static readonly Options: FileSystemOptions = {};
+
+  public static Create(opts: DeviceFileSystemOptions, cb: BFSCallback<DeviceFileSystem>): void {
+    return cb(null, new DeviceFileSystem(opts));
+  }
+
+  public static isAvailable(): boolean {
+    return true;
+  }
+
+  options: DeviceFileSystemOptions;
+
+  constructor(options: DeviceFileSystemOptions) {
+    super();
+    this.options = options;
+  }
+
+  public getName() {
+    return "DeviceFileSystem";
+  }
+  public isReadOnly() {
+    return false;
+  }
+  public supportsProps() {
+    return false;
+  }
+  public supportsSynch() {
+    return false;
+  }
+
+  public openFile(p: string, flag: FileFlag, cb: BFSCallback<File>): void {
+    if (this.options.devices.hasOwnProperty(p)) {
+      this.options.devices[p].open(flag).then(f => cb(undefined, f), e => cb(e));
+    } else {
+      cb(ApiError.ENOENT(p));
+    }
+  }
+  public stat(p: string, isLstat: boolean | null, cb: BFSCallback<Stats>): void {
+    if (this.options.devices.hasOwnProperty(p)) {
+      this.options.devices[p].stat(isLstat).then(s => cb(undefined, s), e => cb(e));
+    } else {
+      cb(ApiError.ENOENT(p));
+    }
+  }
+}

kernel/src/index.ts

@@ -0,0 +1,50 @@
+import MountableFileSystem from "browserfs/dist/node/backend/MountableFileSystem";
+import * as handles from "./stdio_handles";
+import { BFSCallback } from "browserfs/dist/node/core/file_system";
+import Stats from 'browserfs/dist/node/core/node_fs_stats';
+import { File } from "browserfs/dist/node/core/file";
+import { FileFlag } from "browserfs/dist/node/core/file_flag";
+import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error';
+import FS from "browserfs/dist/node/core/FS";
+import { DeviceFileSystem, Device }  from "./DeviceFileSystem";
+import Kernel from "./kernel";
+
+export async function configureKernel(devices: { [name: string]: Device }): Promise<Kernel> {
+  const dfs = await new Promise<DeviceFileSystem>((resolve, reject) => {
+    DeviceFileSystem.Create({ devices: devices }, (e, dfs) => e ? reject(e) : resolve(dfs))
+  });
+  const mfs = await new Promise<MountableFileSystem>((resolve, reject) => {
+    MountableFileSystem.Create({
+      "/dev": dfs
+    }, (e, mfs) => e ? reject(e) : resolve(mfs));
+  });
+
+  const fs = new FS();
+  fs.initialize(mfs);
+
+  const fdMap: {[id: number]: File} = (fs as any).fdMap;
+  fdMap[0] = handles.stdin;
+  fdMap[1] = handles.stdout;
+  fdMap[2] = handles.stderr;
+
+  return new Kernel(fs);
+}
+
+export async function asyncRead(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null)
+: Promise<{ byteLength: number, buffer: Buffer }> {
+  return new Promise<{ byteLength: number, buffer: Buffer }>((resolve, reject) => {
+    fs.read(fd, buffer, offset, length, position,
+            (err, n, buf) => err ? reject(err) : resolve({ byteLength: n, buffer: buf }));
+  });
+}
+
+export async function asyncWrite(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null)
+: Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    fs.write(fd, buffer, offset, length, position, e => e ? reject(e) : resolve())
+  });
+}
+
+// Re-export for device implementors
+export { BFSCallback, Stats, File, FileFlag, FS, ApiError, ErrorCode };
+export * from "./DeviceFileSystem";

kernel/src/kernel.ts

@@ -0,0 +1,9 @@
+import FS from "browserfs/dist/node/core/FS";
+import { makeWorker, PostMessage, OnMessage } from "./worker";
+
+export default class Kernel {
+  fs: FS;
+  constructor(fs: FS) {
+    this.fs = fs;
+  };
+}

kernel/src/node_parent.ts

@@ -0,0 +1,8 @@
+declare module "worker_threads" {
+  export class Worker {
+    constructor(path: string);
+    postMessage(msg: any, transferList: [any]): void;
+    on(event: "message", handler: (msg: any) => void);
+    unref(): void;
+  }
+}