First cut of jupyter output

Signed-off-by: Jon Ludlam <[email protected]>

Author: jonludlam

CHANGES.md

@@ -2,6 +2,8 @@
 
 #### Added
 
+- Add Jupyter Notebook output (#124, @avsm, @CraigFe, @jonludlam)
+
 #### Changed
 
 #### Deprecated

bin/dune

@@ -1,11 +1,23 @@
 (library
  (name cli)
  (modules cli)
- (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx))
+ (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx yojson atdgen))
 
 (executable
  (name main)
  (public_name ocaml-mdx)
  (package mdx)
  (modules :standard \ cli)
  (libraries cli mdx))
+
+(rule
+ (targets notebook_j.ml notebook_j.mli)
+ (deps notebook.atd)
+ (action
+  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets notebook_t.ml notebook_t.mli)
+ (deps notebook.atd)
+ (action
+  (run atdgen -t %{deps})))

bin/jupyter.ml

@@ -0,0 +1,126 @@
+open Mdx.Util.Result.Infix
+open Cmdliner
+
+let raw t =
+  Notebook_t.
+    {
+      cell_type = `Raw;
+      metadata = { collapsed = None; scrolled = None };
+      source = String.concat "\n" t;
+      outputs = None;
+      execution_count = None;
+    }
+
+let txt source =
+  Notebook_t.
+    {
+      cell_type = `Markdown;
+      metadata = { collapsed = None; scrolled = None };
+      source;
+      outputs = None;
+      execution_count = None;
+    }
+
+let execution_count = ref 1
+
+let ocaml contents =
+  let cell =
+    Notebook_t.
+      {
+        cell_type = `Code;
+        metadata = { collapsed = None; scrolled = None };
+        source = String.concat "\n" contents;
+        outputs = Some [];
+        execution_count = Some !execution_count;
+      }
+  in
+  incr execution_count;
+  cell
+
+let toplevel x =
+  let cell =
+    Notebook_t.
+      {
+        cell_type = `Code;
+        metadata = { collapsed = None; scrolled = None };
+        source = String.concat "\n" x.Mdx.Toplevel.command;
+        outputs = Some [];
+        execution_count = Some !execution_count;
+      }
+  in
+  incr execution_count;
+  cell
+
+let metadata =
+  Notebook_t.
+    {
+      kernelspec =
+        {
+          display_name = "OCaml 4.07.1";
+          language = "OCaml";
+          name = "ocaml-jupyter";
+        };
+      language_info =
+        {
+          name = "OCaml";
+          version = "4.07.1";
+          codemirror_mode = Some "text/x-ocaml";
+          file_extension = ".ml";
+          mimetype = "text/x-ocaml";
+          nbconverter_exporter = None;
+          pygments_lexer = "OCaml";
+        };
+    }
+
+let rec collapse_text = function
+  | Mdx.Text x :: Mdx.Text y :: xs ->
+      collapse_text (Mdx.Text (x ^ "\n" ^ y) :: xs)
+  | (Mdx.Section _ as s) :: Mdx.Text y :: xs ->
+      let s = Mdx.to_string [ s ] in
+      collapse_text (Mdx.Text (s ^ "\n" ^ y) :: xs)
+  | (Mdx.Section _ as s) :: xs ->
+      let s = Mdx.to_string [ s ] in
+      collapse_text (Mdx.Text s :: xs)
+  | x :: ys -> x :: collapse_text ys
+  | [] -> []
+
+let run _setup (`Syntax syntax) (`File file) =
+  Mdx.run_to_stdout ?syntax file ~f:(fun _file_contents items ->
+      let syntax =
+        match syntax with
+        | Some s -> s
+        | None -> (
+            match Mdx.Syntax.infer ~file with
+            | Some s -> s
+            | None -> failwith "Couldn't get syntax")
+      in
+      let cells =
+        List.fold_left
+          (fun acc -> function
+            | Mdx.Text "" -> acc
+            | Mdx.Text x -> txt x :: acc
+            | Mdx.Block { value = OCaml { env = User_defined _; _ }; _ }
+            | Mdx.Block { value = Toplevel { env = User_defined _; _ }; _ } ->
+                failwith
+                  "internal error, cannot handle user defined environments"
+            | Mdx.Block { value = OCaml _; contents; _ } ->
+                ocaml contents :: acc
+            | Mdx.Block { value = Toplevel _; contents; loc; _ } ->
+                let blocks = Mdx.Toplevel.of_lines ~syntax ~loc contents in
+                let newcells = List.rev_map toplevel blocks in
+                newcells @ acc
+            | Mdx.Block { value = Raw _; contents; _ } -> raw contents :: acc
+            | x ->
+                failwith
+                  (Printf.sprintf "internal error, cannot handle: %s"
+                     (Mdx.to_string [ x ])))
+          [] (collapse_text items)
+        |> List.rev
+      in
+      Notebook_j.string_of_notebook
+        Notebook_t.{ metadata; nbformat = 4; nbformat_minor = 2; cells })
+  >>! fun () -> 0
+
+let cmd : int Term.t * Term.info =
+  let doc = "Convert an mdx file to a jupyter notebook." in
+  (Term.(pure run $ Cli.setup $ Cli.syntax $ Cli.file), Term.info "jupyter" ~doc)

bin/main.ml

@@ -16,7 +16,7 @@
 
 open Cmdliner
 
-let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd ]
+let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd; Jupyter.cmd ]
 
 let main (`Setup ()) = `Help (`Pager, None)
 

bin/notebook.atd

@@ -0,0 +1,57 @@
+type kernelspec = {
+    display_name : string;
+    language: string;
+    name: string;
+}
+
+type language_info = {
+    name: string;
+    version: string;
+    codemirror_mode: string nullable;
+    file_extension: string;
+    mimetype: string;
+    nbconverter_exporter: string nullable;
+    pygments_lexer: string;
+}
+
+type metadata = {
+    kernelspec: kernelspec;
+    language_info: language_info;
+}
+
+type cell_metadata = {
+    ?collapsed: bool nullable;
+    ?scrolled: bool nullable;
+}
+
+type cell_type = [
+    | Code <json name="code">
+    | Markdown <json name="markdown">
+    | Raw <json name="raw">
+]
+
+type output_type = [
+      Stream <json name="stream">
+    | Display_data <json name="display_data">
+    | Execute_result <json name="execute_result">
+    | Error <json name="error">
+]
+
+type output = {
+    output_type : output_type;
+}
+
+type cell = {
+    cell_type : cell_type;
+    metadata: cell_metadata;
+    source: string;
+    ?outputs: output list nullable;
+    ?execution_count: int option;
+} <json keep_nulls false>
+
+type notebook = {
+    metadata: metadata;
+    nbformat: int;
+    nbformat_minor: int;
+    cells: cell list
+}

dune-project

@@ -27,12 +27,14 @@
   (csexp
    (>= 1.3.2))
   astring
+  atdgen
   (logs (>= 0.7.0))
   (cmdliner
    (>= 1.0.0))
   (re
    (>= 1.7.2))
   result
+  yojson
   (ocaml-version
    (>= 2.3.0))
   (odoc-parser (>= 0.9.0))

mdx.opam

@@ -25,10 +25,12 @@ depends: [
   "cppo" {build}
   "csexp" {>= "1.3.2"}
   "astring"
+  "atdgen"
   "logs" {>= "0.7.0"}
   "cmdliner" {>= "1.0.0"}
   "re" {>= "1.7.2"}
   "result"
+  "yojson"
   "ocaml-version" {>= "2.3.0"}
   "odoc-parser" {>= "0.9.0"}
   "lwt" {with-test}