feat: adding mdx files on generating llm

Author: luguellis

.gitignore

@@ -27,5 +27,8 @@ lib/
 
 src/**/*.mjs
 scripts/**/*.mjs
-public/llms/manual/**/*.txt
-public/llms/react/**/*.txt
+
+# Generated via generate-llms script
+public/llms/manual/**/llm*.txt
+public/llms/react/**/llm*.txt
+pages/docs/**/**/llms.mdx

package.json

@@ -58,10 +58,10 @@
   },
   "scripts": {
     "dev": "next",
-    "build": "rescript && npm run generate-llms && npm run update-index && next build",
+    "build": "rescript && npm run update-index && next build",
     "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs",
     "reanalyze": "reanalyze -all-cmt .",
-    "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml",
+    "update-index": "npm run generate-llms && node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml",
     "generate-llms": "node scripts/generate_llms.mjs"
   },
   "devDependencies": {

public/llms/manual/template.mdx

@@ -0,0 +1,23 @@
+---
+title: "LLMs"
+description: "Documentation for LLMs"
+canonical: "/docs/manual/<VERSION>/llms"
+---
+
+# Documentation for LLMs
+
+We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications.
+
+Currently, we have the following files...
+
+- [/docs/manual/llms.txt](/llms/manual/<VERSION>/llms.txt) — a list of the available files for ReScript language.
+- [/docs/manual/llm-full.txt](/llms/manual/<VERSION>/llm-full.txt) — complete documentation for ReScript language.
+- [/docs/manual/llm-small.txt](/llms/manual/<VERSION>/llm-small.txt) — compressed version of the former, without examples.
+
+...and package-level documentation:
+
+- [/docs/react/llms](/docs/react/latest/llms) — the LLms documentation for ReScript React.
+
+## Notes
+
+- The content is automatically generated from the same source as the official documentation for the specific version

public/llms/manual/template.txt

@@ -0,0 +1,19 @@
+# ReScript Documentation for LLMs
+
+> ReScript is a robustly typed language that compiles to efficient and human-readable JavaScript. It comes with a lightning fast compiler toolchain that scales to any codebase size.
+
+## Documentation Sets
+
+- [Complete documentation](https://rescript-lang.org/llms/manual/<VERSION>/llm-full.txt): The complete ReScript documentation including all examples and additional content
+- [Abridged documentation](https://rescript-lang.org/llms/manual/<VERSION>/llm-small.txt): A minimal version of the ReScript documentation, with the essential content for quick reference
+
+## Individual Package Documentation
+
+- [ReScript React documentation](https://rescript-lang.org/llms/react/latest/llms.txt): This is the developer documentation for ReScript React.
+
+## Notes
+
+- The abridged documentation excludes the detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation for the specific version

public/llms/react/template.mdx

@@ -0,0 +1,23 @@
+---
+title: "LLMs"
+description: "Documentation for LLMs"
+canonical: "/docs/react/<VERSION>/llms"
+---
+
+# Documentation for LLMs
+
+We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications.
+
+Currently, we have the following files...
+
+- [/docs/react/llms.txt](/llms/react/<VERSION>/llms.txt) — a list of the available files for ReScript React.
+- [/docs/react/llms-full.txt](/llms/react/<VERSION>/llm-full.txt) — complete documentation for ReScript React.
+- [/docs/react/llms-small.txt](/llms/react/<VERSION>/llm-small.txt) — compressed version of the former, without examples for ReScript React.
+
+...and the language documentation:
+
+- [/docs/manual/llms](/docs/manual/latest/llms) — the LLms documentation for ReScript.
+
+## Notes
+
+- The content is automatically generated from the same source as the official documentation for the specific version

public/llms/react/template.txt

@@ -0,0 +1,19 @@
+# ReScript React Documentation for LLMs
+
+> ReScript React is a strongly typed interface for React, designed to integrate seamlessly with modern React (>= v18.0) while compiling to idiomatic JavaScript, enabling robust and scalable React applications.
+
+## ReScript React Documentation
+
+- [Complete documentation](https://rescript-lang.org/llms/react/<VERSION>/llm-full.txt): The complete ReScript React documentation including all examples and additional content
+- [Abridged documentation](https://rescript-lang.org/llms/react/<VERSION>/llm-small.txt): A minimal version of the ReScript React documentation, with the essential content for quick reference
+
+## Language Documentation
+
+- [ReScript documentation](https://rescript-lang.org/llms/manual/latest/llms.txt): This is the developer documentation for ReScript.
+
+## Notes
+
+- The abridged documentation excludes the detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation for the specific version

scripts/generate_llms.res

@@ -53,6 +53,11 @@ let removeToDos = (content: string): string => {
   Js.String.replaceByRe(regex, "", content)
 }
 
+let fillContentWithVersion = (content: string, version: string): string => {
+  let regex = Js.Re.fromStringWithFlags("<VERSION>", ~flags="g")
+  Js.String.replaceByRe(regex, version, content)
+}
+
 let createFullFile = (content: string, filePath: string): unit => {
   Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8")
 }
@@ -68,14 +73,30 @@ let createSmallFile = (content: string, filePath: string): unit => {
   Node.Fs.appendFileSync(filePath, smallContent, "utf8")
 }
 
+let createLlmsFiles = (version: string, docsDirectory: string, llmsDirectory: string): unit => {
+  let mdxFileTemplatePath = llmsDirectory->Node.Path.join2("template.mdx")
+  let mdxFilePath = docsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.mdx")
+  let txtFileTemplatePath = llmsDirectory->Node.Path.join2("template.txt")
+  let txtFilePath = llmsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.txt")
+
+  Node.Fs.writeFileSync(
+    mdxFilePath,
+    readMarkdownFile(mdxFileTemplatePath)->fillContentWithVersion(version),
+  )
+
+  Node.Fs.writeFileSync(
+    txtFilePath,
+    readMarkdownFile(txtFileTemplatePath)->fillContentWithVersion(version),
+  )
+}
+
 let processVersions = (
   versions: array<string>,
   docsDirectory: string,
   llmsDirectory: string,
 ): unit => {
   let fullFileName = "llm-full.txt"
   let smallFileName = "llm-small.txt"
-  let llmsFileName = "llms.txt"
 
   versions->Array.forEach(version => {
     let versionDir = docsDirectory->Node.Path.join2(version)
@@ -87,6 +108,8 @@ let processVersions = (
     clearFile(fullFilePath)
     clearFile(smallFilePath)
 
+    createLlmsFiles(version, docsDirectory, llmsDirectory)
+
     versionDir
     ->collectFiles
     ->Array.forEach(filePath => {