feat(ai): add evaluation API reporting

.github/workflows/ci.yaml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  PNPM_VERSION: 10.16.1
+  NODE_VERSION: 22
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node:
+          - 20.x
+          - 22.x
+          - 24.x
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: ${{ env.PNPM_VERSION }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: 'pnpm'
+      - name: Cache turbo build setup
+        uses: actions/cache@v4
+        with:
+          path: ./node_modules/.cache/turbo
+          key: ${{ runner.os }}-turbo-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-turbo-
+      - run: pnpm install --frozen-lockfile --ignore-scripts
+      - run: pnpm build
+      - run: pnpm format:check
+      - run: pnpm lint
+      - run: pnpm typecheck
+      - run: pnpm test

.github/workflows/pr-title-lint.yaml

@@ -0,0 +1,19 @@
+name: "Lint PR Title"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - reopened
+
+jobs:
+  main:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish-any-commit.yaml

@@ -0,0 +1,24 @@
+name: Publish any commit
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - run: corepack enable
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        run: pnpm build
+
+      - run: pnpm dlx pkg-pr-new publish packages/ai

.github/workflows/release-ai-sdk.yaml

@@ -0,0 +1,24 @@
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  pull-requests: write
+
+name: release-ai-sdk
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: googleapis/release-please-action@v4
+        with:
+          # this assumes that you have created a personal access token
+          # (PAT) and configured it as a GitHub action secret named
+          # `RELEASE_PLEASE_TOKEN` (this secret name is not important).
+          token: ${{ secrets.AXIOM_AUTOMATION_TOKEN }}
+          config-file: release-please-config.json
+          manifest-file: .release-please-manifest.json
+          # 🚨 all other config is in `release-please-config.json`

.github/workflows/release-on-tag.yaml

@@ -0,0 +1,48 @@
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - axiom-v*
+
+env:
+  PNPM_VERSION: 10.16.1
+  NODE_VERSION: 22
+
+
+name: release-ai-sdk-on-tag
+
+jobs:
+  publish:
+    name: Publish AI package
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: ${{ env.PNPM_VERSION }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          registry-url: 'https://registry.npmjs.org'
+          cache: 'pnpm'
+          scope: '@axiomhq'
+      - name: Cache turbo build setup
+        uses: actions/cache@v4
+        with:
+          path: ./node_modules/.cache/turbo
+          key: ${{ runner.os }}-turbo-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-turbo-
+      - run: pnpm install --frozen-lockfile --ignore-scripts
+      - run: pnpm build
+      - name: Validate package.json
+        working-directory: packages/ai
+        run: pnpm pack
+      - name: Publish to Package Registry
+        working-directory: packages/ai
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_AXIOM_TOKEN }}
+        run: pnpm publish --access public --no-git-checks

.gitignore

@@ -4,4 +4,5 @@ node_modules/
 **/*/dist
 .env
 .DS_Store
-axiomhq-ai-*.tgz
+axiom-*.tgz
+coverage

.npmrc

@@ -6,5 +6,3 @@
 # hoist eslint and prettier to the root
 public-hoist-pattern[]=eslint*
 public-hoist-pattern[]=prettier*
-
-@axiomhq:registry=https://npm.pkg.github.com

.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  "packages/ai": "0.20.0"
+}

LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Axiom, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,9 +1,47 @@
 # Axiom AI
 
-This repo contains all the SDKs and libraries needed for AI models obsverability.
+Axiom AI SDK is a TypeScript toolkit designed to help you build an observability setup around your GenAI-powered apps based on the OpenTelemetry spec. 
 
-## Install Axiom SDK
+## Documentation
 
-```bash
-npm install @axiomhq/ai
-```
+For more information about how to set up and use the Axiom JavaScript SDK, read documentation on [axiom.co/docs/ai-engineering/quickstart](https://axiom.co/docs/ai-engineering/quickstart).
+
+## Contributing
+
+Contributions to Axiom AI SDK are welcome and highly appreciated! This project follows semantic commit conventions and maintains high code quality standards.
+
+### Prerequisites
+
+- Node.js 20.x, 22.x, or 24.x
+- pnpm package manager
+- knowledge of OpenTelemetry and TypeScript
+
+### Development setup
+
+1. Fork and clone the repository
+2. Install dependencies:
+   ```bash
+   pnpm install
+   ```
+
+3. Build the project:
+   ```bash
+   pnpm build
+   ```
+
+4. Run tests to ensure everything works:
+   ```bash
+   pnpm test
+   ```
+
+### Making changes & Pull Request process
+
+1. Push your changes to your fork
+2. Create a pull request with a semantic title
+3. Fill out the pull request description
+4. Ensure all CI checks pass
+5. Wait for review and address any feedback
+
+## Authors
+
+The SDK is created by [Axiom](https://axiom.co) team members.

examples/example-evals-nextjs/.env-example

@@ -0,0 +1,5 @@
+AXIOM_URL="https://api.axiom.co"
+AXIOM_TOKEN="xaat-******"
+AXIOM_DATASET="my_dataset"
+
+OPENAI_API_KEY="sk-proj-******"

examples/example-evals-nextjs/.gitignore

@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/example-evals-nextjs/.npmrc

@@ -0,0 +1 @@
+public-hoist-pattern[]=*require-in-the-middle*

examples/example-evals-nextjs/README.md

@@ -0,0 +1,17 @@
+# Eval examples
+
+To run all evals:
+
+- `axiom eval`
+
+To run one eval file:
+
+- `axiom eval feature.eval.ts`
+
+To override flags from CLI:
+
+- `axiom eval feature.eval.ts --flag.behavior.strategy=smart `
+
+To override flags from JSON:
+
+- `axiom eval feature.eval.ts --flags-config=experiment-example.json`

examples/example-evals-nextjs/eslint.config.js

@@ -0,0 +1,8 @@
+import { config } from '@repo/eslint-config';
+
+export default [
+  {
+    ignores: ['.next/**'],
+  },
+  ...config,
+];

examples/example-evals-nextjs/experiment-example.json

@@ -0,0 +1,5 @@
+{
+  "behavior": {
+    "strategy": "smart"
+  }
+}

examples/example-evals-nextjs/next.config.js

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+export default nextConfig;

examples/example-evals-nextjs/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "example-evals-nextjs",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "next",
+    "build": "next build",
+    "start": "next start",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "eslint './**/*.{js,ts}'",
+    "typecheck": "tsc --noEmit",
+    "eval": "axiom eval .",
+    "eval:by-file": "axiom eval src/lib/capabilities/classify-ticket/evaluations/ticket-classification.eval.ts",
+    "eval:by-glob": "axiom eval **/ticket*.ts",
+    "eval:by-name": "axiom eval \"Spam\""
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "2.0.0-beta.11",
+    "@ai-sdk/react": "2.0.0-beta.25",
+    "@ai-sdk/rsc": "1.0.0-beta.25",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/exporter-jaeger": "^2.0.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
+    "@opentelemetry/resources": "^2.1.0",
+    "@opentelemetry/sdk-node": "^0.202.0",
+    "@opentelemetry/sdk-trace-node": "^2.0.1",
+    "@opentelemetry/semantic-conventions": "^1.34.0",
+    "ai": "5.0.0-beta.24",
+    "axiom": "workspace:*",
+    "next": "latest",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "zod": "catalog:"
+  },
+  "devDependencies": {
+    "@repo/eslint-config": "workspace:*",
+    "@types/node": "^22.17.2",
+    "@types/react": "^18.2.8",
+    "autoevals": "^0.0.130",
+    "eslint": "catalog:",
+    "typescript": "catalog:"
+  }
+}