Merge pull request #236 from weaviate/text2vec-weaviate

Text2vec weaviate

Author: tsmith023

package-lock.json

@@ -24,8 +24,8 @@
     "test:coverage": "npm run test -- --coverage",
     "build": "npm run build:node",
     "build:web": "tsup",
-    "build:cjs": "tsc --module commonjs --outDir dist/node/cjs && touch dist/node/cjs/package.json && echo '{\"type\": \"commonjs\"}' > dist/node/cjs/package.json",
-    "build:esm": "tsc --module esnext --outDir dist/node/esm && touch dist/node/esm/package.json && echo '{\"type\": \"module\"}' > dist/node/esm/package.json",
+    "build:cjs": "tsc --module commonjs --moduleResolution node10 --outDir dist/node/cjs && touch dist/node/cjs/package.json && echo '{\"type\": \"commonjs\"}' > dist/node/cjs/package.json",
+    "build:esm": "tsc --outDir dist/node/esm && touch dist/node/esm/package.json && echo '{\"type\": \"module\"}' > dist/node/esm/package.json",
     "build:node": "npm run lint && npm run build:cjs && npm run build:esm && prettier --write --no-error-on-unmatched-pattern '**/dist/**/*.{ts,js}'",
     "prepack": "npm run build",
     "lint": "eslint --ext .ts,.js .",
@@ -95,7 +95,7 @@
     "tsup": "^8.0.2",
     "typedoc": "^0.25.12",
     "typedoc-plugin-extras": "^3.0.0",
-    "typescript": "5.1.3"
+    "typescript": "^5.3.3"
   },
   "lint-staged": {
     "*.{ts,js}": [

package.json

@@ -40,6 +40,7 @@ export type Vectorizer =
   | 'text2vec-google'
   | 'text2vec-transformers'
   | 'text2vec-voyageai'
+  | 'text2vec-weaviate'
   | 'none';
 
 /** The configuration for image vectorization using a neural network module.
@@ -406,6 +407,20 @@ export type Text2VecVoyageAIConfig = {
   vectorizeCollectionName?: boolean;
 };
 
+/**
+ * The configuration for text vectorization using Weaviate's self-hosted text-based embedding models.
+ *
+ * TODO: add documentation reference once available.
+ */
+export type Text2VecWeaviateConfig = {
+  /** The base URL to use where API requests should go. */
+  baseURL?: string;
+  /** The model to use. */
+  model?: 'Snowflake/snowflake-arctic-embed-m-v1.5' | string;
+  /** Whether to vectorize the collection name. */
+  vectorizeCollectionName?: boolean;
+};
+
 export type NoVectorizerConfig = {};
 
 export type VectorizerConfig =
@@ -428,6 +443,7 @@ export type VectorizerConfig =
   | Text2VecPalmConfig
   | Text2VecTransformersConfig
   | Text2VecVoyageAIConfig
+  | Text2VecWeaviateConfig
   | NoVectorizerConfig;
 
 export type VectorizerConfigType<V> = V extends 'img2vec-neural'
@@ -474,6 +490,8 @@ export type VectorizerConfigType<V> = V extends 'img2vec-neural'
   ? Text2VecTransformersConfig | undefined
   : V extends 'text2vec-voyageai'
   ? Text2VecVoyageAIConfig | undefined
+  : V extends 'text2vec-weaviate'
+  ? Text2VecWeaviateConfig | undefined
   : V extends 'none'
   ? {}
   : V extends undefined

src/collections/config/types/vectorizer.ts

@@ -17,6 +17,7 @@ import {
   Text2VecOpenAIConfig,
   Text2VecTransformersConfig,
   Text2VecVoyageAIConfig,
+  Text2VecWeaviateConfig,
   VectorIndexType,
   Vectorizer,
   VectorizerConfigType,
@@ -182,6 +183,8 @@ export type Text2VecTransformersConfigCreate = Text2VecTransformersConfig;
 
 export type Text2VecVoyageAIConfigCreate = Text2VecVoyageAIConfig;
 
+export type Text2VecWeaviateConfigCreate = Text2VecWeaviateConfig;
+
 export type VectorizerConfigCreateType<V> = V extends 'img2vec-neural'
   ? Img2VecNeuralConfigCreate | undefined
   : V extends 'multi2vec-clip'
@@ -226,6 +229,8 @@ export type VectorizerConfigCreateType<V> = V extends 'img2vec-neural'
   ? Text2VecTransformersConfigCreate | undefined
   : V extends 'text2vec-voyageai'
   ? Text2VecVoyageAIConfigCreate | undefined
+  : V extends 'text2vec-weaviate'
+  ? Text2VecWeaviateConfigCreate | undefined
   : V extends 'none'
   ? {}
   : V extends undefined

src/collections/configure/types/vectorizer.ts

@@ -1342,6 +1342,45 @@ describe('Unit testing of the vectorizer factory class', () => {
       },
     });
   });
+
+  it('should create the correct Text2VecWeaviateConfig type with defaults', () => {
+    const config = configure.vectorizer.text2VecWeaviate();
+    expect(config).toEqual<VectorConfigCreate<never, undefined, 'hnsw', 'text2vec-weaviate'>>({
+      name: undefined,
+      vectorIndex: {
+        name: 'hnsw',
+        config: undefined,
+      },
+      vectorizer: {
+        name: 'text2vec-weaviate',
+        config: undefined,
+      },
+    });
+  });
+
+  it('should create the correct Text2VecWeaviateConfig type with all values', () => {
+    const config = configure.vectorizer.text2VecWeaviate({
+      name: 'test',
+      baseURL: 'base-url',
+      model: 'model',
+      vectorizeCollectionName: true,
+    });
+    expect(config).toEqual<VectorConfigCreate<never, 'test', 'hnsw', 'text2vec-weaviate'>>({
+      name: 'test',
+      vectorIndex: {
+        name: 'hnsw',
+        config: undefined,
+      },
+      vectorizer: {
+        name: 'text2vec-weaviate',
+        config: {
+          baseURL: 'base-url',
+          model: 'model',
+          vectorizeCollectionName: true,
+        },
+      },
+    });
+  });
 });
 
 describe('Unit testing of the generative factory class', () => {

src/collections/configure/unit.test.ts

@@ -600,4 +600,26 @@ export const vectorizer = {
       },
     });
   },
+
+  /**
+   * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-weaviate'`.
+   *
+   * TODO: add documentation reference once available.
+   *
+   * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-weaviate'>} [opts] The configuration for the `text2vec-weaviate` vectorizer.
+   * @returns {VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-weaviate'>} The configuration object.
+   */
+  text2VecWeaviate: <T, N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
+    opts?: ConfigureTextVectorizerOptions<T, N, I, 'text2vec-weaviate'>
+  ): VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-weaviate'> => {
+    const { name, sourceProperties, vectorIndexConfig, ...config } = opts || {};
+    return makeVectorizer(name, {
+      sourceProperties,
+      vectorIndexConfig,
+      vectorizerConfig: {
+        name: 'text2vec-weaviate',
+        config: Object.keys(config).length === 0 ? undefined : config,
+      },
+    });
+  },
 };

src/collections/configure/vectorizer.ts

@@ -1,4 +1,5 @@
 import weaviate from '../index.js';
+import { connectToWeaviateCloud } from './helpers.js';
 
 const WCD_URL = 'https://piblpmmdsiknacjnm1ltla.c1.europe-west3.gcp.weaviate.cloud';
 const WCD_KEY = 'cy4ua772mBlMdfw3YnclqAWzFhQt0RLIN0sl';
@@ -29,4 +30,57 @@ describe('Testing of the connection helper methods', () => {
         throw new Error('it should not have errord: ' + e);
       });
   });
+
+  describe('adds Weaviate Embedding Service headers', () => {
+    it('to empty headers', async () => {
+      const clientMakerMock = jest.fn().mockResolvedValue(undefined);
+
+      await connectToWeaviateCloud(WCD_URL, clientMakerMock, {
+        authCredentials: new weaviate.ApiKey(WCD_KEY),
+      });
+
+      expect(clientMakerMock.mock.calls[0][0].headers).toEqual({
+        'X-Weaviate-Api-Key': WCD_KEY,
+        'X-Weaviate-Cluster-Url': WCD_URL,
+      });
+    });
+
+    it('to existing headers', async () => {
+      const clientMakerMock = jest.fn().mockResolvedValue(undefined);
+
+      await connectToWeaviateCloud(WCD_URL, clientMakerMock, {
+        authCredentials: new weaviate.ApiKey(WCD_KEY),
+        headers: { existingHeader: 'existingValue' },
+      });
+
+      expect(clientMakerMock.mock.calls[0][0].headers).toEqual({
+        existingHeader: 'existingValue',
+        'X-Weaviate-Api-Key': WCD_KEY,
+        'X-Weaviate-Cluster-Url': WCD_URL,
+      });
+    });
+  });
+
+  describe('does not add Weaviate Embedding Service headers when not using API key', () => {
+    it('to empty headers', async () => {
+      const clientMakerMock = jest.fn().mockResolvedValue(undefined);
+
+      await connectToWeaviateCloud(WCD_URL, clientMakerMock, {
+        authCredentials: new weaviate.AuthUserPasswordCredentials({ username: 'test' }),
+      });
+
+      expect(clientMakerMock.mock.calls[0][0].headers).toBe(undefined);
+    });
+
+    it('to existing headers', async () => {
+      const clientMakerMock = jest.fn().mockResolvedValue(undefined);
+
+      await connectToWeaviateCloud(WCD_URL, clientMakerMock, {
+        authCredentials: new weaviate.AuthUserPasswordCredentials({ username: 'test' }),
+        headers: { existingHeader: 'existingValue' },
+      });
+
+      expect(clientMakerMock.mock.calls[0][0].headers).toEqual({ existingHeader: 'existingValue' });
+    });
+  });
 });

src/connection/helpers.test.ts

@@ -1,6 +1,6 @@
 import { WeaviateStartUpError } from '../errors.js';
 import { ClientParams, WeaviateClient } from '../index.js';
-import { AuthCredentials } from './auth.js';
+import { AuthCredentials, isApiKey, mapApiKey } from './auth.js';
 import { ProxiesParams, TimeoutParams } from './http.js';
 
 /** The options available to the `weaviate.connectToWeaviateCloud` method. */
@@ -100,7 +100,7 @@ export function connectToWeaviateCloud(
       },
     },
     auth: options?.authCredentials,
-    headers: options?.headers,
+    headers: addWeaviateEmbeddingServiceHeaders(clusterURL, options),
   }).catch((e) => {
     throw new WeaviateStartUpError(`Weaviate failed to startup with message: ${e.message}`);
   });
@@ -155,3 +155,15 @@ export function connectToCustom(
     throw new WeaviateStartUpError(`Weaviate failed to startup with message: ${e.message}`);
   });
 }
+
+function addWeaviateEmbeddingServiceHeaders(clusterURL: string, options?: ConnectToWeaviateCloudOptions) {
+  if (!isApiKey(options?.authCredentials)) {
+    return options?.headers;
+  }
+
+  return {
+    ...options.headers,
+    'X-Weaviate-Api-Key': mapApiKey(options.authCredentials).apiKey,
+    'X-Weaviate-Cluster-Url': clusterURL,
+  };
+}

src/connection/helpers.ts

@@ -35,29 +35,29 @@ describe('connection', () => {
       });
   });
 
-  it('makes an Azure logged-in request with client credentials', async () => {
-    if (process.env.AZURE_CLIENT_SECRET == undefined || process.env.AZURE_CLIENT_SECRET == '') {
-      console.warn('Skipping because `AZURE_CLIENT_SECRET` is not set');
-      return Promise.resolve();
-    }
-
-    const client = await weaviate.connectToLocal({
-      port: 8081,
-      authCredentials: new AuthClientCredentials({
-        clientSecret: process.env.AZURE_CLIENT_SECRET,
-        silentRefresh: false,
-      }),
-    });
-
-    return client
-      .getMeta()
-      .then((res) => {
-        expect(res.version).toBeDefined();
-      })
-      .catch((e) => {
-        throw new Error('it should not have errord: ' + e);
-      });
-  });
+  // it('makes an Azure logged-in request with client credentials', async () => {
+  //   if (process.env.AZURE_CLIENT_SECRET == undefined || process.env.AZURE_CLIENT_SECRET == '') {
+  //     console.warn('Skipping because `AZURE_CLIENT_SECRET` is not set');
+  //     return Promise.resolve();
+  //   }
+
+  //   const client = await weaviate.connectToLocal({
+  //     port: 8081,
+  //     authCredentials: new AuthClientCredentials({
+  //       clientSecret: process.env.AZURE_CLIENT_SECRET,
+  //       silentRefresh: false,
+  //     }),
+  //   });
+
+  //   return client
+  //     .getMeta()
+  //     .then((res) => {
+  //       expect(res.version).toBeDefined();
+  //     })
+  //     .catch((e) => {
+  //       throw new Error('it should not have errord: ' + e);
+  //     });
+  // });
 
   it('makes an Okta logged-in request with client credentials', async () => {
     if (process.env.OKTA_CLIENT_SECRET == undefined || process.env.OKTA_CLIENT_SECRET == '') {