Skip to content

Commit a27b837

Browse files
rickzxrickzx
authored
[JSON] Add Hermes-2-Pro function calling example with JSON schema (#390)
This PR adds a function calling example with JSON schema, using Hermes-2-Pro-Mistral-7B model. The system prompt is adapted from: https://github.com/NousResearch/Hermes-Function-Calling/blob/main/prompt_assets/sys_prompt.yml#L29 Query: ``` What is the current weather in celsius in Pittsburgh and Tokyo? ``` Output JSON schema: ``` const T = Type.Object({ tool_calls: Type.Array( Type.Object({ arguments: Type.Any(), name: Type.String(), }) ) }); ``` Generated output: ``` {"tool_calls": [{"arguments": {"location": "Pittsburgh, PA", "unit": "celsius"}, "name": "get_current_weather"}, {"arguments": {"location": "Tokyo, Japan", "unit": "celsius"}, "name": "get_current_weather"}]} ```
1 parent ce725b6 commit a27b837

File tree

4 files changed

+215
-51
lines changed

4 files changed

+215
-51
lines changed

‎examples/json-schema/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@
1515
"url": "^0.11.3"
1616
},
1717
"dependencies": {
18-
"@mlc-ai/web-llm": "^0.2.35"
18+
"@mlc-ai/web-llm": "file:../.."
1919
}
2020
}

‎examples/json-schema/src/json_schema.ts‎

Lines changed: 205 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,219 @@
11
import * as webllm from "@mlc-ai/web-llm";
2+
import { Type, Static } from "@sinclair/typebox";
23

34
function setLabel(id: string, text: string) {
4-
const label = document.getElementById(id);
5-
if (label == null) {
6-
throw Error("Cannot find label " + id);
7-
}
8-
label.innerText = text;
5+
const label = document.getElementById(id);
6+
if (label == null) {
7+
throw Error("Cannot find label " + id);
8+
}
9+
label.innerText = text;
910
}
1011

11-
// There are several options of providing such a schema
12-
// 1. You can directly define a schema in string
13-
const schema1 = `{
14-
"properties": {
15-
"size": {"title": "Size", "type": "integer"},
16-
"is_accepted": {"title": "Is Accepted", "type": "boolean"},
17-
"num": {"title": "Num", "type": "number"}
18-
},
19-
"required": ["size", "is_accepted", "num"],
20-
"title": "Schema", "type": "object"
21-
}`;
12+
async function simpleStructuredTextExample() {
13+
// There are several options of providing such a schema
14+
// 1. You can directly define a schema in string
15+
const schema1 = `{
16+
"properties": {
17+
"size": {"title": "Size", "type": "integer"},
18+
"is_accepted": {"title": "Is Accepted", "type": "boolean"},
19+
"num": {"title": "Num", "type": "number"}
20+
},
21+
"required": ["size", "is_accepted", "num"],
22+
"title": "Schema", "type": "object"
23+
}`;
2224

23-
// 2. You can use 3rdparty libraries like typebox to create a schema
24-
import { Type, type Static } from '@sinclair/typebox'
25-
const T = Type.Object({
25+
// 2. You can use 3rdparty libraries like typebox to create a schema
26+
const T = Type.Object({
2627
size: Type.Integer(),
2728
is_accepted: Type.Boolean(),
2829
num: Type.Number(),
29-
})
30-
type T = Static<typeof T>;
31-
const schema2 = JSON.stringify(T);
32-
console.log(schema2);
33-
// {"type":"object","properties":{"size":{"type":"integer"},"is_accepted":{"type":"boolean"},
34-
// "num":{"type":"number"}},"required":["size","is_accepted","num"]}
30+
});
31+
type T = Static<typeof T>;
32+
const schema2 = JSON.stringify(T);
33+
console.log(schema2);
34+
// {"type":"object","properties":{"size":{"type":"integer"},"is_accepted":{"type":"boolean"},
35+
// "num":{"type":"number"}},"required":["size","is_accepted","num"]}
36+
37+
const initProgressCallback = (report: webllm.InitProgressReport) => {
38+
setLabel("init-label", report.text);
39+
};
40+
const engine: webllm.EngineInterface = await webllm.CreateEngine(
41+
"Llama-2-7b-chat-hf-q4f16_1",
42+
{ initProgressCallback: initProgressCallback }
43+
);
44+
45+
const request: webllm.ChatCompletionRequest = {
46+
stream: false, // works with streaming, logprobs, top_logprobs as well
47+
messages: [
48+
{
49+
role: "user",
50+
content:
51+
"Generate a json containing three fields: an integer field named size, a " +
52+
"boolean field named is_accepted, and a float field named num.",
53+
},
54+
],
55+
max_gen_len: 128,
56+
response_format: {
57+
type: "json_object",
58+
schema: schema2,
59+
} as webllm.ResponseFormat,
60+
};
61+
62+
const reply0 = await engine.chatCompletion(request);
63+
console.log(reply0);
64+
console.log("Output:\n" + (await engine.getMessage()));
65+
console.log(await engine.runtimeStatsText());
66+
}
67+
68+
// The json schema and prompt is taken from
69+
// https://github.com/sgl-project/sglang/tree/main?tab=readme-ov-file#json-decoding
70+
async function harryPotterExample() {
71+
const T = Type.Object({
72+
name: Type.String(),
73+
house: Type.Enum({
74+
Gryffindor: "Gryffindor",
75+
Hufflepuff: "Hufflepuff",
76+
Ravenclaw: "Ravenclaw",
77+
Slytherin: "Slytherin",
78+
}),
79+
blood_status: Type.Enum({
80+
"Pure-blood": "Pure-blood",
81+
"Half-blood": "Half-blood",
82+
"Muggle-born": "Muggle-born",
83+
}),
84+
occupation: Type.Enum({
85+
Student: "Student",
86+
Professor: "Professor",
87+
"Ministry of Magic": "Ministry of Magic",
88+
Other: "Other",
89+
}),
90+
wand: Type.Object({
91+
wood: Type.String(),
92+
core: Type.String(),
93+
length: Type.Number(),
94+
}),
95+
alive: Type.Boolean(),
96+
patronus: Type.String(),
97+
});
98+
99+
type T = Static<typeof T>;
100+
const schema = JSON.stringify(T);
101+
console.log(schema);
102+
103+
const initProgressCallback = (report: webllm.InitProgressReport) => {
104+
setLabel("init-label", report.text);
105+
};
106+
107+
const engine: webllm.EngineInterface = await webllm.CreateEngine(
108+
"Llama-2-7b-chat-hf-q4f16_1",
109+
{ initProgressCallback: initProgressCallback }
110+
);
111+
112+
const request: webllm.ChatCompletionRequest = {
113+
stream: false,
114+
messages: [
115+
{
116+
role: "user",
117+
content:
118+
"Hermione Granger is a character in Harry Potter. Please fill in the following information about this character in JSON format." +
119+
"Name is a string of character name. House is one of Gryffindor, Hufflepuff, Ravenclaw, Slytherin. Blood status is one of Pure-blood, Half-blood, Muggle-born. Occupation is one of Student, Professor, Ministry of Magic, Other. Wand is an object with wood, core, and length. Alive is a boolean. Patronus is a string.",
120+
},
121+
],
122+
max_gen_len: 128,
123+
response_format: {
124+
type: "json_object",
125+
schema: schema,
126+
} as webllm.ResponseFormat,
127+
};
128+
129+
const reply = await engine.chatCompletion(request);
130+
console.log(reply);
131+
console.log("Output:\n" + (await engine.getMessage()));
132+
console.log(await engine.runtimeStatsText());
133+
}
134+
135+
async function functionCallingExample() {
136+
const T = Type.Object({
137+
tool_calls: Type.Array(
138+
Type.Object({
139+
arguments: Type.Any(),
140+
name: Type.String(),
141+
})
142+
),
143+
});
144+
type T = Static<typeof T>;
145+
const schema = JSON.stringify(T);
146+
console.log(schema);
147+
148+
const tools: Array<webllm.ChatCompletionTool> = [
149+
{
150+
type: "function",
151+
function: {
152+
name: "get_current_weather",
153+
description: "Get the current weather in a given location",
154+
parameters: {
155+
type: "object",
156+
properties: {
157+
location: {
158+
type: "string",
159+
description: "The city and state, e.g. San Francisco, CA",
160+
},
161+
unit: { type: "string", enum: ["celsius", "fahrenheit"] },
162+
},
163+
required: ["location"],
164+
},
165+
},
166+
},
167+
];
168+
169+
const initProgressCallback = (report: webllm.InitProgressReport) => {
170+
setLabel("init-label", report.text);
171+
};
172+
173+
const selectedModel = "Hermes-2-Pro-Mistral-7B-q4f16_1";
174+
const engine: webllm.EngineInterface = await webllm.CreateEngine(
175+
selectedModel,
176+
{
177+
initProgressCallback: initProgressCallback,
178+
}
179+
);
180+
181+
const request: webllm.ChatCompletionRequest = {
182+
stream: false,
183+
messages: [
184+
{
185+
role: "system",
186+
content: `You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> ${JSON.stringify(
187+
tools
188+
)} </tools>. Do not stop calling functions until the task has been accomplished or you've reached max iteration of 10.
189+
Calling multiple functions at once can overload the system and increase cost so call one function at a time please.
190+
If you plan to continue with analysis, always call another function.
191+
Return a valid json object (using double quotes) in the following schema: ${JSON.stringify(
192+
schema
193+
)}.`,
194+
},
195+
{
196+
role: "user",
197+
content:
198+
"What is the current weather in celsius in Pittsburgh and Tokyo?",
199+
},
200+
],
201+
response_format: {
202+
type: "json_object",
203+
schema: schema,
204+
} as webllm.ResponseFormat,
205+
};
206+
207+
const reply = await engine.chat.completions.create(request);
208+
console.log(reply.choices[0].message.content);
209+
210+
console.log(await engine.runtimeStatsText());
211+
}
35212

36213
async function main() {
37-
const initProgressCallback = (report: webllm.InitProgressReport) => {
38-
setLabel("init-label", report.text);
39-
};
40-
const engine: webllm.EngineInterface = await webllm.CreateEngine(
41-
"Llama-2-7b-chat-hf-q4f16_1", { initProgressCallback: initProgressCallback }
42-
);
43-
44-
const request: webllm.ChatCompletionRequest = {
45-
stream: false, // works with streaming, logprobs, top_logprobs as well
46-
messages: [
47-
{
48-
"role": "user",
49-
"content": "Generate a json containing three fields: an integer field named size, a " +
50-
"boolean field named is_accepted, and a float field named num."
51-
}
52-
],
53-
max_gen_len: 128,
54-
response_format: { type: "json_object", schema: schema2 } as webllm.ResponseFormat
55-
};
56-
57-
const reply0 = await engine.chatCompletion(request);
58-
console.log(reply0);
59-
console.log("Output:\n" + await engine.getMessage());
60-
console.log(await engine.runtimeStatsText());
214+
// await simpleStructuredTextExample();
215+
// await harryPotterExample();
216+
await functionCallingExample();
61217
}
62218

63219
main();

‎package-lock.json‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/config.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,14 @@ export const prebuiltAppConfig: AppConfig = {
359359
"low_resource_required": false,
360360
"required_features": ["shader-f16"],
361361
},
362+
{
363+
"model_url": "https://huggingface.co/mlc-ai/Hermes-2-Pro-Mistral-7B-q4f16_1-MLC/resolve/main/",
364+
"model_id": "Hermes-2-Pro-Mistral-7B-q4f16_1",
365+
"model_lib_url": modelLibURLPrefix + modelVersion + "/Hermes-2-Pro-Mistral-7B-q4f16_1-sw4k_cs1k-webgpu.wasm",
366+
"vram_required_MB": 4033.28,
367+
"low_resource_required": false,
368+
"required_features": ["shader-f16"],
369+
},
362370
// Gemma-2B
363371
{
364372
"model_url": "https://huggingface.co/mlc-ai/gemma-2b-it-q4f16_1-MLC/resolve/main/",

0 commit comments

Comments
 (0)