From 90cfb2ce3d33801c3a4862c359e486cab43aa2fa Mon Sep 17 00:00:00 2001 From: Aaryan Divate Date: Wed, 17 Sep 2025 12:07:27 -0700 Subject: [PATCH] Add Prompt dataclass with initial methods --- scripts/api_generator.py | 2 + scripts/openapi_transform.py | 2 + src/judgeval/api/__init__.py | 46 +++++++++ src/judgeval/api/api_types.py | 22 ++++- src/judgeval/data/judgment_types.py | 22 ++++- src/judgeval/prompts/prompt.py | 96 +++++++++++++++++++ .../api_scorers/prompt_scorer.py | 12 --- 7 files changed, 188 insertions(+), 14 deletions(-) create mode 100644 src/judgeval/prompts/prompt.py diff --git a/scripts/api_generator.py b/scripts/api_generator.py index 50f084be..d328edcd 100644 --- a/scripts/api_generator.py +++ b/scripts/api_generator.py @@ -44,6 +44,8 @@ "/e2e_fetch_trace/", "/e2e_fetch_span_score/", "/e2e_fetch_trace_scorer_span_score/", + "/prompts/insert/", + "/prompts/fetch/", ] diff --git a/scripts/openapi_transform.py b/scripts/openapi_transform.py index 2e938422..5993c68a 100644 --- a/scripts/openapi_transform.py +++ b/scripts/openapi_transform.py @@ -42,6 +42,8 @@ "/projects/resolve/", "/e2e_fetch_trace/", "/e2e_fetch_span_score/", + "/prompts/insert/", + "/prompts/fetch/", ] diff --git a/src/judgeval/api/__init__.py b/src/judgeval/api/__init__.py index ebc31ea5..2e02cf73 100644 --- a/src/judgeval/api/__init__.py +++ b/src/judgeval/api/__init__.py @@ -199,6 +199,28 @@ def upload_custom_scorer( payload, ) + def prompts_insert(self, payload: PromptInsertRequest) -> PromptInsertResponse: + return self._request( + "POST", + url_for("/prompts/insert/"), + payload, + ) + + def prompts_fetch( + self, name: str, commit_id: Optional[str] = None, tag: Optional[str] = None + ) -> PromptFetchResponse: + query_params = {} + query_params["name"] = name + if commit_id is not None: + query_params["commit_id"] = commit_id + if tag is not None: + query_params["tag"] = tag + return self._request( + "GET", + url_for("/prompts/fetch/"), + query_params, + ) + def projects_resolve( self, payload: ResolveProjectNameRequest ) -> ResolveProjectNameResponse: @@ -410,6 +432,30 @@ async def upload_custom_scorer( payload, ) + async def prompts_insert( + self, payload: PromptInsertRequest + ) -> PromptInsertResponse: + return await self._request( + "POST", + url_for("/prompts/insert/"), + payload, + ) + + async def prompts_fetch( + self, name: str, commit_id: Optional[str] = None, tag: Optional[str] = None + ) -> PromptFetchResponse: + query_params = {} + query_params["name"] = name + if commit_id is not None: + query_params["commit_id"] = commit_id + if tag is not None: + query_params["tag"] = tag + return await self._request( + "GET", + url_for("/prompts/fetch/"), + query_params, + ) + async def projects_resolve( self, payload: ResolveProjectNameRequest ) -> ResolveProjectNameResponse: diff --git a/src/judgeval/api/api_types.py b/src/judgeval/api/api_types.py index 262110a6..7c83d1f1 100644 --- a/src/judgeval/api/api_types.py +++ b/src/judgeval/api/api_types.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: .openapi.json -# timestamp: 2025-09-12T16:54:35+00:00 +# timestamp: 2025-09-17T19:04:58+00:00 from __future__ import annotations from typing import Any, Dict, List, Literal, Optional, TypedDict, Union @@ -77,6 +77,26 @@ class CustomScorerTemplateResponse(TypedDict): message: str +class PromptInsertRequest(TypedDict): + name: str + prompt: str + tags: List[str] + + +class PromptInsertResponse(TypedDict): + commit_id: str + parent_commit_id: NotRequired[Optional[str]] + + +class PromptFetchResponse(TypedDict): + name: str + prompt: str + tags: List[str] + commit_id: str + parent_commit_id: NotRequired[Optional[str]] + created_at: str + + class ResolveProjectNameRequest(TypedDict): project_name: str diff --git a/src/judgeval/data/judgment_types.py b/src/judgeval/data/judgment_types.py index 0f8db698..86922971 100644 --- a/src/judgeval/data/judgment_types.py +++ b/src/judgeval/data/judgment_types.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: .openapi.json -# timestamp: 2025-09-12T16:54:34+00:00 +# timestamp: 2025-09-17T19:04:57+00:00 from __future__ import annotations from typing import Annotated, Any, Dict, List, Optional, Union @@ -79,6 +79,26 @@ class CustomScorerTemplateResponse(BaseModel): message: Annotated[str, Field(title="Message")] +class PromptInsertRequest(BaseModel): + name: Annotated[str, Field(title="Name")] + prompt: Annotated[str, Field(title="Prompt")] + tags: Annotated[List[str], Field(title="Tags")] + + +class PromptInsertResponse(BaseModel): + commit_id: Annotated[str, Field(title="Commit Id")] + parent_commit_id: Annotated[Optional[str], Field(title="Parent Commit Id")] = None + + +class PromptFetchResponse(BaseModel): + name: Annotated[str, Field(title="Name")] + prompt: Annotated[str, Field(title="Prompt")] + tags: Annotated[List[str], Field(title="Tags")] + commit_id: Annotated[str, Field(title="Commit Id")] + parent_commit_id: Annotated[Optional[str], Field(title="Parent Commit Id")] = None + created_at: Annotated[str, Field(title="Created At")] + + class ResolveProjectNameRequest(BaseModel): project_name: Annotated[str, Field(title="Project Name")] diff --git a/src/judgeval/prompts/prompt.py b/src/judgeval/prompts/prompt.py new file mode 100644 index 00000000..d0d893ae --- /dev/null +++ b/src/judgeval/prompts/prompt.py @@ -0,0 +1,96 @@ +from typing import List, Optional +import os +from judgeval.api import JudgmentSyncClient +from judgeval.exceptions import JudgmentAPIError +from dataclasses import dataclass, field +import re +from string import Template + + +def push_prompt( + name: str, + prompt: str, + tags: List[str], + judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "", + organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "", +) -> tuple[str, Optional[str]]: + client = JudgmentSyncClient(judgment_api_key, organization_id) + try: + r = client.prompts_insert( + payload={"name": name, "prompt": prompt, "tags": tags} + ) + return r["commit_id"], r["parent_commit_id"] + except JudgmentAPIError as e: + raise JudgmentAPIError( + status_code=e.status_code, + detail=f"Failed to save prompt: {e.detail}", + response=e.response, + ) + + +def fetch_prompt( + name: str, + commit_id: Optional[str] = None, + tag: Optional[str] = None, + judgment_api_key: str = os.getenv("JUDGMENT_API_KEY") or "", + organization_id: str = os.getenv("JUDGMENT_ORG_ID") or "", +): + client = JudgmentSyncClient(judgment_api_key, organization_id) + try: + prompt_config = client.prompts_fetch(name, commit_id, tag) + return prompt_config + except JudgmentAPIError as e: + raise JudgmentAPIError( + status_code=e.status_code, + detail=f"Failed to fetch prompt '{name}': {e.detail}", + response=e.response, + ) + + +@dataclass +class Prompt: + name: str + prompt: str + tags: List[str] + commit_id: str + parent_commit_id: Optional[str] = None + _template: Template = field(init=False, repr=False) + + def __post_init__(self): + template_str = re.sub(r"\{\{(\w+)\}\}", r"$\1", self.prompt) + self._template = Template(template_str) + + @classmethod + def create(cls, name: str, prompt: str, tags: Optional[List[str]] = None): + if not tags: + tags = [] + commit_id, parent_commit_id = push_prompt(name, prompt, tags) + return cls( + name=name, + prompt=prompt, + tags=tags, + commit_id=commit_id, + parent_commit_id=parent_commit_id, + ) + + @classmethod + def get(cls, name: str, commit_id: Optional[str] = None, tag: Optional[str] = None): + if commit_id is not None and tag is not None: + raise ValueError( + "You cannot fetch a prompt by both commit_id and tag at the same time" + ) + prompt_config = fetch_prompt(name, commit_id, tag) + return cls( + name=prompt_config["name"], + prompt=prompt_config["prompt"], + tags=prompt_config["tags"], + commit_id=prompt_config["commit_id"], + parent_commit_id=prompt_config["parent_commit_id"], + ) + + def compile(self, **kwargs) -> str: + try: + return self._template.substitute(**kwargs) + except KeyError as e: + missing_var = str(e).strip("'") + raise ValueError(f"Missing required variable: {missing_var}") diff --git a/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py b/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py index 060d1263..e8869a2c 100644 --- a/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py +++ b/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py @@ -32,12 +32,6 @@ def push_prompt_scorer( } ) except JudgmentAPIError as e: - if e.status_code == 500: - raise JudgmentAPIError( - status_code=e.status_code, - detail=f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.detail}", - response=e.response, - ) raise JudgmentAPIError( status_code=e.status_code, detail=f"Failed to save prompt scorer: {e.detail}", @@ -58,12 +52,6 @@ def fetch_prompt_scorer( scorer_config.pop("updated_at") return scorer_config except JudgmentAPIError as e: - if e.status_code == 500: - raise JudgmentAPIError( - status_code=e.status_code, - detail=f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.detail}", - response=e.response, - ) raise JudgmentAPIError( status_code=e.status_code, detail=f"Failed to fetch prompt scorer '{name}': {e.detail}",