feat(client-bedrock): Automated Reasoning checks in Amazon Bedrock Guardrails now automatically generate Q&A tests for new Automated Reasoning policies. The GetAutomatedReasoningPolicyBuildWorkflowResultAssets API adds GENERATED_TEST_CASES asset type, allowing customers to retrieve tests generated by the build workflow.

Author: awstools

clients/client-bedrock/src/commands/GetAutomatedReasoningPolicyBuildWorkflowResultAssetsCommand.ts

@@ -45,7 +45,7 @@ export interface GetAutomatedReasoningPolicyBuildWorkflowResultAssetsCommandOutp
  * const input = { // GetAutomatedReasoningPolicyBuildWorkflowResultAssetsRequest
  *   policyArn: "STRING_VALUE", // required
  *   buildWorkflowId: "STRING_VALUE", // required
- *   assetType: "BUILD_LOG" || "QUALITY_REPORT" || "POLICY_DEFINITION", // required
+ *   assetType: "BUILD_LOG" || "QUALITY_REPORT" || "POLICY_DEFINITION" || "GENERATED_TEST_CASES", // required
  * };
  * const command = new GetAutomatedReasoningPolicyBuildWorkflowResultAssetsCommand(input);
  * const response = await client.send(command);
@@ -268,6 +268,15 @@ export interface GetAutomatedReasoningPolicyBuildWorkflowResultAssetsCommandOutp
  * //         },
  * //       ],
  * //     },
+ * //     generatedTestCases: { // AutomatedReasoningPolicyGeneratedTestCases
+ * //       generatedTestCases: [ // AutomatedReasoningPolicyGeneratedTestCaseList // required
+ * //         { // AutomatedReasoningPolicyGeneratedTestCase
+ * //           queryContent: "STRING_VALUE", // required
+ * //           guardContent: "STRING_VALUE", // required
+ * //           expectedAggregatedFindingsResult: "VALID" || "INVALID" || "SATISFIABLE" || "IMPOSSIBLE" || "TRANSLATION_AMBIGUOUS" || "TOO_COMPLEX" || "NO_TRANSLATION", // required
+ * //         },
+ * //       ],
+ * //     },
  * //   },
  * // };
  *

clients/client-bedrock/src/models/models_0.ts

@@ -1712,6 +1712,7 @@ export interface GetAutomatedReasoningPolicyBuildWorkflowResponse {
  */
 export const AutomatedReasoningPolicyBuildResultAssetType = {
   BUILD_LOG: "BUILD_LOG",
+  GENERATED_TEST_CASES: "GENERATED_TEST_CASES",
   POLICY_DEFINITION: "POLICY_DEFINITION",
   QUALITY_REPORT: "QUALITY_REPORT",
 } as const;
@@ -2297,6 +2298,42 @@ export interface AutomatedReasoningPolicyBuildLog {
   entries: AutomatedReasoningPolicyBuildLogEntry[] | undefined;
 }
 
+/**
+ * <p>Represents a generated test case, consisting of query content, guard content, and expected results.</p>
+ * @public
+ */
+export interface AutomatedReasoningPolicyGeneratedTestCase {
+  /**
+   * <p>The input query or prompt that generated the content. This provides context for the validation.</p>
+   * @public
+   */
+  queryContent: string | undefined;
+
+  /**
+   * <p>The output content that's validated by the Automated Reasoning policy. This represents the foundation model response that will be checked for accuracy.</p>
+   * @public
+   */
+  guardContent: string | undefined;
+
+  /**
+   * <p>The expected results of the generated test case. Possible values include:</p> <ul> <li> <p> <code>VALID</code> - The claims are true. The claims are implied by the premises and the Automated Reasoning policy. Given the Automated Reasoning policy and premises, it is not possible for these claims to be false. In other words, there are no alternative answers that are true that contradict the claims.</p> </li> <li> <p> <code>INVALID</code> - The claims are false. The claims are not implied by the premises and Automated Reasoning policy. Furthermore, there exists different claims that are consistent with the premises and Automated Reasoning policy.</p> </li> <li> <p> <code>SATISFIABLE</code> - The claims can be true or false. It depends on what assumptions are made for the claim to be implied from the premises and Automated Reasoning policy rules. In this situation, different assumptions can make input claims false and alternative claims true.</p> </li> <li> <p> <code>IMPOSSIBLE</code> - Automated Reasoning can’t make a statement about the claims. This can happen if the premises are logically incorrect, or if there is a conflict within the Automated Reasoning policy itself.</p> </li> </ul>
+   * @public
+   */
+  expectedAggregatedFindingsResult: AutomatedReasoningCheckResult | undefined;
+}
+
+/**
+ * <p>Contains a comprehensive test suite generated by the build workflow, providing validation capabilities for automated reasoning policies.</p>
+ * @public
+ */
+export interface AutomatedReasoningPolicyGeneratedTestCases {
+  /**
+   * <p>Represents a collection of generated test cases.</p>
+   * @public
+   */
+  generatedTestCases: AutomatedReasoningPolicyGeneratedTestCase[] | undefined;
+}
+
 /**
  * <p>Represents a set of rules that operate on completely separate variables, indicating they address different concerns or domains within the policy.</p>
  * @public
@@ -2388,11 +2425,12 @@ export interface AutomatedReasoningPolicyDefinitionQualityReport {
 }
 
 /**
- * <p>Contains the various assets generated during a policy build workflow, including logs, quality reports, and the final policy definition.</p>
+ * <p>Contains the various assets generated during a policy build workflow, including logs, quality reports, test cases, and the final policy definition.</p>
  * @public
  */
 export type AutomatedReasoningPolicyBuildResultAssets =
   | AutomatedReasoningPolicyBuildResultAssets.BuildLogMember
+  | AutomatedReasoningPolicyBuildResultAssets.GeneratedTestCasesMember
   | AutomatedReasoningPolicyBuildResultAssets.PolicyDefinitionMember
   | AutomatedReasoningPolicyBuildResultAssets.QualityReportMember
   | AutomatedReasoningPolicyBuildResultAssets.$UnknownMember;
@@ -2409,6 +2447,7 @@ export namespace AutomatedReasoningPolicyBuildResultAssets {
     policyDefinition: AutomatedReasoningPolicyDefinition;
     qualityReport?: never;
     buildLog?: never;
+    generatedTestCases?: never;
     $unknown?: never;
   }
 
@@ -2420,6 +2459,7 @@ export namespace AutomatedReasoningPolicyBuildResultAssets {
     policyDefinition?: never;
     qualityReport: AutomatedReasoningPolicyDefinitionQualityReport;
     buildLog?: never;
+    generatedTestCases?: never;
     $unknown?: never;
   }
 
@@ -2431,6 +2471,19 @@ export namespace AutomatedReasoningPolicyBuildResultAssets {
     policyDefinition?: never;
     qualityReport?: never;
     buildLog: AutomatedReasoningPolicyBuildLog;
+    generatedTestCases?: never;
+    $unknown?: never;
+  }
+
+  /**
+   * <p>A comprehensive test suite generated by the build workflow, providing validation capabilities for automated reasoning policies.</p>
+   * @public
+   */
+  export interface GeneratedTestCasesMember {
+    policyDefinition?: never;
+    qualityReport?: never;
+    buildLog?: never;
+    generatedTestCases: AutomatedReasoningPolicyGeneratedTestCases;
     $unknown?: never;
   }
 
@@ -2441,6 +2494,7 @@ export namespace AutomatedReasoningPolicyBuildResultAssets {
     policyDefinition?: never;
     qualityReport?: never;
     buildLog?: never;
+    generatedTestCases?: never;
     $unknown: [string, any];
   }
 
@@ -2452,6 +2506,7 @@ export namespace AutomatedReasoningPolicyBuildResultAssets {
     policyDefinition: (value: AutomatedReasoningPolicyDefinition) => T;
     qualityReport: (value: AutomatedReasoningPolicyDefinitionQualityReport) => T;
     buildLog: (value: AutomatedReasoningPolicyBuildLog) => T;
+    generatedTestCases: (value: AutomatedReasoningPolicyGeneratedTestCases) => T;
     _: (name: string, value: any) => T;
   }
 }
@@ -7409,35 +7464,3 @@ export interface GuardrailTopicConfig {
    */
   outputEnabled?: boolean | undefined;
 }
-
-/**
- * <p>Contains details about topics that the guardrail should identify and deny.</p>
- * @public
- */
-export interface GuardrailTopicPolicyConfig {
-  /**
-   * <p>A list of policies related to topics that the guardrail should deny.</p>
-   * @public
-   */
-  topicsConfig: GuardrailTopicConfig[] | undefined;
-
-  /**
-   * <p>The tier that your guardrail uses for denied topic filters.</p>
-   * @public
-   */
-  tierConfig?: GuardrailTopicsTierConfig | undefined;
-}
-
-/**
- * @public
- * @enum
- */
-export const GuardrailWordAction = {
-  BLOCK: "BLOCK",
-  NONE: "NONE",
-} as const;
-
-/**
- * @public
- */
-export type GuardrailWordAction = (typeof GuardrailWordAction)[keyof typeof GuardrailWordAction];