From 09cdc526de6ee037ab0a406f8075ef061da447ff Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 24 Jun 2025 12:40:04 +1000 Subject: [PATCH 1/7] wip: working on env egress schema --- package-lock.json | 131 ++++++++++++++++++++++++++++++++++++++++------ package.json | 2 + 2 files changed, 117 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 76e720dc..3b259313 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,8 @@ "@typescript-eslint/eslint-plugin": "^5.61.0", "@typescript-eslint/parser": "^5.61.0", "@yao-pkg/pkg": "^6.4.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", "commander": "^8.3.0", "cross-env": "^7.0.3", "esbuild": "0.19.4", @@ -1206,6 +1208,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "peer": true + }, "node_modules/@eslint/js": { "version": "8.51.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.51.0.tgz", @@ -2374,6 +2400,22 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@matrixai/lint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/@matrixai/lint/node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -2555,6 +2597,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@matrixai/lint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/@matrixai/logger": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/@matrixai/logger/-/logger-4.0.3.tgz", @@ -3621,21 +3669,38 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -6169,6 +6234,23 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", @@ -6196,6 +6278,13 @@ "node": ">=4.0" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "peer": true + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -6443,6 +6532,22 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ] + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -8493,9 +8598,9 @@ "dev": true }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true }, "node_modules/json-stable-stringify-without-jsonify": { @@ -9760,12 +9865,6 @@ "integrity": "sha512-0CNTVCLZggSh7bc5VkX5WWPWO+cyZbNd07IHIsSXLia/eAq+r836hgk+8BKoEh7949Mda87VUOitx5OddVj64A==", "dev": true }, - "node_modules/polykey/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/polykey/node_modules/minimatch": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", diff --git a/package.json b/package.json index f2249501..2feaec9b 100644 --- a/package.json +++ b/package.json @@ -149,6 +149,8 @@ "@typescript-eslint/eslint-plugin": "^5.61.0", "@typescript-eslint/parser": "^5.61.0", "@yao-pkg/pkg": "^6.4.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", "commander": "^8.3.0", "cross-env": "^7.0.3", "esbuild": "0.19.4", From b3c6f90c1e3500a1d321b15ddc5f570c55f1c23f Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Mon, 30 Jun 2025 17:50:17 +1000 Subject: [PATCH 2/7] feat: compositional env validation --- npmDepsHash | 2 +- package-lock.json | 24 ++++++++++++++++++ package.json | 1 + ref.schema.json | 14 +++++++++++ src/secrets/CommandEnv.ts | 51 +++++++++++++++++++++++++++++++++++++-- src/types.ts | 29 ++++++++++++++++++++++ src/utils/options.ts | 6 +++++ src/utils/utils.ts | 44 +++++++++++++++++++++++++++++++++ test.schema.json | 11 +++++++++ 9 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 ref.schema.json create mode 100644 test.schema.json diff --git a/npmDepsHash b/npmDepsHash index c9e52b67..47abbf8a 100644 --- a/npmDepsHash +++ b/npmDepsHash @@ -1 +1 @@ -sha256-/SFcicpWX1SFllzmz0acUYxtIV0LuBw1PU32Uk5hLzw= +sha256-5lr9ukYvA7s8sAgnX5rN6ef12tOb9lPwAHGlOrD0k+w= diff --git a/package-lock.json b/package-lock.json index 3b259313..c80a95d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "polykey": "dist/polykey.mjs" }, "devDependencies": { + "@apidevtools/json-schema-ref-parser": "^11.9.3", "@fast-check/jest": "^2.1.1", "@matrixai/errors": "^2.1.3", "@matrixai/exec": "^1.0.3", @@ -95,6 +96,23 @@ "node": ">=6.0.0" } }, + "node_modules/@apidevtools/json-schema-ref-parser": { + "version": "11.9.3", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.9.3.tgz", + "integrity": "sha512-60vepv88RwcJtSHrD6MjIL6Ta3SOYbgfnkHb+ppAVK+o9mXprRtulx7VlRl3lN3bbvysAfCS7WMVfhUYemB0IQ==", + "dev": true, + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.15", + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/philsturgeon" + } + }, "node_modules/@babel/code-frame": { "version": "7.22.13", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", @@ -1965,6 +1983,12 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", + "dev": true + }, "node_modules/@matrixai/async-cancellable": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@matrixai/async-cancellable/-/async-cancellable-2.0.1.tgz", diff --git a/package.json b/package.json index 2feaec9b..4201dced 100644 --- a/package.json +++ b/package.json @@ -136,6 +136,7 @@ "sodium-native": "*" }, "devDependencies": { + "@apidevtools/json-schema-ref-parser": "^11.9.3", "@fast-check/jest": "^2.1.1", "@matrixai/errors": "^2.1.3", "@matrixai/exec": "^1.0.3", diff --git a/ref.schema.json b/ref.schema.json new file mode 100644 index 00000000..d4c3987e --- /dev/null +++ b/ref.schema.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "allOf": [ + { "$ref": "./test.schema.json"} + ], + "properties": { + "SECRET2": { + "type": "string" + } + }, + "required": [ + "SECRET2" + ] +} \ No newline at end of file diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index 0bc041b2..c86b7fed 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -1,7 +1,9 @@ import type PolykeyClient from 'polykey/PolykeyClient.js'; -import type { ParsedSecretPathValue } from '../types.js'; +import type { JSONSchema, ParsedSecretPathValue } from '../types.js'; import path from 'node:path'; import os from 'node:os'; +import $RefParser from '@apidevtools/json-schema-ref-parser'; +import { Ajv2019 as Ajv } from 'ajv/dist/2019.js'; import { InvalidArgumentError } from 'commander'; import * as utils from 'polykey/utils/index.js'; import CommandPolykey from '../CommandPolykey.js'; @@ -26,6 +28,7 @@ class CommandEnv extends CommandPolykey { this.addOption(binOptions.envDuplicate); this.addOption(binOptions.envExport); this.addOption(binOptions.preserveNewline); + this.addOption(binOptions.egressSchema); this.argument( '', 'command and arguments formatted as [-- cmd [cmdArgs...]]', @@ -53,6 +56,7 @@ class CommandEnv extends CommandPolykey { if (secretPath == null) preservedSecrets.add(vaultName); else preservedSecrets.add(`${vaultName}:${secretPath}`); } + // There are a few stages here // 1. parse the desired secrets // 2. obtain the desired secrets @@ -226,7 +230,50 @@ class CommandEnv extends CommandPolykey { }; } await writeP; - return [envp, envpPath]; + + // Apply validation using the schema + // TODO: filter before pulling instead of after + const filteredEnvp: Record = {}; + if (options.egressSchema != null) { + // Resolve references and bundle schema + const schema: JSONSchema = await $RefParser.bundle( + options.egressSchema, + ); + + // Validate the incoming secrets against the schema + const ajv = new Ajv({ + coerceTypes: true, + useDefaults: false, + allErrors: true, + }); + const validate = ajv.compile(schema); + validate(envp); + + // Extract relevant keys, discarding the rest + const { requiredKeys, allKeys, defaults } = + binUtils.loadSchema(schema); + + for (const key of allKeys) { + let value = envp[key]; + if (value == null && defaults[key] != null) { + value = defaults[key]; + } + if ( + requiredKeys.includes(key) && + (value == null || value === '') + ) { + throw new Error('TMP missing required variable'); + } + if (value != null) { + filteredEnvp[key] = value.toString(); + } + } + } + + return [ + utils.isEmptyObject(filteredEnvp) ? envp : filteredEnvp, + envpPath, + ]; }, meta); // End connection early to avoid errors on server await pkClient.stop(); diff --git a/src/types.ts b/src/types.ts index acf2bdea..40d1e68c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -63,6 +63,32 @@ type PromiseDeconstructed = { type ParsedSecretPathValue = [string, string?, string?]; +type JSONSchemaProps = Record< + string, + { + type?: string; + default?: unknown; + [key: string]: unknown; + } +>; + +type JSONSchema = { + type?: string; + properties?: Record; + required?: Array; + allOf?: Array; + anyOf?: Array; + oneOf?: Array; + [key: string]: unknown; +}; + +// Only strings are supported for the time being +type JSONSchemaInfo = { + allKeys: Array; + requiredKeys: Array; + defaults: Record; +}; + export type { TableRow, TableOptions, @@ -72,4 +98,7 @@ export type { AgentChildProcessOutput, PromiseDeconstructed, ParsedSecretPathValue, + JSONSchemaProps, + JSONSchema, + JSONSchemaInfo, }; diff --git a/src/utils/options.ts b/src/utils/options.ts index 1146c154..0347606f 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -325,6 +325,11 @@ const returnURLPath = new Option( 'Which path on the website to send the token to', ); +const egressSchema = new Option( + '--egress-schema ', + 'A JSON schema controlling the egressing secrets', +); + export { nodePath, format, @@ -371,4 +376,5 @@ export { parents, preserveNewline, returnURLPath, + egressSchema, }; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 882a2f93..3d562171 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -5,6 +5,8 @@ import type { TableOptions, DictOptions, PromiseDeconstructed, + JSONSchema, + JSONSchemaInfo, } from '../types.js'; import process from 'node:process'; import { LogLevel } from '@matrixai/logger'; @@ -646,6 +648,47 @@ function jsonToCompactJWT(token: SignedTokenEncoded): string { return `${token.signatures[0].protected}.${token.payload}.${token.signatures[0].signature}`; } +function loadSchema(bundledSchema: JSONSchema): JSONSchemaInfo { + const props: Set = new Set(); + const required: Set = new Set(); + const defaults: Record = {}; + + const unwrapSchema = (schema: JSONSchema) => { + // Collect properties and their defaults + if (schema.properties != null) { + for (const [k, p] of Object.entries(schema.properties)) { + props.add(k); + if (p.default == null) continue; + if (typeof p.default !== 'string') { + throw new Error('TMP wrong type'); + } + defaults[k] = p.default; + } + } + + // Collect required properties + if (schema.required != null) { + schema.required.forEach((requiredSecret) => required.add(requiredSecret)); + } + + // Process composition keywords + const compositionKeywords = ['allOf', 'anyOf', 'oneOf'] as const; + compositionKeywords.forEach( + (keyword) => + schema[keyword] != null && + Array.isArray(schema[keyword]) && + schema[keyword]!.forEach(unwrapSchema), + ); + }; + unwrapSchema(bundledSchema); + + return { + allKeys: [...props], + requiredKeys: [...required], + defaults: defaults, + }; +} + export { verboseToLogLevel, standardErrorReplacer, @@ -670,6 +713,7 @@ export { promise, importFS, jsonToCompactJWT, + loadSchema, }; export type { OutputObject }; diff --git a/test.schema.json b/test.schema.json new file mode 100644 index 00000000..e2279068 --- /dev/null +++ b/test.schema.json @@ -0,0 +1,11 @@ +{ + "type": "object", + "properties": { + "SECRET1": { + "type": "string" + } + }, + "required": [ + "SECRET1" + ] +} \ No newline at end of file From 9667a14461e5eb31909c2337070343bc632a1bca Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 1 Jul 2025 17:46:02 +1000 Subject: [PATCH 3/7] wip: working on applying schema before requesting [ci skip] --- src/secrets/CommandEnv.ts | 121 ++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 32 deletions(-) diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index c86b7fed..44f92ed5 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -1,11 +1,14 @@ import type PolykeyClient from 'polykey/PolykeyClient.js'; -import type { JSONSchema, ParsedSecretPathValue } from '../types.js'; +import type { + JSONSchema, + JSONSchemaInfo, + ParsedSecretPathValue, +} from '../types.js'; import path from 'node:path'; import os from 'node:os'; import $RefParser from '@apidevtools/json-schema-ref-parser'; import { Ajv2019 as Ajv } from 'ajv/dist/2019.js'; import { InvalidArgumentError } from 'commander'; -import * as utils from 'polykey/utils/index.js'; import CommandPolykey from '../CommandPolykey.js'; import * as binProcessors from '../utils/processors.js'; import * as binUtils from '../utils/index.js'; @@ -37,6 +40,7 @@ class CommandEnv extends CommandPolykey { const { default: PolykeyClient } = await import( 'polykey/PolykeyClient.js' ); + const utils = await import('polykey/utils/index.js'); const { envInvalid, envDuplicate, @@ -122,27 +126,66 @@ class CommandEnv extends CommandPolykey { logger: this.logger.getChild(PolykeyClient.name), }); + let schema: JSONSchema | undefined = undefined; + let unwrappedSchema: JSONSchemaInfo | undefined = undefined; + if (options.egressSchema != null) { + schema = (await $RefParser.bundle( + options.egressSchema, + )) satisfies JSONSchema; + unwrappedSchema = binUtils.loadSchema(schema!); + } + // Getting envs const [envp] = await binUtils.retryAuthentication(async (auth) => { const responseStream = await pkClient.rpcClient.methods.vaultsSecretsEnv(); + // Writing desired secrets const secretRenameMap = new Map(); - const writeP = (async () => { - const writer = responseStream.writable.getWriter(); - let first = true; - for (const envVariable of envVariables) { - const [nameOrId, secretName, secretNameNew] = envVariable; - secretRenameMap.set(secretName ?? '/', secretNameNew); + const writer = responseStream.writable.getWriter(); + let first = true; + for (const envVariable of envVariables) { + const [nameOrId, secretName, secretNameNew] = envVariable; + secretRenameMap.set(secretName ?? '/', secretNameNew); + + // If there is no secret name provided, then attempt to export the + // secrets from the entire vault. Otherwise, check if the selected + // secret exists in the schema before requesting it. This will + // only run if a schema has been specified. + if (schema != null && unwrappedSchema != null) { + const { allKeys } = unwrappedSchema; + if (nameOrId != null && secretName == null) { + // Only vault specified + for (const key of allKeys) { + // TODO: handle secret renames, allKeys key might not be the same in vault + await writer.write({ + nameOrId: nameOrId, + secretName: key, + metadata: first ? auth : undefined, + }); + } + } else { + // Individual secret name specified + const name: string = secretNameNew != null ? secretNameNew : secretName!; + if (allKeys.includes(name)) { + await writer.write({ + nameOrId: nameOrId, + secretName: name, + metadata: first ? auth : undefined, + }); + } + } + } else { + // No schema specified await writer.write({ nameOrId: nameOrId, secretName: secretName ?? '/', metadata: first ? auth : undefined, }); - first = false; } - await writer.close(); - })(); + first = false; + } + await writer.close(); const envp: Record = {}; const envpPath: Record< @@ -153,6 +196,34 @@ class CommandEnv extends CommandPolykey { } > = {}; for await (const value of responseStream.readable) { + if (value.type === 'ErrorMessage') { + switch (value.code) { + case 'EINVAL': + // It is expected for the data to be populated with the offending + // vault name if the vault was not found. + process.stderr.write( + binUtils.outputFormatterError( + `Vault "${value.data?.nameOrId}" does not exist`, + ), + ); + break; + case 'ENOENT': + // It is expected for the data to be populated with the offending + // secret and vault name if a secret was not found. + process.stderr.write( + binUtils.outputFormatterError( + `Secret "${value.data?.secretName}" does not exist in vault "${value.data?.nameOrId}"`, + ), + ); + break; + default: + utils.never( + `Expected code to be one of EINVAL, ENOENT, received ${value.code}`, + ); + } + continue; + } + const { nameOrId, secretName, secretContent } = value; let newName = secretRenameMap.get(secretName); if (newName == null) { @@ -229,30 +300,16 @@ class CommandEnv extends CommandPolykey { secretName, }; } - await writeP; - // Apply validation using the schema - // TODO: filter before pulling instead of after + // Apply defaults using the schema const filteredEnvp: Record = {}; - if (options.egressSchema != null) { - // Resolve references and bundle schema - const schema: JSONSchema = await $RefParser.bundle( - options.egressSchema, - ); - - // Validate the incoming secrets against the schema - const ajv = new Ajv({ - coerceTypes: true, - useDefaults: false, - allErrors: true, - }); - const validate = ajv.compile(schema); - validate(envp); - - // Extract relevant keys, discarding the rest - const { requiredKeys, allKeys, defaults } = - binUtils.loadSchema(schema); + if (unwrappedSchema != null) { + // Parse the schema for manual filtering + const { requiredKeys, allKeys, defaults } = unwrappedSchema; + // Add allowed secrets to a filtered set of secrets. This runs after + // the duplication is processed, so all secrets here are guaranteed + // to be unique. for (const key of allKeys) { let value = envp[key]; if (value == null && defaults[key] != null) { From f3520f119cecde1041dab934dc4f3ce6002bab98 Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 1 Jul 2025 18:00:31 +1000 Subject: [PATCH 4/7] wip: finalising and testing changes [ci skip] --- src/secrets/CommandEnv.ts | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index 44f92ed5..4ced8fbd 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -157,7 +157,8 @@ class CommandEnv extends CommandPolykey { if (nameOrId != null && secretName == null) { // Only vault specified for (const key of allKeys) { - // TODO: handle secret renames, allKeys key might not be the same in vault + // When exporting secrets from a vault, it is impossible to + // rename the resulting secrets. await writer.write({ nameOrId: nameOrId, secretName: key, @@ -166,7 +167,8 @@ class CommandEnv extends CommandPolykey { } } else { // Individual secret name specified - const name: string = secretNameNew != null ? secretNameNew : secretName!; + const name: string = + secretNameNew != null ? secretNameNew : secretName!; if (allKeys.includes(name)) { await writer.write({ nameOrId: nameOrId, @@ -201,21 +203,26 @@ class CommandEnv extends CommandPolykey { case 'EINVAL': // It is expected for the data to be populated with the offending // vault name if the vault was not found. - process.stderr.write( - binUtils.outputFormatterError( - `Vault "${value.data?.nameOrId}" does not exist`, - ), + throw new Error( + `TMP Vault "${value.data?.nameOrId}" does not exist`, ); - break; case 'ENOENT': + // If we have a default for this key, then don't bother + // reporting the missing key. + if ( + unwrappedSchema != null && + Object.keys(unwrappedSchema.defaults).includes( + value.data!.secretName!.toString(), + ) + ) { + break; + } + // It is expected for the data to be populated with the offending // secret and vault name if a secret was not found. - process.stderr.write( - binUtils.outputFormatterError( - `Secret "${value.data?.secretName}" does not exist in vault "${value.data?.nameOrId}"`, - ), + throw new Error( + `TMP Secret "${value.data?.secretName}" does not exist in vault "${value.data?.nameOrId}"`, ); - break; default: utils.never( `Expected code to be one of EINVAL, ENOENT, received ${value.code}`, From e9abc7870bcb999401dc286b0b61cf915d2aafcf Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Wed, 2 Jul 2025 16:39:10 +1000 Subject: [PATCH 5/7] feat: added schema compositionality and tests [ci skip] --- src/errors.ts | 6 + src/secrets/CommandEnv.ts | 35 ++--- src/utils/utils.ts | 4 +- tests/secrets/env.test.ts | 286 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 313 insertions(+), 18 deletions(-) diff --git a/src/errors.ts b/src/errors.ts index e44809dd..2c5201ad 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -166,6 +166,11 @@ class ErrorPolykeyCLIDuplicateEnvName extends ErrorPolykeyCLI { exitCode = sysexits.USAGE; } +class ErrorPolykeyCLIMissingRequiredEnvName extends ErrorPolykeyCLI { + static description = 'A required environment variable is not present'; + exitCode = sysexits.USAGE; +} + class ErrorPolykeyCLIMakeDirectory extends ErrorPolykeyCLI { static description = 'Failed to create one or more directories'; exitCode = 1; @@ -223,6 +228,7 @@ export { ErrorPolykeyCLINodePingFailed, ErrorPolykeyCLIInvalidEnvName, ErrorPolykeyCLIDuplicateEnvName, + ErrorPolykeyCLIMissingRequiredEnvName, ErrorPolykeyCLIMakeDirectory, ErrorPolykeyCLIRenameSecret, ErrorPolykeyCLIRemoveSecret, diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index 4ced8fbd..9064d258 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -172,7 +172,7 @@ class CommandEnv extends CommandPolykey { if (allKeys.includes(name)) { await writer.write({ nameOrId: nameOrId, - secretName: name, + secretName: secretName!, metadata: first ? auth : undefined, }); } @@ -201,25 +201,18 @@ class CommandEnv extends CommandPolykey { if (value.type === 'ErrorMessage') { switch (value.code) { case 'EINVAL': - // It is expected for the data to be populated with the offending - // vault name if the vault was not found. + // It is expected for the data to be populated with the + // offending vault name if the vault was not found. throw new Error( `TMP Vault "${value.data?.nameOrId}" does not exist`, ); case 'ENOENT': - // If we have a default for this key, then don't bother - // reporting the missing key. - if ( - unwrappedSchema != null && - Object.keys(unwrappedSchema.defaults).includes( - value.data!.secretName!.toString(), - ) - ) { - break; - } + // If we are working with schemas, then missing keys will be + // validated later. + if (unwrappedSchema != null) break; - // It is expected for the data to be populated with the offending - // secret and vault name if a secret was not found. + // It is expected for the data to be populated with the + // offending secret and vault name if a secret was not found. throw new Error( `TMP Secret "${value.data?.secretName}" does not exist in vault "${value.data?.nameOrId}"`, ); @@ -310,7 +303,7 @@ class CommandEnv extends CommandPolykey { // Apply defaults using the schema const filteredEnvp: Record = {}; - if (unwrappedSchema != null) { + if (schema != null && unwrappedSchema != null) { // Parse the schema for manual filtering const { requiredKeys, allKeys, defaults } = unwrappedSchema; @@ -326,12 +319,20 @@ class CommandEnv extends CommandPolykey { requiredKeys.includes(key) && (value == null || value === '') ) { - throw new Error('TMP missing required variable'); + throw new binErrors.ErrorPolykeyCLIMissingRequiredEnvName( + `Expected definition for ${key}`, + ); } if (value != null) { filteredEnvp[key] = value.toString(); } } + + // Validate the schema using ajv. All defaults have already been + // applied. This is now the final state of the exported variables. + const ajv = new Ajv({ allErrors: true }); + const validate = ajv.compile(schema); + validate(envp); } return [ diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 3d562171..09b13deb 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -457,7 +457,9 @@ function outputFormatterError(err: any): string { output += '\n'; } } - output += `${indent}cause: `; + if (err.cause) { + output += `${indent}cause: `; + } err = err.cause; } else if (err instanceof ErrorPolykey) { output += `${err.name}: ${err.description}`; diff --git a/tests/secrets/env.test.ts b/tests/secrets/env.test.ts index 21cbd0d0..7a43582a 100644 --- a/tests/secrets/env.test.ts +++ b/tests/secrets/env.test.ts @@ -953,4 +953,290 @@ describe('commandEnv', () => { expect(result.stdout).toEqual(formatResult[format]); }, ); + + describe('should apply json schema to control secrets egress', () => { + test('should filter secrets based on a schema', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + const secretName2 = 'SECRET2'; + const secretName3 = 'SECRET3'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + await vaultOps.addSecret(vault, secretName2, secretName2); + await vaultOps.addSecret(vault, secretName3, secretName3); + }); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'string' }, + }, + required: ['SECRET1', 'SECRET2'], + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(0); + + // Confirm only the specified secrets were exported + expect(result.stdout).toContain("SECRET1='SECRET1'"); + expect(result.stdout).toContain("SECRET2='SECRET2'"); + expect(result.stdout).not.toContain("SECRET3='SECRET3'"); + }); + + test('should apply schema to secrets from multiple vaults', async () => { + // Write secrets to vault + const vaultName1 = 'vault1'; + const vaultName2 = 'vault2'; + const vaultId1 = await polykeyAgent.vaultManager.createVault(vaultName1); + const vaultId2 = await polykeyAgent.vaultManager.createVault(vaultName2); + const secretName1 = 'SECRET1'; + const secretName2 = 'SECRET2'; + const secretName3 = 'SECRET3'; + await polykeyAgent.vaultManager.withVaults( + [vaultId1, vaultId2], + async (vault1, vault2) => { + await vaultOps.addSecret(vault1, secretName1, secretName1); + await vaultOps.addSecret(vault2, secretName2, secretName2); + await vaultOps.addSecret(vault1, secretName3, secretName3); + }, + ); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'string' }, + }, + required: ['SECRET1', 'SECRET2'], + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName1, + vaultName2, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(0); + + // Confirm only the specified secrets were exported + expect(result.stdout).toContain("SECRET1='SECRET1'"); + expect(result.stdout).toContain("SECRET2='SECRET2'"); + expect(result.stdout).not.toContain("SECRET3='SECRET3'"); + }); + + test('should handle secret renames', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + const secretName2 = 'SECRET2'; + const secretName3 = 'SECRET3'; + const secretRename = 'RENAMED'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + await vaultOps.addSecret(vault, secretName2, secretName2); + await vaultOps.addSecret(vault, secretName3, secretName3); + }); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + RENAMED: { type: 'string' }, + }, + required: ['SECRET1', 'RENAMED'], + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema. Should first export all relevant secrets + // from the vault, then process the renamed secret. + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName, + `${vaultName}:${secretName2}=${secretRename}`, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(0); + + // Confirm only the specified secrets were exported + expect(result.stdout).toContain("SECRET1='SECRET1'"); + expect(result.stdout).toContain("RENAMED='SECRET2'"); + expect(result.stdout).not.toContain("SECRET2='SECRET2'"); + expect(result.stdout).not.toContain("SECRET3='SECRET3'"); + }); + + test('should not fail when missing non-required secret', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + }); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'string' }, + }, + required: ['SECRET1'], + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema. Should first export all relevant secrets + // from the vault, then process the renamed secret. + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(0); + + // Confirm only the specified secrets were exported + expect(result.stdout).toContain("SECRET1='SECRET1'"); + expect(result.stdout).not.toContain('SECRET2'); + }); + + test('should fail when missing required secret', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + }); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'string' }, + }, + required: ['SECRET1', 'SECRET2'], + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(64); + + // Confirm the validity of the error + expect(result.stderr).toInclude('ErrorPolykeyCLIMissingRequiredEnvName'); + expect(result.stderr).toInclude('SECRET2'); + }); + + test('should replace variable with default if present', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + }); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'string', default: 'abc' }, + }, + required: ['SECRET1', 'SECRET2'], + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(0); + + // Confirm only the specified secrets were exported + expect(result.stdout).toInclude("SECRET1='SECRET1'"); + expect(result.stdout).toInclude("SECRET2='abc"); + }); + }); }); From c7c15d79c73b3d7b7ab0fac71910c62cde300e2a Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 8 Jul 2025 17:33:14 +1000 Subject: [PATCH 6/7] feat: aligned schemas with standard schema behaviour --- ref.schema.json | 14 --- src/errors.ts | 8 +- src/secrets/CommandEnv.ts | 126 ++++++---------------- test.schema.json | 11 -- tests/secrets/env.test.ts | 217 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 247 insertions(+), 129 deletions(-) delete mode 100644 ref.schema.json delete mode 100644 test.schema.json diff --git a/ref.schema.json b/ref.schema.json deleted file mode 100644 index d4c3987e..00000000 --- a/ref.schema.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "object", - "allOf": [ - { "$ref": "./test.schema.json"} - ], - "properties": { - "SECRET2": { - "type": "string" - } - }, - "required": [ - "SECRET2" - ] -} \ No newline at end of file diff --git a/src/errors.ts b/src/errors.ts index 2c5201ad..2e62c3b6 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -202,10 +202,15 @@ class ErrorPolykeyCLITouchSecret extends ErrorPolykeyCLI { } class ErrorPolykeyCLIInvalidJWT extends ErrorPolykeyCLI { - static description: 'JWT is not valid'; + static description = 'JWT is not valid'; exitCode = sysexits.USAGE; } +class ErrorPolykeyCLISchemaInvalid extends ErrorPolykeyCLI { + static description = 'The provided JSON schema is invalid'; + exitCode = sysexits.CONFIG; +} + export { ErrorPolykeyCLI, ErrorPolykeyCLIUncaughtException, @@ -236,4 +241,5 @@ export { ErrorPolykeyCLIEditSecret, ErrorPolykeyCLITouchSecret, ErrorPolykeyCLIInvalidJWT, + ErrorPolykeyCLISchemaInvalid, }; diff --git a/src/secrets/CommandEnv.ts b/src/secrets/CommandEnv.ts index 9064d258..43d5a98d 100644 --- a/src/secrets/CommandEnv.ts +++ b/src/secrets/CommandEnv.ts @@ -1,13 +1,9 @@ import type PolykeyClient from 'polykey/PolykeyClient.js'; -import type { - JSONSchema, - JSONSchemaInfo, - ParsedSecretPathValue, -} from '../types.js'; +import type { ParsedSecretPathValue } from '../types.js'; import path from 'node:path'; import os from 'node:os'; import $RefParser from '@apidevtools/json-schema-ref-parser'; -import { Ajv2019 as Ajv } from 'ajv/dist/2019.js'; +import { Ajv2019 } from 'ajv/dist/2019.js'; import { InvalidArgumentError } from 'commander'; import CommandPolykey from '../CommandPolykey.js'; import * as binProcessors from '../utils/processors.js'; @@ -126,65 +122,25 @@ class CommandEnv extends CommandPolykey { logger: this.logger.getChild(PolykeyClient.name), }); - let schema: JSONSchema | undefined = undefined; - let unwrappedSchema: JSONSchemaInfo | undefined = undefined; - if (options.egressSchema != null) { - schema = (await $RefParser.bundle( - options.egressSchema, - )) satisfies JSONSchema; - unwrappedSchema = binUtils.loadSchema(schema!); - } - // Getting envs const [envp] = await binUtils.retryAuthentication(async (auth) => { const responseStream = await pkClient.rpcClient.methods.vaultsSecretsEnv(); - // Writing desired secrets + // Writing desired secrets. Attempt to get all the required secrets. + // If the schema is provided, then the resulting variables will be + // validated. const secretRenameMap = new Map(); const writer = responseStream.writable.getWriter(); let first = true; for (const envVariable of envVariables) { const [nameOrId, secretName, secretNameNew] = envVariable; secretRenameMap.set(secretName ?? '/', secretNameNew); - - // If there is no secret name provided, then attempt to export the - // secrets from the entire vault. Otherwise, check if the selected - // secret exists in the schema before requesting it. This will - // only run if a schema has been specified. - if (schema != null && unwrappedSchema != null) { - const { allKeys } = unwrappedSchema; - if (nameOrId != null && secretName == null) { - // Only vault specified - for (const key of allKeys) { - // When exporting secrets from a vault, it is impossible to - // rename the resulting secrets. - await writer.write({ - nameOrId: nameOrId, - secretName: key, - metadata: first ? auth : undefined, - }); - } - } else { - // Individual secret name specified - const name: string = - secretNameNew != null ? secretNameNew : secretName!; - if (allKeys.includes(name)) { - await writer.write({ - nameOrId: nameOrId, - secretName: secretName!, - metadata: first ? auth : undefined, - }); - } - } - } else { - // No schema specified - await writer.write({ - nameOrId: nameOrId, - secretName: secretName ?? '/', - metadata: first ? auth : undefined, - }); - } + await writer.write({ + nameOrId: nameOrId, + secretName: secretName ?? '/', + metadata: first ? auth : undefined, + }); first = false; } await writer.close(); @@ -207,10 +163,6 @@ class CommandEnv extends CommandPolykey { `TMP Vault "${value.data?.nameOrId}" does not exist`, ); case 'ENOENT': - // If we are working with schemas, then missing keys will be - // validated later. - if (unwrappedSchema != null) break; - // It is expected for the data to be populated with the // offending secret and vault name if a secret was not found. throw new Error( @@ -301,44 +253,34 @@ class CommandEnv extends CommandPolykey { }; } - // Apply defaults using the schema - const filteredEnvp: Record = {}; - if (schema != null && unwrappedSchema != null) { - // Parse the schema for manual filtering - const { requiredKeys, allKeys, defaults } = unwrappedSchema; + // Validate the schema + if (options.egressSchema != null) { + // Compose the schema as ajv cannot parse cross-schema refs + const schema = await $RefParser.bundle(options.egressSchema); - // Add allowed secrets to a filtered set of secrets. This runs after - // the duplication is processed, so all secrets here are guaranteed - // to be unique. - for (const key of allKeys) { - let value = envp[key]; - if (value == null && defaults[key] != null) { - value = defaults[key]; - } - if ( - requiredKeys.includes(key) && - (value == null || value === '') - ) { - throw new binErrors.ErrorPolykeyCLIMissingRequiredEnvName( - `Expected definition for ${key}`, - ); - } - if (value != null) { - filteredEnvp[key] = value.toString(); - } - } - - // Validate the schema using ajv. All defaults have already been - // applied. This is now the final state of the exported variables. - const ajv = new Ajv({ allErrors: true }); + // Validate the schema using ajv. This will also apply defaults and + // coerce types as necessary. + const ajv = new Ajv2019({ + strict: true, + allErrors: true, + useDefaults: true, + coerceTypes: true, + }); const validate = ajv.compile(schema); - validate(envp); + const valid = validate(envp); + if (!valid && validate.errors != null) { + throw new binErrors.ErrorPolykeyCLISchemaInvalid( + 'JSON schema validation failed', + { + data: { + errors: [...validate.errors], + }, + }, + ); + } } - return [ - utils.isEmptyObject(filteredEnvp) ? envp : filteredEnvp, - envpPath, - ]; + return [envp, envpPath]; }, meta); // End connection early to avoid errors on server await pkClient.stop(); diff --git a/test.schema.json b/test.schema.json deleted file mode 100644 index e2279068..00000000 --- a/test.schema.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "object", - "properties": { - "SECRET1": { - "type": "string" - } - }, - "required": [ - "SECRET1" - ] -} \ No newline at end of file diff --git a/tests/secrets/env.test.ts b/tests/secrets/env.test.ts index 7a43582a..ff2c0d38 100644 --- a/tests/secrets/env.test.ts +++ b/tests/secrets/env.test.ts @@ -955,7 +955,7 @@ describe('commandEnv', () => { ); describe('should apply json schema to control secrets egress', () => { - test('should filter secrets based on a schema', async () => { + test('should validate secrets based on a schema', async () => { // Write secrets to vault const vaultName = 'vault'; const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); @@ -997,10 +997,58 @@ describe('commandEnv', () => { }); expect(result.exitCode).toBe(0); - // Confirm only the specified secrets were exported + // Confirm all the secrets were exported expect(result.stdout).toContain("SECRET1='SECRET1'"); expect(result.stdout).toContain("SECRET2='SECRET2'"); - expect(result.stdout).not.toContain("SECRET3='SECRET3'"); + expect(result.stdout).toContain("SECRET3='SECRET3'"); + }); + + test('should fail if no additional properties are expected', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + const secretName2 = 'SECRET2'; + const secretName3 = 'SECRET3'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + await vaultOps.addSecret(vault, secretName2, secretName2); + await vaultOps.addSecret(vault, secretName3, secretName3); + }); + + // Write schema to file + const schema = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'string' }, + }, + required: ['SECRET1', 'SECRET2'], + additionalProperties: false, + }; + const schemaPath = path.join(dataDir, 'egress.schema.json'); + await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); + + // Run command with the schema + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schemaPath, + vaultName, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + expect(result.exitCode).toBe(78); + + // Check if the errors are valid and correct + expect(result.stderr).toContain('ErrorPolykeyCLISchemaInvalid'); + expect(result.stderr).toContain('additionalProperties'); }); test('should apply schema to secrets from multiple vaults', async () => { @@ -1027,8 +1075,9 @@ describe('commandEnv', () => { properties: { SECRET1: { type: 'string' }, SECRET2: { type: 'string' }, + SECRET3: { type: 'string' }, }, - required: ['SECRET1', 'SECRET2'], + required: ['SECRET1', 'SECRET2', 'SECRET3'], }; const schemaPath = path.join(dataDir, 'egress.schema.json'); await fs.promises.writeFile(schemaPath, JSON.stringify(schema, null, 2)); @@ -1054,7 +1103,7 @@ describe('commandEnv', () => { // Confirm only the specified secrets were exported expect(result.stdout).toContain("SECRET1='SECRET1'"); expect(result.stdout).toContain("SECRET2='SECRET2'"); - expect(result.stdout).not.toContain("SECRET3='SECRET3'"); + expect(result.stdout).toContain("SECRET3='SECRET3'"); }); test('should handle secret renames', async () => { @@ -1063,12 +1112,10 @@ describe('commandEnv', () => { const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); const secretName1 = 'SECRET1'; const secretName2 = 'SECRET2'; - const secretName3 = 'SECRET3'; const secretRename = 'RENAMED'; await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { await vaultOps.addSecret(vault, secretName1, secretName1); await vaultOps.addSecret(vault, secretName2, secretName2); - await vaultOps.addSecret(vault, secretName3, secretName3); }); // Write schema to file @@ -1094,7 +1141,7 @@ describe('commandEnv', () => { 'unix', '--egress-schema', schemaPath, - vaultName, + `${vaultName}:${secretName1}`, `${vaultName}:${secretName2}=${secretRename}`, ]; const result = await testUtils.pkExec(command, { @@ -1106,7 +1153,6 @@ describe('commandEnv', () => { expect(result.stdout).toContain("SECRET1='SECRET1'"); expect(result.stdout).toContain("RENAMED='SECRET2'"); expect(result.stdout).not.toContain("SECRET2='SECRET2'"); - expect(result.stdout).not.toContain("SECRET3='SECRET3'"); }); test('should not fail when missing non-required secret', async () => { @@ -1189,10 +1235,10 @@ describe('commandEnv', () => { const result = await testUtils.pkExec(command, { env: { PK_PASSWORD: password }, }); - expect(result.exitCode).toBe(64); + expect(result.exitCode).toBe(78); // Confirm the validity of the error - expect(result.stderr).toInclude('ErrorPolykeyCLIMissingRequiredEnvName'); + expect(result.stderr).toInclude('ErrorPolykeyCLISchemaInvalid'); expect(result.stderr).toInclude('SECRET2'); }); @@ -1238,5 +1284,154 @@ describe('commandEnv', () => { expect(result.stdout).toInclude("SECRET1='SECRET1'"); expect(result.stdout).toInclude("SECRET2='abc"); }); + + test('should validate variable types', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + const secretData1 = 'secret string'; + const secretName2 = 'SECRET2'; + const secretData2 = '123'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretData1); + await vaultOps.addSecret(vault, secretName2, secretData2); + }); + + // Write schema to file. Schema 1 should be valid, and schema 2 should + // fail validation of the secrets. + const schema1 = { + type: 'object', + properties: { + SECRET1: { type: 'string' }, + SECRET2: { type: 'number' }, + }, + required: ['SECRET1', 'SECRET2'], + }; + const schema2 = { + type: 'object', + properties: { + SECRET1: { type: 'number' }, + SECRET2: { type: 'string' }, + }, + required: ['SECRET1', 'SECRET2'], + }; + const schema1Path = path.join(dataDir, 'egress1.schema.json'); + const schema2Path = path.join(dataDir, 'egress2.schema.json'); + await fs.promises.writeFile( + schema1Path, + JSON.stringify(schema1, null, 2), + ); + await fs.promises.writeFile( + schema2Path, + JSON.stringify(schema2, null, 2), + ); + + // Run command with the valid schema + const command1 = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schema1Path, + vaultName, + ]; + const result1 = await testUtils.pkExec(command1, { + env: { PK_PASSWORD: password }, + }); + expect(result1.exitCode).toBe(0); + + // Confirm only the specified secrets were exported + expect(result1.stdout).toInclude(`${secretName1}='${secretData1}'`); + expect(result1.stdout).toInclude(`${secretName2}='${secretData2}'`); + + // Run command with the invalid schema + const command2 = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schema2Path, + vaultName, + ]; + const result2 = await testUtils.pkExec(command2, { + env: { PK_PASSWORD: password }, + }); + expect(result2.exitCode).toBe(78); + + // Confirm the error matches expectations + expect(result2.stderr).toContain('ErrorPolykeyCLISchemaInvalid'); + expect(result2.stderr).toContain('SECRET1'); + expect(result2.stderr).toContain('must be number'); + }); + + test('should validate with composed schemas', async () => { + // Write secrets to vault + const vaultName = 'vault'; + const vaultId = await polykeyAgent.vaultManager.createVault(vaultName); + const secretName1 = 'SECRET1'; + const secretName2 = 'SECRET2'; + await polykeyAgent.vaultManager.withVaults([vaultId], async (vault) => { + await vaultOps.addSecret(vault, secretName1, secretName1); + await vaultOps.addSecret(vault, secretName2, secretName2); + }); + + // Write schema to file. Schema 1 should be valid, and schema 2 should + // fail validation of the secrets. + const schema1Path = path.join(dataDir, 'egress1.schema.json'); + const schema2Path = path.join(dataDir, 'egress2.schema.json'); + const schema1 = { + allOf: [{ $ref: schema2Path }], + type: 'object', + properties: { + SECRET1: { type: 'string' }, + }, + required: ['SECRET1'], + }; + const schema2 = { + type: 'object', + properties: { + SECRET2: { type: 'number' }, + }, + required: ['SECRET2'], + }; + await fs.promises.writeFile( + schema1Path, + JSON.stringify(schema1, null, 2), + ); + await fs.promises.writeFile( + schema2Path, + JSON.stringify(schema2, null, 2), + ); + + // The command can only fail if it read the second schema and realised + // that SECRET2 is a string and is meant to be a number. + const command = [ + 'secrets', + 'env', + '-np', + dataDir, + '--env-format', + 'unix', + '--egress-schema', + schema1Path, + vaultName, + ]; + const result = await testUtils.pkExec(command, { + env: { PK_PASSWORD: password }, + }); + + // Confirm the error matches expectations + expect(result.exitCode).toBe(78); + expect(result.stderr).toContain('ErrorPolykeyCLISchemaInvalid'); + expect(result.stderr).toContain('SECRET2'); + expect(result.stderr).toContain('must be number'); + }); }); }); From e0b9eed8b6a62a6a8fbcbe3563848d904f30b31d Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 8 Jul 2025 18:23:56 +1000 Subject: [PATCH 7/7] deps: updated polykey from 2.4.0 to 2.5.0 --- npmDepsHash | 2 +- package-lock.json | 8 ++++---- package.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/npmDepsHash b/npmDepsHash index 47abbf8a..c4b2f81a 100644 --- a/npmDepsHash +++ b/npmDepsHash @@ -1 +1 @@ -sha256-5lr9ukYvA7s8sAgnX5rN6ef12tOb9lPwAHGlOrD0k+w= +sha256-q99IFZwwPBR/p2m18c5DOApks12rB779c8rUW160k3g= diff --git a/package-lock.json b/package-lock.json index c80a95d8..09cf6a7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,7 @@ "nexpect": "^0.6.0", "node-gyp-build": "^4.8.4", "open": "^10.1.2", - "polykey": "^2.5.0", + "polykey": "^2.6.0", "shelljs": "^0.8.5", "shx": "^0.3.4", "tsx": "^3.12.7", @@ -9812,9 +9812,9 @@ } }, "node_modules/polykey": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/polykey/-/polykey-2.5.0.tgz", - "integrity": "sha512-cAtC5mdJQTnrcgPR0PEBzCCNvqoQ64/9CdQ56eEBKZeDXJmOUj66LjpyZkTuNqUgR9kN4n1dN98k3zAMOscATA==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/polykey/-/polykey-2.6.0.tgz", + "integrity": "sha512-213GWkrP0oBLvSEB7qYDMa+iqxy7QuZ1XaX6k9oSF9np9mvrHdWlopv45FtUkSm1vrDNtl9HOxWkH+Ij6EujAg==", "dev": true, "dependencies": { "@matrixai/async-cancellable": "^2.0.1", diff --git a/package.json b/package.json index 4201dced..bd5a85fb 100644 --- a/package.json +++ b/package.json @@ -165,7 +165,7 @@ "nexpect": "^0.6.0", "node-gyp-build": "^4.8.4", "open": "^10.1.2", - "polykey": "^2.5.0", + "polykey": "^2.6.0", "shelljs": "^0.8.5", "shx": "^0.3.4", "tsx": "^3.12.7",