Skip to content

Commit 59e1c06

Browse files
rayokotarayokota
authored
Add more unit tests; minor fixes for KMS clients (#86)
* Minor fixes for KMS clients * Add JSON 2020-12 test
1 parent 8b6521c commit 59e1c06

File tree

15 files changed

+1095
-59
lines changed

15 files changed

+1095
-59
lines changed

‎schemaregistry/mock-schemaregistry-client.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,12 @@ class MockClient implements Client {
201201
const parsedKey = JSON.parse(key);
202202
if (parsedKey.subject === subject && (!value.softDeleted || deleted)) {
203203
if (parsedKey.schema.metadata && this.isSubset(metadata, parsedKey.schema.metadata.properties)) {
204-
results.push({ id: parsedKey.schema.id, version: value.version, subject, schema: parsedKey.schema.schema });
204+
results.push({
205+
id: parsedKey.schema.id,
206+
version: value.version,
207+
subject,
208+
...parsedKey.schema
209+
});
205210
}
206211
}
207212
}

‎schemaregistry/rules/encryption/azurekms/azure-client.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ export class AzureKmsClient implements KmsClient {
2222
}
2323

2424
async encrypt(plaintext: Buffer): Promise<Buffer> {
25-
const result = await this.kmsClient.encrypt(AzureKmsClient.ALGORITHM, plaintext)
25+
const result = await this.kmsClient.encrypt(AzureKmsClient.ALGORITHM, plaintext)
2626
return Buffer.from(result.result)
2727
}
2828

‎schemaregistry/rules/encryption/azurekms/azure-driver.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import {AzureKmsClient} from "./azure-client";
55
export class AzureKmsDriver implements KmsDriver {
66

77
static PREFIX = 'azure-kms://'
8-
static TENANT_ID = 'tenant_id'
8+
static TENANT_ID = 'tenant.id'
99
static CLIENT_ID = 'client.id'
1010
static CLIENT_SECRET = 'client.secret'
1111

‎schemaregistry/rules/encryption/encrypt-executor.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,31 @@ interface DekId {
5454
deleted: boolean
5555
}
5656

57+
export class Clock {
58+
now(): number {
59+
return Date.now()
60+
}
61+
}
62+
5763
export class FieldEncryptionExecutor extends FieldRuleExecutor {
5864
client: Client | null = null
65+
clock: Clock
5966

6067
static register(): FieldEncryptionExecutor {
61-
const executor = new FieldEncryptionExecutor()
68+
return this.registerWithClock(new Clock())
69+
}
70+
71+
static registerWithClock(clock: Clock): FieldEncryptionExecutor {
72+
const executor = new FieldEncryptionExecutor(clock)
6273
RuleRegistry.registerRuleExecutor(executor)
6374
return executor
6475
}
6576

77+
constructor(clock: Clock = new Clock()) {
78+
super()
79+
this.clock = clock
80+
}
81+
6682
override configure(clientConfig: ClientConfig, config: Map<string, string>) {
6783
this.client = DekRegistryClient.newClient(clientConfig)
6884
this.config = config
@@ -416,7 +432,7 @@ export class FieldEncryptionExecutorTransform implements FieldTransform {
416432
}
417433

418434
isExpired(ctx: RuleContext, dek: Dek | null): boolean {
419-
const now = Date.now()
435+
const now = this.executor.clock.now()
420436
return ctx.ruleMode !== RuleMode.READ &&
421437
this.dekExpiryDays > 0 &&
422438
dek != null &&

‎schemaregistry/rules/encryption/gcpkms/gcp-client.ts‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,9 @@ export class GcpKmsClient implements KmsClient {
1212
throw new Error(`key uri must start with ${GcpKmsDriver.PREFIX}`)
1313
}
1414
this.keyId = keyUri.substring(GcpKmsDriver.PREFIX.length)
15-
const tokens = this.keyId.split(':')
16-
if (tokens.length < 4) {
17-
throw new Error(`invalid key uri ${this.keyId}`)
18-
}
1915
this.kmsClient = creds != null
20-
? new KeyManagementServiceClient()
21-
: new KeyManagementServiceClient({credentials: creds})
16+
? new KeyManagementServiceClient({credentials: creds})
17+
: new KeyManagementServiceClient()
2218
}
2319

2420
supported(keyUri: string): boolean {

‎schemaregistry/rules/encryption/hcvault/hcvault-client.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,14 @@ export class HcVaultClient implements KmsClient {
3636
}
3737

3838
async encrypt(plaintext: Buffer): Promise<Buffer> {
39-
const data = await this.kmsClient.encryptData({name: this.keyName, plainText: plaintext.toString('base64') })
40-
return Buffer.from(data.ciphertext, 'base64')
39+
const response = await this.kmsClient.encryptData({name: this.keyName, plaintext: plaintext.toString('base64') })
40+
let data = response.data.ciphertext
41+
return Buffer.from(data, 'utf8')
4142
}
4243

4344
async decrypt(ciphertext: Buffer): Promise<Buffer> {
44-
const data = await this.kmsClient.decryptData({name: this.keyName, cipherText: ciphertext.toString('base64') })
45-
return Buffer.from(data.plaintext, 'base64')
45+
const response = await this.kmsClient.decryptData({name: this.keyName, ciphertext: ciphertext.toString('utf8') })
46+
let data = response.data.plaintext
47+
return Buffer.from(data, 'base64');
4648
}
4749
}

‎schemaregistry/rules/encryption/hcvault/hcvault-driver.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import {HcVaultClient} from "./hcvault-client";
33

44
export class HcVaultDriver implements KmsDriver {
55

6-
static PREFIX = 'hcvault-kms://'
6+
static PREFIX = 'hcvault://'
77
static TOKEN_ID = 'token.id'
88
static NAMESPACE = 'namespace'
99

‎schemaregistry/rules/encryption/kms-registry.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,8 @@ export function getKmsClient(keyUrl: string): KmsClient | null {
4242
return null
4343
}
4444

45+
export function clearKmsClients(): void {
46+
kmsClients.length = 0
47+
}
48+
49+

‎schemaregistry/serde/avro.ts‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -53,31 +53,7 @@ export class AvroSerializer extends Serializer implements AvroSerde {
5353
throw new Error('message is empty')
5454
}
5555

56-
let enumIndex = 1
57-
let fixedIndex = 1
58-
let recordIndex = 1
59-
60-
const namingHook: TypeHook = (
61-
avroSchema: avro.Schema,
62-
opts: ForSchemaOptions,
63-
) => {
64-
let schema = avroSchema as any
65-
switch (schema.type) {
66-
case 'enum':
67-
schema.name = `Enum${enumIndex++}`;
68-
break;
69-
case 'fixed':
70-
schema.name = `Fixed${fixedIndex++}`;
71-
break;
72-
case 'record':
73-
schema.name = `Record${recordIndex++}`;
74-
break;
75-
default:
76-
}
77-
return undefined
78-
}
79-
80-
let avroSchema = Type.forValue(msg, { typeHook: namingHook })
56+
let avroSchema = AvroSerializer.messageToSchema(msg)
8157
const schema: SchemaInfo = {
8258
schemaType: 'AVRO',
8359
schema: JSON.stringify(avroSchema),
@@ -104,6 +80,34 @@ export class AvroSerializer extends Serializer implements AvroSerde {
10480
return deps
10581
})
10682
}
83+
84+
static messageToSchema(msg: any): avro.Type {
85+
let enumIndex = 1
86+
let fixedIndex = 1
87+
let recordIndex = 1
88+
89+
const namingHook: TypeHook = (
90+
avroSchema: avro.Schema,
91+
opts: ForSchemaOptions,
92+
) => {
93+
let schema = avroSchema as any
94+
switch (schema.type) {
95+
case 'enum':
96+
schema.name = `Enum${enumIndex++}`;
97+
break;
98+
case 'fixed':
99+
schema.name = `Fixed${fixedIndex++}`;
100+
break;
101+
case 'record':
102+
schema.name = `Record${recordIndex++}`;
103+
break;
104+
default:
105+
}
106+
return undefined
107+
}
108+
109+
return Type.forValue(msg, { typeHook: namingHook })
110+
}
107111
}
108112

109113
export type AvroDeserializerConfig = DeserializerConfig & AvroSerdeConfig

‎schemaregistry/serde/json.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ export class JsonSerializer extends Serializer implements JsonSerde {
7373
throw new Error('message is empty')
7474
}
7575

76-
const jsonSchema = generateSchema(msg)
76+
const jsonSchema = JsonSerializer.messageToSchema(msg)
7777
const schema: SchemaInfo = {
7878
schemaType: 'JSON',
7979
schema: JSON.stringify(jsonSchema),
@@ -92,14 +92,14 @@ export class JsonSerializer extends Serializer implements JsonSerde {
9292
}
9393

9494
async fieldTransform(ctx: RuleContext, fieldTransform: FieldTransform, msg: any): Promise<any> {
95-
const schema = this.toType(ctx.target)
95+
const schema = await this.toType(ctx.target)
9696
if (typeof schema === 'boolean') {
9797
return msg
9898
}
9999
return await transform(ctx, schema, '$', msg, fieldTransform)
100100
}
101101

102-
toType(info: SchemaInfo): DereferencedJSONSchema {
102+
async toType(info: SchemaInfo): Promise<DereferencedJSONSchema> {
103103
return toType(this.client, this.conf as JsonDeserializerConfig, this, info, async (client, info) => {
104104
const deps = new Map<string, string>()
105105
await this.resolveReferences(client, info, deps)
@@ -115,6 +115,10 @@ export class JsonSerializer extends Serializer implements JsonSerde {
115115
},
116116
)
117117
}
118+
119+
static messageToSchema(msg: any): DereferencedJSONSchema {
120+
return generateSchema(msg)
121+
}
118122
}
119123

120124
export type JsonDeserializerConfig = DeserializerConfig & JsonSerdeConfig
@@ -173,7 +177,7 @@ export class JsonDeserializer extends Deserializer implements JsonSerde {
173177
}
174178

175179
async fieldTransform(ctx: RuleContext, fieldTransform: FieldTransform, msg: any): Promise<any> {
176-
const schema = this.toType(ctx.target)
180+
const schema = await this.toType(ctx.target)
177181
return await transform(ctx, schema, '$', msg, fieldTransform)
178182
}
179183

@@ -211,14 +215,17 @@ async function toValidateFunction(
211215

212216
const json = JSON.parse(info.schema)
213217
const spec = json.$schema
214-
if (spec === 'http://json-schema.org/draft/2020-12/schema') {
218+
if (spec === 'http://json-schema.org/draft/2020-12/schema'
219+
|| spec === 'https://json-schema.org/draft/2020-12/schema') {
215220
const ajv2020 = new Ajv2020(conf as JsonSerdeConfig)
221+
ajv2020.addKeyword("confluent:tags")
216222
deps.forEach((schema, name) => {
217223
ajv2020.addSchema(JSON.parse(schema), name)
218224
})
219225
fn = ajv2020.compile(json)
220226
} else {
221227
const ajv = new Ajv2019(conf as JsonSerdeConfig)
228+
ajv.addKeyword("confluent:tags")
222229
ajv.addMetaSchema(draft6MetaSchema)
223230
ajv.addMetaSchema(draft7MetaSchema)
224231
deps.forEach((schema, name) => {

0 commit comments

Comments
 (0)