diff --git a/etl-api/src/k8s/http.rs b/etl-api/src/k8s/http.rs index 57db15e77..0291298a8 100644 --- a/etl-api/src/k8s/http.rs +++ b/etl-api/src/k8s/http.rs @@ -117,9 +117,7 @@ impl HttpK8sClient { /// /// Prefers in-cluster configuration and falls back to the local kubeconfig /// when running outside the cluster. - pub async fn new() -> Result { - let client = Client::try_default().await?; - + pub async fn new(client: Client) -> Result { let secrets_api: Api = Api::namespaced(client.clone(), DATA_PLANE_NAMESPACE); let config_maps_api: Api = Api::namespaced(client.clone(), DATA_PLANE_NAMESPACE); let stateful_sets_api: Api = @@ -331,12 +329,20 @@ impl K8sClient for HttpK8sClient { destination_type, ); + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(prefix, &environment, &config); + let volumes = create_volumes_json(prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + let stateful_set_json = create_replicator_stateful_set_json( prefix, &stateful_set_name, - &config, replicator_image, container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, ); let stateful_set: StatefulSet = serde_json::from_value(stateful_set_json)?; @@ -555,27 +561,35 @@ fn create_container_environment_json( //TODO: set APP_VERSION to proper version instead of the replicator image name "value": replicator_image }), - json!({ - "name": "APP_SENTRY__DSN", - "valueFrom": { - "secretKeyRef": { - "name": SENTRY_DSN_SECRET_NAME, - "key": "dsn", - "optional": true - } - } - }), - json!({ - "name": "APP_SUPABASE__API_KEY", - "valueFrom": { - "secretKeyRef": { - "name": SUPABASE_API_KEY_SECRET_NAME, - "key": "key", - "optional": true - } - } - }), ]; + + match environment { + Environment::Dev => { + // We do not configure sentry for dev environments + } + Environment::Staging | Environment::Prod => { + container_environment.push(json!({ + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "name": SENTRY_DSN_SECRET_NAME, + "key": "dsn" + } + } + })); + container_environment.push(json!({ + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "name": SUPABASE_API_KEY_SECRET_NAME, + "key": "key", + "optional": true + } + } + })); + } + } + match destination_type { DestinationType::Memory => {} DestinationType::BigQuery => { @@ -612,6 +626,123 @@ fn create_container_environment_json( container_environment } +fn create_node_selector_json(environment: &Environment) -> serde_json::Value { + // In staging and prod, pin pods to nodes labeled with `nodeType=workloads`. + match environment { + Environment::Dev => json!({}), + Environment::Staging | Environment::Prod => json!({ + "nodeType": "workloads" + }), + } +} + +fn create_init_containers_json( + prefix: &str, + environment: &Environment, + config: &ReplicatorResourceConfig, +) -> serde_json::Value { + let vector_container_name = create_vector_container_name(prefix); + // In staging and prod, run vector init container to collect logs + match environment { + Environment::Dev => json!([]), + Environment::Staging | Environment::Prod => json!([ + { + "name": vector_container_name, + "image": VECTOR_IMAGE_NAME, + "restartPolicy": "Always", + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "name": LOGFLARE_SECRET_NAME, + "key": "key" + } + } + } + ], + "resources": { + "limits": { + "memory": config.max_memory, + "cpu": config.max_cpu, + }, + "requests": { + "memory": config.max_memory, + "cpu": config.max_cpu, + } + }, + "volumeMounts": [ + { + "name": VECTOR_CONFIG_FILE_VOLUME_NAME, + "mountPath": "/etc/vector" + }, + { + "name": LOGS_VOLUME_NAME, + "mountPath": "/var/log" + } + ] + } + ]), + } +} + +fn create_volumes_json(prefix: &str, environment: &Environment) -> Vec { + let replicator_config_map_name = create_replicator_config_map_name(prefix); + let mut volumes = vec![json!( + { + "name": REPLICATOR_CONFIG_FILE_VOLUME_NAME, + "configMap": { + "name": replicator_config_map_name + } + } + )]; + + match environment { + Environment::Dev => { + // We do not configure vector or logs volumes for dev environments + } + Environment::Staging | Environment::Prod => { + volumes.push(json!( + { + "name": VECTOR_CONFIG_FILE_VOLUME_NAME, + "configMap": { + "name": VECTOR_CONFIG_MAP_NAME + } + })); + volumes.push(json!({ + "name": LOGS_VOLUME_NAME, + "emptyDir": {} + })); + } + } + + volumes +} + +fn create_volume_mounts_json(environment: &Environment) -> Vec { + let mut volume_mounts = vec![json!( + { + "name": REPLICATOR_CONFIG_FILE_VOLUME_NAME, + "mountPath": "/app/configuration" + } + )]; + + match environment { + Environment::Dev => { + // We do not configure logs volume mount for dev environments + } + Environment::Staging | Environment::Prod => { + volume_mounts.push(json!( + { + "name": LOGS_VOLUME_NAME, + "mountPath": "/app/logs" + })); + } + } + + volume_mounts +} + fn create_postgres_secret_env_var_json(postgres_secret_name: &str) -> serde_json::Value { json!({ "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", @@ -674,17 +805,19 @@ fn create_iceberg_s3_secret_access_key_env_var_json( }) } +#[expect(clippy::too_many_arguments)] fn create_replicator_stateful_set_json( prefix: &str, stateful_set_name: &str, - config: &ReplicatorResourceConfig, replicator_image: &str, container_environment: Vec, + node_selector: serde_json::Value, + init_containers: serde_json::Value, + volumes: Vec, + volume_mounts: Vec, ) -> serde_json::Value { let replicator_app_name = create_replicator_app_name(prefix); let restarted_at_annotation = get_restarted_at_annotation_value(); - let replicator_config_map_name = create_replicator_config_map_name(prefix); - let vector_container_name = create_vector_container_name(prefix); let replicator_container_name = create_replicator_container_name(prefix); json!({ @@ -713,24 +846,7 @@ fn create_replicator_stateful_set_json( } }, "spec": { - "volumes": [ - { - "name": REPLICATOR_CONFIG_FILE_VOLUME_NAME, - "configMap": { - "name": replicator_config_map_name - } - }, - { - "name": VECTOR_CONFIG_FILE_VOLUME_NAME, - "configMap": { - "name": VECTOR_CONFIG_MAP_NAME - } - }, - { - "name": LOGS_VOLUME_NAME, - "emptyDir": {} - } - ], + "volumes": volumes, // Allow scheduling onto nodes tainted with `nodeType=workloads`. "tolerations": [ { @@ -740,51 +856,11 @@ fn create_replicator_stateful_set_json( "effect": "NoSchedule" } ], - // Pin pods to nodes labeled with `nodeType=workloads`. - "nodeSelector": { - "nodeType": "workloads" - }, + "nodeSelector": node_selector, // We want to wait at most 5 minutes before K8S sends a `SIGKILL` to the containers, // this way we let the system finish any in-flight transaction, if there are any. "terminationGracePeriodSeconds": 300, - "initContainers": [ - { - "name": vector_container_name, - "image": VECTOR_IMAGE_NAME, - "restartPolicy": "Always", - "env": [ - { - "name": "LOGFLARE_API_KEY", - "valueFrom": { - "secretKeyRef": { - "name": LOGFLARE_SECRET_NAME, - "key": "key" - } - } - } - ], - "resources": { - "limits": { - "memory": config.max_memory, - "cpu": config.max_cpu, - }, - "requests": { - "memory": config.max_memory, - "cpu": config.max_cpu, - } - }, - "volumeMounts": [ - { - "name": VECTOR_CONFIG_FILE_VOLUME_NAME, - "mountPath": "/etc/vector" - }, - { - "name": LOGS_VOLUME_NAME, - "mountPath": "/var/log" - } - ] - } - ], + "initContainers": init_containers, "containers": [ { "name": replicator_container_name, @@ -797,16 +873,7 @@ fn create_replicator_stateful_set_json( } ], "env": container_environment, - "volumeMounts": [ - { - "name": REPLICATOR_CONFIG_FILE_VOLUME_NAME, - "mountPath": "/app/configuration" - }, - { - "name": LOGS_VOLUME_NAME, - "mountPath": "/app/logs" - }, - ] + "volumeMounts": volume_mounts } ] } @@ -997,43 +1064,140 @@ mod tests { #[test] fn test_create_bq_container_environment() { let prefix = create_k8s_object_prefix(TENANT_ID, 42); - let environment = Environment::Prod; let replicator_image = "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e"; + let environment = Environment::Dev; let container_environment = create_container_environment_json( &prefix, &environment, replicator_image, DestinationType::BigQuery, ); + assert_json_snapshot!(container_environment); + let environment = Environment::Staging; + let container_environment = create_container_environment_json( + &prefix, + &environment, + replicator_image, + DestinationType::BigQuery, + ); + assert_json_snapshot!(container_environment); + + let environment = Environment::Prod; + let container_environment = create_container_environment_json( + &prefix, + &environment, + replicator_image, + DestinationType::BigQuery, + ); assert_json_snapshot!(container_environment); } #[test] fn test_create_iceberg_container_environment() { let prefix = create_k8s_object_prefix(TENANT_ID, 42); - let environment = Environment::Prod; let replicator_image = "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e"; let container_environment = create_container_environment_json( &prefix, - &environment, + &Environment::Dev, replicator_image, DestinationType::Iceberg, ); + assert_json_snapshot!(container_environment); + let container_environment = create_container_environment_json( + &prefix, + &Environment::Staging, + replicator_image, + DestinationType::Iceberg, + ); + assert_json_snapshot!(container_environment); + + let container_environment = create_container_environment_json( + &prefix, + &Environment::Prod, + replicator_image, + DestinationType::Iceberg, + ); assert_json_snapshot!(container_environment); } #[test] - fn test_create_bq_replicator_stateful_set_json() { + fn test_create_node_selector() { + let node_selector = create_node_selector_json(&Environment::Dev); + assert_json_snapshot!(node_selector); + + let node_selector = create_node_selector_json(&Environment::Staging); + assert_json_snapshot!(node_selector); + + let node_selector = create_node_selector_json(&Environment::Prod); + assert_json_snapshot!(node_selector); + } + + #[test] + fn test_create_init_containers() { let prefix = create_k8s_object_prefix(TENANT_ID, 42); - let stateful_set_name = create_stateful_set_name(&prefix); + + let environment = Environment::Dev; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + let node_selector = create_init_containers_json(&prefix, &environment, &config); + assert_json_snapshot!(node_selector); + + let environment = Environment::Staging; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + let node_selector = create_init_containers_json(&prefix, &environment, &config); + assert_json_snapshot!(node_selector); + let environment = Environment::Prod; let config = ReplicatorResourceConfig::load(&environment).unwrap(); + let node_selector = create_init_containers_json(&prefix, &environment, &config); + assert_json_snapshot!(node_selector); + } + + #[test] + fn test_create_volumes() { + let prefix = create_k8s_object_prefix(TENANT_ID, 42); + + let environment = Environment::Dev; + let volumes = create_volumes_json(&prefix, &environment); + assert_json_snapshot!(volumes); + + let environment = Environment::Staging; + let volumes = create_volumes_json(&prefix, &environment); + assert_json_snapshot!(volumes); + + let environment = Environment::Prod; + let volumes = create_volumes_json(&prefix, &environment); + assert_json_snapshot!(volumes); + } + + #[test] + fn test_create_volume_mounts() { + let environment = Environment::Dev; + let volume_mounts = create_volume_mounts_json(&environment); + assert_json_snapshot!(volume_mounts); + + let environment = Environment::Staging; + let volume_mounts = create_volume_mounts_json(&environment); + assert_json_snapshot!(volume_mounts); + + let environment = Environment::Prod; + let volume_mounts = create_volume_mounts_json(&environment); + assert_json_snapshot!(volume_mounts); + } + + #[test] + fn test_create_bq_replicator_stateful_set_json() { + let prefix = create_k8s_object_prefix(TENANT_ID, 42); + let stateful_set_name = create_stateful_set_name(&prefix); let replicator_image = "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e"; + // Dev env + let environment = Environment::Dev; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + let container_environment = create_container_environment_json( &prefix, &environment, @@ -1041,16 +1205,83 @@ mod tests { DestinationType::BigQuery, ); + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(&prefix, &environment, &config); + let volumes = create_volumes_json(&prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + let stateful_set_json = create_replicator_stateful_set_json( &prefix, &stateful_set_name, - &config, replicator_image, container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, ); assert_json_snapshot!(stateful_set_json, { ".spec.template.metadata.annotations[\"etl.supabase.com/restarted-at\"]" => "[timestamp]"}); + let _stateful_set: StatefulSet = serde_json::from_value(stateful_set_json).unwrap(); + // Staging env + let environment = Environment::Staging; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + + let container_environment = create_container_environment_json( + &prefix, + &environment, + replicator_image, + DestinationType::BigQuery, + ); + + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(&prefix, &environment, &config); + let volumes = create_volumes_json(&prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + + let stateful_set_json = create_replicator_stateful_set_json( + &prefix, + &stateful_set_name, + replicator_image, + container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, + ); + + assert_json_snapshot!(stateful_set_json, { ".spec.template.metadata.annotations[\"etl.supabase.com/restarted-at\"]" => "[timestamp]"}); + let _stateful_set: StatefulSet = serde_json::from_value(stateful_set_json).unwrap(); + + // Prod env + let environment = Environment::Prod; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + + let container_environment = create_container_environment_json( + &prefix, + &environment, + replicator_image, + DestinationType::BigQuery, + ); + + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(&prefix, &environment, &config); + let volumes = create_volumes_json(&prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + + let stateful_set_json = create_replicator_stateful_set_json( + &prefix, + &stateful_set_name, + replicator_image, + container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, + ); + + assert_json_snapshot!(stateful_set_json, { ".spec.template.metadata.annotations[\"etl.supabase.com/restarted-at\"]" => "[timestamp]"}); let _stateful_set: StatefulSet = serde_json::from_value(stateful_set_json).unwrap(); } @@ -1058,10 +1289,12 @@ mod tests { fn test_create_iceberg_replicator_stateful_set_json() { let prefix = create_k8s_object_prefix(TENANT_ID, 42); let stateful_set_name = create_stateful_set_name(&prefix); - let environment = Environment::Prod; - let config = ReplicatorResourceConfig::load(&environment).unwrap(); let replicator_image = "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e"; + // Dev env + let environment = Environment::Dev; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + let container_environment = create_container_environment_json( &prefix, &environment, @@ -1069,16 +1302,83 @@ mod tests { DestinationType::Iceberg, ); + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(&prefix, &environment, &config); + let volumes = create_volumes_json(&prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + let stateful_set_json = create_replicator_stateful_set_json( &prefix, &stateful_set_name, - &config, replicator_image, container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, ); assert_json_snapshot!(stateful_set_json, { ".spec.template.metadata.annotations[\"etl.supabase.com/restarted-at\"]" => "[timestamp]"}); + let _stateful_set: StatefulSet = serde_json::from_value(stateful_set_json).unwrap(); + + // Staging env + let environment = Environment::Staging; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + + let container_environment = create_container_environment_json( + &prefix, + &environment, + replicator_image, + DestinationType::Iceberg, + ); + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(&prefix, &environment, &config); + let volumes = create_volumes_json(&prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + + let stateful_set_json = create_replicator_stateful_set_json( + &prefix, + &stateful_set_name, + replicator_image, + container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, + ); + + assert_json_snapshot!(stateful_set_json, { ".spec.template.metadata.annotations[\"etl.supabase.com/restarted-at\"]" => "[timestamp]"}); + let _stateful_set: StatefulSet = serde_json::from_value(stateful_set_json).unwrap(); + + // Prod env + let environment = Environment::Prod; + let config = ReplicatorResourceConfig::load(&environment).unwrap(); + + let container_environment = create_container_environment_json( + &prefix, + &environment, + replicator_image, + DestinationType::Iceberg, + ); + + let node_selector = create_node_selector_json(&environment); + let init_containers = create_init_containers_json(&prefix, &environment, &config); + let volumes = create_volumes_json(&prefix, &environment); + let volume_mounts = create_volume_mounts_json(&environment); + + let stateful_set_json = create_replicator_stateful_set_json( + &prefix, + &stateful_set_name, + replicator_image, + container_environment, + node_selector, + init_containers, + volumes, + volume_mounts, + ); + + assert_json_snapshot!(stateful_set_json, { ".spec.template.metadata.annotations[\"etl.supabase.com/restarted-at\"]" => "[timestamp]"}); let _stateful_set: StatefulSet = serde_json::from_value(stateful_set_json).unwrap(); } } diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment-2.snap new file mode 100644 index 000000000..2ce5432c2 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment-2.snap @@ -0,0 +1,51 @@ +--- +source: etl-api/src/k8s/http.rs +expression: container_environment +--- +[ + { + "name": "APP_ENVIRONMENT", + "value": "staging" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__BIG_QUERY__SERVICE_ACCOUNT_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "service-account-key", + "name": "abcdefghijklmnopqrst-42-bq-service-account-key" + } + } + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment-3.snap new file mode 100644 index 000000000..0e57328ed --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment-3.snap @@ -0,0 +1,51 @@ +--- +source: etl-api/src/k8s/http.rs +expression: container_environment +--- +[ + { + "name": "APP_ENVIRONMENT", + "value": "prod" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__BIG_QUERY__SERVICE_ACCOUNT_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "service-account-key", + "name": "abcdefghijklmnopqrst-42-bq-service-account-key" + } + } + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment.snap index 94a24c666..f480c0310 100644 --- a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment.snap +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_container_environment.snap @@ -5,32 +5,12 @@ expression: container_environment [ { "name": "APP_ENVIRONMENT", - "value": "prod" + "value": "dev" }, { "name": "APP_VERSION", "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" }, - { - "name": "APP_SENTRY__DSN", - "valueFrom": { - "secretKeyRef": { - "key": "dsn", - "name": "replicator-sentry-dsn", - "optional": true - } - } - }, - { - "name": "APP_SUPABASE__API_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "key", - "name": "supabase-api-key", - "optional": true - } - } - }, { "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", "valueFrom": { diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-2.snap new file mode 100644 index 000000000..f47e24e43 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-2.snap @@ -0,0 +1,171 @@ +--- +source: etl-api/src/k8s/http.rs +expression: stateful_set_json +--- +{ + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": { + "name": "abcdefghijklmnopqrst-42-replicator-stateful-set", + "namespace": "etl-data-plane" + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "template": { + "metadata": { + "annotations": { + "etl.supabase.com/restarted-at": "[timestamp]" + }, + "labels": { + "app": "etl-replicator-app", + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "APP_ENVIRONMENT", + "value": "staging" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__BIG_QUERY__SERVICE_ACCOUNT_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "service-account-key", + "name": "abcdefghijklmnopqrst-42-bq-service-account-key" + } + } + } + ], + "image": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e", + "name": "abcdefghijklmnopqrst-42-replicator", + "ports": [ + { + "containerPort": 9000, + "name": "metrics", + "protocol": "TCP" + } + ], + "volumeMounts": [ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + }, + { + "mountPath": "/app/logs", + "name": "logs" + } + ] + } + ], + "initContainers": [ + { + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "replicator-logflare-api-key" + } + } + } + ], + "image": "timberio/vector:0.46.1-distroless-libc", + "name": "abcdefghijklmnopqrst-42-vector", + "resources": { + "limits": { + "cpu": "100m", + "memory": "100Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + }, + "restartPolicy": "Always", + "volumeMounts": [ + { + "mountPath": "/etc/vector", + "name": "vector-config-file" + }, + { + "mountPath": "/var/log", + "name": "logs" + } + ] + } + ], + "nodeSelector": { + "nodeType": "workloads" + }, + "terminationGracePeriodSeconds": 300, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nodeType", + "operator": "Equal", + "value": "workloads" + } + ], + "volumes": [ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + }, + { + "configMap": { + "name": "replicator-vector-config" + }, + "name": "vector-config-file" + }, + { + "emptyDir": {}, + "name": "logs" + } + ] + } + } + } +} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-3.snap new file mode 100644 index 000000000..b622c1d7e --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json-3.snap @@ -0,0 +1,171 @@ +--- +source: etl-api/src/k8s/http.rs +expression: stateful_set_json +--- +{ + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": { + "name": "abcdefghijklmnopqrst-42-replicator-stateful-set", + "namespace": "etl-data-plane" + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "template": { + "metadata": { + "annotations": { + "etl.supabase.com/restarted-at": "[timestamp]" + }, + "labels": { + "app": "etl-replicator-app", + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "APP_ENVIRONMENT", + "value": "prod" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__BIG_QUERY__SERVICE_ACCOUNT_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "service-account-key", + "name": "abcdefghijklmnopqrst-42-bq-service-account-key" + } + } + } + ], + "image": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e", + "name": "abcdefghijklmnopqrst-42-replicator", + "ports": [ + { + "containerPort": 9000, + "name": "metrics", + "protocol": "TCP" + } + ], + "volumeMounts": [ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + }, + { + "mountPath": "/app/logs", + "name": "logs" + } + ] + } + ], + "initContainers": [ + { + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "replicator-logflare-api-key" + } + } + } + ], + "image": "timberio/vector:0.46.1-distroless-libc", + "name": "abcdefghijklmnopqrst-42-vector", + "resources": { + "limits": { + "cpu": "100m", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "500Mi" + } + }, + "restartPolicy": "Always", + "volumeMounts": [ + { + "mountPath": "/etc/vector", + "name": "vector-config-file" + }, + { + "mountPath": "/var/log", + "name": "logs" + } + ] + } + ], + "nodeSelector": { + "nodeType": "workloads" + }, + "terminationGracePeriodSeconds": 300, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nodeType", + "operator": "Equal", + "value": "workloads" + } + ], + "volumes": [ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + }, + { + "configMap": { + "name": "replicator-vector-config" + }, + "name": "vector-config-file" + }, + { + "emptyDir": {}, + "name": "logs" + } + ] + } + } + } +} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json.snap index 41a32ea95..080f553fb 100644 --- a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json.snap +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_bq_replicator_stateful_set_json.snap @@ -32,32 +32,12 @@ expression: stateful_set_json "env": [ { "name": "APP_ENVIRONMENT", - "value": "prod" + "value": "dev" }, { "name": "APP_VERSION", "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" }, - { - "name": "APP_SENTRY__DSN", - "valueFrom": { - "secretKeyRef": { - "key": "dsn", - "name": "replicator-sentry-dsn", - "optional": true - } - } - }, - { - "name": "APP_SUPABASE__API_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "key", - "name": "supabase-api-key", - "optional": true - } - } - }, { "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", "valueFrom": { @@ -90,55 +70,12 @@ expression: stateful_set_json { "mountPath": "/app/configuration", "name": "replicator-config-file" - }, - { - "mountPath": "/app/logs", - "name": "logs" } ] } ], - "initContainers": [ - { - "env": [ - { - "name": "LOGFLARE_API_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "key", - "name": "replicator-logflare-api-key" - } - } - } - ], - "image": "timberio/vector:0.46.1-distroless-libc", - "name": "abcdefghijklmnopqrst-42-vector", - "resources": { - "limits": { - "cpu": "100m", - "memory": "500Mi" - }, - "requests": { - "cpu": "100m", - "memory": "500Mi" - } - }, - "restartPolicy": "Always", - "volumeMounts": [ - { - "mountPath": "/etc/vector", - "name": "vector-config-file" - }, - { - "mountPath": "/var/log", - "name": "logs" - } - ] - } - ], - "nodeSelector": { - "nodeType": "workloads" - }, + "initContainers": [], + "nodeSelector": {}, "terminationGracePeriodSeconds": 300, "tolerations": [ { @@ -154,16 +91,6 @@ expression: stateful_set_json "name": "abcdefghijklmnopqrst-42-replicator-config" }, "name": "replicator-config-file" - }, - { - "configMap": { - "name": "replicator-vector-config" - }, - "name": "vector-config-file" - }, - { - "emptyDir": {}, - "name": "logs" } ] } diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment-2.snap new file mode 100644 index 000000000..d2ebd3ed1 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment-2.snap @@ -0,0 +1,69 @@ +--- +source: etl-api/src/k8s/http.rs +expression: container_environment +--- +[ + { + "name": "APP_ENVIRONMENT", + "value": "staging" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__CATALOG_TOKEN", + "valueFrom": { + "secretKeyRef": { + "key": "catalog-token", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_ACCESS_KEY_ID", + "valueFrom": { + "secretKeyRef": { + "key": "s3-access-key-id", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_SECRET_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "s3-secret-access-key", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment-3.snap new file mode 100644 index 000000000..266927b9a --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment-3.snap @@ -0,0 +1,69 @@ +--- +source: etl-api/src/k8s/http.rs +expression: container_environment +--- +[ + { + "name": "APP_ENVIRONMENT", + "value": "prod" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__CATALOG_TOKEN", + "valueFrom": { + "secretKeyRef": { + "key": "catalog-token", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_ACCESS_KEY_ID", + "valueFrom": { + "secretKeyRef": { + "key": "s3-access-key-id", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_SECRET_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "s3-secret-access-key", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment.snap index a2149cd9f..647e6bcb0 100644 --- a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment.snap +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_container_environment.snap @@ -5,32 +5,12 @@ expression: container_environment [ { "name": "APP_ENVIRONMENT", - "value": "prod" + "value": "dev" }, { "name": "APP_VERSION", "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" }, - { - "name": "APP_SENTRY__DSN", - "valueFrom": { - "secretKeyRef": { - "key": "dsn", - "name": "replicator-sentry-dsn", - "optional": true - } - } - }, - { - "name": "APP_SUPABASE__API_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "key", - "name": "supabase-api-key", - "optional": true - } - } - }, { "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", "valueFrom": { diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-2.snap new file mode 100644 index 000000000..ba96c0537 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-2.snap @@ -0,0 +1,189 @@ +--- +source: etl-api/src/k8s/http.rs +expression: stateful_set_json +--- +{ + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": { + "name": "abcdefghijklmnopqrst-42-replicator-stateful-set", + "namespace": "etl-data-plane" + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "template": { + "metadata": { + "annotations": { + "etl.supabase.com/restarted-at": "[timestamp]" + }, + "labels": { + "app": "etl-replicator-app", + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "APP_ENVIRONMENT", + "value": "staging" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__CATALOG_TOKEN", + "valueFrom": { + "secretKeyRef": { + "key": "catalog-token", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_ACCESS_KEY_ID", + "valueFrom": { + "secretKeyRef": { + "key": "s3-access-key-id", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_SECRET_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "s3-secret-access-key", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + } + ], + "image": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e", + "name": "abcdefghijklmnopqrst-42-replicator", + "ports": [ + { + "containerPort": 9000, + "name": "metrics", + "protocol": "TCP" + } + ], + "volumeMounts": [ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + }, + { + "mountPath": "/app/logs", + "name": "logs" + } + ] + } + ], + "initContainers": [ + { + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "replicator-logflare-api-key" + } + } + } + ], + "image": "timberio/vector:0.46.1-distroless-libc", + "name": "abcdefghijklmnopqrst-42-vector", + "resources": { + "limits": { + "cpu": "100m", + "memory": "100Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + }, + "restartPolicy": "Always", + "volumeMounts": [ + { + "mountPath": "/etc/vector", + "name": "vector-config-file" + }, + { + "mountPath": "/var/log", + "name": "logs" + } + ] + } + ], + "nodeSelector": { + "nodeType": "workloads" + }, + "terminationGracePeriodSeconds": 300, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nodeType", + "operator": "Equal", + "value": "workloads" + } + ], + "volumes": [ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + }, + { + "configMap": { + "name": "replicator-vector-config" + }, + "name": "vector-config-file" + }, + { + "emptyDir": {}, + "name": "logs" + } + ] + } + } + } +} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-3.snap new file mode 100644 index 000000000..1d4d406fb --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json-3.snap @@ -0,0 +1,189 @@ +--- +source: etl-api/src/k8s/http.rs +expression: stateful_set_json +--- +{ + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": { + "name": "abcdefghijklmnopqrst-42-replicator-stateful-set", + "namespace": "etl-data-plane" + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "template": { + "metadata": { + "annotations": { + "etl.supabase.com/restarted-at": "[timestamp]" + }, + "labels": { + "app": "etl-replicator-app", + "app-name": "abcdefghijklmnopqrst-42-replicator-app" + } + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "APP_ENVIRONMENT", + "value": "prod" + }, + { + "name": "APP_VERSION", + "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" + }, + { + "name": "APP_SENTRY__DSN", + "valueFrom": { + "secretKeyRef": { + "key": "dsn", + "name": "replicator-sentry-dsn" + } + } + }, + { + "name": "APP_SUPABASE__API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "supabase-api-key", + "optional": true + } + } + }, + { + "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "password", + "name": "abcdefghijklmnopqrst-42-postgres-password" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__CATALOG_TOKEN", + "valueFrom": { + "secretKeyRef": { + "key": "catalog-token", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_ACCESS_KEY_ID", + "valueFrom": { + "secretKeyRef": { + "key": "s3-access-key-id", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + }, + { + "name": "APP_DESTINATION__ICEBERG__SUPABASE__S3_SECRET_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "s3-secret-access-key", + "name": "abcdefghijklmnopqrst-42-iceberg" + } + } + } + ], + "image": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e", + "name": "abcdefghijklmnopqrst-42-replicator", + "ports": [ + { + "containerPort": 9000, + "name": "metrics", + "protocol": "TCP" + } + ], + "volumeMounts": [ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + }, + { + "mountPath": "/app/logs", + "name": "logs" + } + ] + } + ], + "initContainers": [ + { + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "replicator-logflare-api-key" + } + } + } + ], + "image": "timberio/vector:0.46.1-distroless-libc", + "name": "abcdefghijklmnopqrst-42-vector", + "resources": { + "limits": { + "cpu": "100m", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "500Mi" + } + }, + "restartPolicy": "Always", + "volumeMounts": [ + { + "mountPath": "/etc/vector", + "name": "vector-config-file" + }, + { + "mountPath": "/var/log", + "name": "logs" + } + ] + } + ], + "nodeSelector": { + "nodeType": "workloads" + }, + "terminationGracePeriodSeconds": 300, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "nodeType", + "operator": "Equal", + "value": "workloads" + } + ], + "volumes": [ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + }, + { + "configMap": { + "name": "replicator-vector-config" + }, + "name": "vector-config-file" + }, + { + "emptyDir": {}, + "name": "logs" + } + ] + } + } + } +} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json.snap index 92d6f40bd..2d48acbd6 100644 --- a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json.snap +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_iceberg_replicator_stateful_set_json.snap @@ -32,32 +32,12 @@ expression: stateful_set_json "env": [ { "name": "APP_ENVIRONMENT", - "value": "prod" + "value": "dev" }, { "name": "APP_VERSION", "value": "ramsup/etl-replicator:2a41356af735f891de37d71c0e1a62864fe4630e" }, - { - "name": "APP_SENTRY__DSN", - "valueFrom": { - "secretKeyRef": { - "key": "dsn", - "name": "replicator-sentry-dsn", - "optional": true - } - } - }, - { - "name": "APP_SUPABASE__API_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "key", - "name": "supabase-api-key", - "optional": true - } - } - }, { "name": "APP_PIPELINE__PG_CONNECTION__PASSWORD", "valueFrom": { @@ -108,55 +88,12 @@ expression: stateful_set_json { "mountPath": "/app/configuration", "name": "replicator-config-file" - }, - { - "mountPath": "/app/logs", - "name": "logs" } ] } ], - "initContainers": [ - { - "env": [ - { - "name": "LOGFLARE_API_KEY", - "valueFrom": { - "secretKeyRef": { - "key": "key", - "name": "replicator-logflare-api-key" - } - } - } - ], - "image": "timberio/vector:0.46.1-distroless-libc", - "name": "abcdefghijklmnopqrst-42-vector", - "resources": { - "limits": { - "cpu": "100m", - "memory": "500Mi" - }, - "requests": { - "cpu": "100m", - "memory": "500Mi" - } - }, - "restartPolicy": "Always", - "volumeMounts": [ - { - "mountPath": "/etc/vector", - "name": "vector-config-file" - }, - { - "mountPath": "/var/log", - "name": "logs" - } - ] - } - ], - "nodeSelector": { - "nodeType": "workloads" - }, + "initContainers": [], + "nodeSelector": {}, "terminationGracePeriodSeconds": 300, "tolerations": [ { @@ -172,16 +109,6 @@ expression: stateful_set_json "name": "abcdefghijklmnopqrst-42-replicator-config" }, "name": "replicator-config-file" - }, - { - "configMap": { - "name": "replicator-vector-config" - }, - "name": "vector-config-file" - }, - { - "emptyDir": {}, - "name": "logs" } ] } diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers-2.snap new file mode 100644 index 000000000..9220692de --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers-2.snap @@ -0,0 +1,42 @@ +--- +source: etl-api/src/k8s/http.rs +expression: node_selector +--- +[ + { + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "replicator-logflare-api-key" + } + } + } + ], + "image": "timberio/vector:0.46.1-distroless-libc", + "name": "abcdefghijklmnopqrst-42-vector", + "resources": { + "limits": { + "cpu": "100m", + "memory": "100Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + }, + "restartPolicy": "Always", + "volumeMounts": [ + { + "mountPath": "/etc/vector", + "name": "vector-config-file" + }, + { + "mountPath": "/var/log", + "name": "logs" + } + ] + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers-3.snap new file mode 100644 index 000000000..13e8768bf --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers-3.snap @@ -0,0 +1,42 @@ +--- +source: etl-api/src/k8s/http.rs +expression: node_selector +--- +[ + { + "env": [ + { + "name": "LOGFLARE_API_KEY", + "valueFrom": { + "secretKeyRef": { + "key": "key", + "name": "replicator-logflare-api-key" + } + } + } + ], + "image": "timberio/vector:0.46.1-distroless-libc", + "name": "abcdefghijklmnopqrst-42-vector", + "resources": { + "limits": { + "cpu": "100m", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "500Mi" + } + }, + "restartPolicy": "Always", + "volumeMounts": [ + { + "mountPath": "/etc/vector", + "name": "vector-config-file" + }, + { + "mountPath": "/var/log", + "name": "logs" + } + ] + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers.snap new file mode 100644 index 000000000..de0f6ff2b --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_init_containers.snap @@ -0,0 +1,5 @@ +--- +source: etl-api/src/k8s/http.rs +expression: node_selector +--- +[] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector-2.snap new file mode 100644 index 000000000..a2c9d3df5 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector-2.snap @@ -0,0 +1,7 @@ +--- +source: etl-api/src/k8s/http.rs +expression: node_selector +--- +{ + "nodeType": "workloads" +} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector-3.snap new file mode 100644 index 000000000..a2c9d3df5 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector-3.snap @@ -0,0 +1,7 @@ +--- +source: etl-api/src/k8s/http.rs +expression: node_selector +--- +{ + "nodeType": "workloads" +} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector.snap new file mode 100644 index 000000000..8202f56e0 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_node_selector.snap @@ -0,0 +1,5 @@ +--- +source: etl-api/src/k8s/http.rs +expression: node_selector +--- +{} diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts-2.snap new file mode 100644 index 000000000..901ffd078 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts-2.snap @@ -0,0 +1,14 @@ +--- +source: etl-api/src/k8s/http.rs +expression: volume_mounts +--- +[ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + }, + { + "mountPath": "/app/logs", + "name": "logs" + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts-3.snap new file mode 100644 index 000000000..901ffd078 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts-3.snap @@ -0,0 +1,14 @@ +--- +source: etl-api/src/k8s/http.rs +expression: volume_mounts +--- +[ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + }, + { + "mountPath": "/app/logs", + "name": "logs" + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts.snap new file mode 100644 index 000000000..117198001 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volume_mounts.snap @@ -0,0 +1,10 @@ +--- +source: etl-api/src/k8s/http.rs +expression: volume_mounts +--- +[ + { + "mountPath": "/app/configuration", + "name": "replicator-config-file" + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes-2.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes-2.snap new file mode 100644 index 000000000..9f4d6604e --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes-2.snap @@ -0,0 +1,22 @@ +--- +source: etl-api/src/k8s/http.rs +expression: volumes +--- +[ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + }, + { + "configMap": { + "name": "replicator-vector-config" + }, + "name": "vector-config-file" + }, + { + "emptyDir": {}, + "name": "logs" + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes-3.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes-3.snap new file mode 100644 index 000000000..9f4d6604e --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes-3.snap @@ -0,0 +1,22 @@ +--- +source: etl-api/src/k8s/http.rs +expression: volumes +--- +[ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + }, + { + "configMap": { + "name": "replicator-vector-config" + }, + "name": "vector-config-file" + }, + { + "emptyDir": {}, + "name": "logs" + } +] diff --git a/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes.snap b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes.snap new file mode 100644 index 000000000..fcc1a45d7 --- /dev/null +++ b/etl-api/src/k8s/snapshots/etl_api__k8s__http__tests__create_volumes.snap @@ -0,0 +1,12 @@ +--- +source: etl-api/src/k8s/http.rs +expression: volumes +--- +[ + { + "configMap": { + "name": "abcdefghijklmnopqrst-42-replicator-config" + }, + "name": "replicator-config-file" + } +] diff --git a/etl-api/src/startup.rs b/etl-api/src/startup.rs index 0f5148005..a3b87c79b 100644 --- a/etl-api/src/startup.rs +++ b/etl-api/src/startup.rs @@ -5,16 +5,18 @@ use actix_web_httpauth::middleware::HttpAuthentication; use actix_web_metrics::ActixWebMetricsBuilder; use aws_lc_rs::aead::{AES_256_GCM, RandomizedNonceKey}; use base64::{Engine, prelude::BASE64_STANDARD}; +use etl_config::Environment; use etl_config::shared::{IntoConnectOptions, PgConnectionConfig}; use etl_telemetry::metrics::init_metrics_handle; +use kube::config::KubeConfigOptions; use sqlx::{PgPool, postgres::PgPoolOptions}; -use tracing::warn; +use tracing::{error, info, warn}; use tracing_actix_web::TracingLogger; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; -use crate::k8s::K8sClient; use crate::k8s::http::HttpK8sClient; +use crate::k8s::{K8sClient, K8sError}; use crate::{ authentication::auth_validator, config::ApiConfig, @@ -99,7 +101,24 @@ impl Application { key, }; - let k8s_client = match HttpK8sClient::new().await { + let kube_client = match Environment::load()? { + Environment::Staging | Environment::Prod => kube::Client::try_default().await?, + Environment::Dev => { + let options = KubeConfigOptions { + context: Some("orbstack".to_string()), + cluster: Some("orbstack".to_string()), + user: Some("orbstack".to_string()), + }; + let kube_config = kube::config::Config::from_kubeconfig(&options).await?; + let kube_client: kube::Client = kube_config.try_into()?; + + test_orbstack_connection(&kube_client).await?; + + kube_client + } + }; + + let k8s_client = match HttpK8sClient::new(kube_client).await { Ok(client) => Some(Arc::new(client) as Arc), Err(e) => { warn!( @@ -144,6 +163,25 @@ impl Application { } } +async fn test_orbstack_connection(client: &kube::Client) -> Result<(), K8sError> { + match client.apiserver_version().await { + Ok(version) => { + info!( + "successfully connected to orbstack kubernetes api server version: {}.{}", + version.major, version.minor + ); + } + Err(e) => { + error!( + "failed to connect to orbstack, make sure you have orbstack installed and kubernetes enabled in it." + ); + return Err(e.into()); + } + } + + Ok(()) +} + /// Creates a Postgres connection pool from the provided configuration. pub fn get_connection_pool(config: &PgConnectionConfig) -> PgPool { PgPoolOptions::new().connect_lazy_with(config.with_db()) diff --git a/scripts/etl-data-plane.yaml b/scripts/etl-data-plane.yaml new file mode 100644 index 000000000..c44c083e1 --- /dev/null +++ b/scripts/etl-data-plane.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: etl-data-plane diff --git a/scripts/init.sh b/scripts/init.sh index aaf12037a..b65d6ddd0 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -13,6 +13,13 @@ if ! [ -x "$(command -v docker-compose)" ]; then exit 1 fi +# kubectl check +if ! [ -x "$(command -v kubectl)" ]; then + echo >&2 "❌ Error: kubectl is not installed." + echo >&2 "Please install kubectl: https://kubernetes.io/docs/tasks/tools/" + exit 1 +fi + # Database configuration echo "πŸ”§ Configuring database settings..." DB_USER="${POSTGRES_USER:=postgres}" @@ -64,4 +71,23 @@ echo "πŸ”„ Running database migrations..." SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" bash "${SCRIPT_DIR}/../etl-api/scripts/run_migrations.sh" +# Seed default replicator image (idempotent). +echo "πŸ–ΌοΈ Seeding default replicator image..." +DEFAULT_REPLICATOR_IMAGE="${REPLICATOR_IMAGE:-ramsup/replicator:0.0.22}" +psql "${DATABASE_URL}" -v ON_ERROR_STOP=1 -c "\ +insert into app.images (name, is_default) \ +select '${DEFAULT_REPLICATOR_IMAGE}', true \ +where not exists (select 1 from app.images where name = '${DEFAULT_REPLICATOR_IMAGE}');" + +# Ensure OrbStack Kubernetes context is available +if ! kubectl config get-contexts -o name | grep -qx "orbstack"; then + echo >&2 "❌ Error: Kubernetes context 'orbstack' not found." + echo >&2 "Please install OrbStack (https://orbstack.dev) and enable Kubernetes in its settings." + exit 1 +fi + +echo "☸️ Configuring Kubernetes environment..." +kubectl --context orbstack apply -f "${SCRIPT_DIR}/etl-data-plane.yaml" +kubectl --context orbstack apply -f "${SCRIPT_DIR}/trusted-root-certs-config.yaml" + echo "✨ Complete development environment setup finished! Ready to go!" diff --git a/scripts/trusted-root-certs-config.yaml b/scripts/trusted-root-certs-config.yaml new file mode 100644 index 000000000..a4013c805 --- /dev/null +++ b/scripts/trusted-root-certs-config.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trusted-root-certs-config + namespace: etl-data-plane +data: + trusted_root_certs: |- + -----BEGIN CERTIFICATE----- + MIID1DCCArygAwIBAgIUbYRdq/8/uNq8G9stMCdOFSBgA2MwDQYJKoZIhvcNAQEL + BQAwczELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0RlbHdhcmUxEzARBgNVBAcMCk5l + dyBDYXN0bGUxFTATBgNVBAoMDFN1cGFiYXNlIEluYzEmMCQGA1UEAwwdU3VwYWJh + c2UgU3RhZ2luZyBSb290IDIwMjEgQ0EwHhcNMjEwNDI4MTAzNjEzWhcNMzEwNDI2 + MTAzNjEzWjBzMQswCQYDVQQGEwJVUzEQMA4GA1UECAwHRGVsd2FyZTETMBEGA1UE + BwwKTmV3IENhc3RsZTEVMBMGA1UECgwMU3VwYWJhc2UgSW5jMSYwJAYDVQQDDB1T + dXBhYmFzZSBTdGFnaW5nIFJvb3QgMjAyMSBDQTCCASIwDQYJKoZIhvcNAQEBBQAD + ggEPADCCAQoCggEBAN0AKRE8a56O8LaZxiOAcHFUFnwiKUvPoXPq26Ifw+Nv+7zg + N2V5WnMZbbw24q61Os60ZUn0XmbVtuIeJ+stPHsO7qxxuL+bmPR+qU5tkDrIOyEe + YD/2u8/q6ssVv42k4XcXbhM6RVz7CkCDY0TiBm1bMtRZso3xB6E9wAjxDf43XfV5 + PAGs3JI+Zo/vyqCDlN0hHOrB/aBl01JXqQWI84Gia5ooucq4SjA1CyawBcQ2IAvG + rXuy1BouY+xM3zRuNvtfFP6rb5Mta+jCYEMh1AZ8yP8sYUWAyhxX6k9EbOb009wQ + aZljbUCh/UglGWuBxdzePavx+zPjzWXB1NyVkpkCAwEAAaNgMF4wCwYDVR0PBAQD + AgEGMB0GA1UdDgQWBBQFx+PHLf27iIo/PMfIfGqXF7Zb+DAfBgNVHSMEGDAWgBQF + x+PHLf27iIo/PMfIfGqXF7Zb+DAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEB + CwUAA4IBAQB/xIiz5dDqzGXjqYqXZYx4iSfSxsVayeOPDMfmaiCfSMJEUG4cUiwG + OvMPGztaUEYeip5SCvSKuAAjVkXyP7ahKR7t7lZ9mErVXyxSZoVLbOd578CuYiZk + OgT17UjPv66WMzEKEr8wGpomTYWWfEkuqt8ENdiM1Z4LNFahdKj36+jm6/a+9R8K + 25VIL68DTaQpBxFWG6ixC1HRMHJ12lDhKsshIi099BVpkGibESlxPrQOdKKqBB/J + vIX+/Hb+mS4H5zYMeK2wX0onp+GBcD6X9L1UJuXMVd+BRan8RFidXL5s3++xXjQq + Nzbc6lnA69urKffvcT07YwMsY/OmHzVa + -----END CERTIFICATE-----