From d735d3516644c131659af9caa8458ea9150dc238 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 1 May 2024 16:31:02 +0100 Subject: [PATCH] adding helm output file --- 1.chunkedgraph.md | 8 +- terraform/helm_config.tf | 8 ++ terraform/helm_values.tpl | 168 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 terraform/helm_config.tf create mode 100644 terraform/helm_values.tpl diff --git a/1.chunkedgraph.md b/1.chunkedgraph.md index 0a3abb7..f412ebb 100644 --- a/1.chunkedgraph.md +++ b/1.chunkedgraph.md @@ -48,7 +48,7 @@ terraform plan terraform apply ``` -This will output some variables useful for next steps: +This will output some variables useful for next steps. ``` kubernetes_cluster_context = "gcloud container clusters get-credentials chunkedgraph-ingest --zone us-east1-b --project neuromancer-seung-import" @@ -59,8 +59,9 @@ region = "us-east1" zone = "us-east1-b" ``` +it will also pregenerate a yaml file to be the start of your values file, which will be placed in helm/config/{common_name}.yaml based on the common_name you configured in your terraform.tfvars file. + Use value of `kubernetes_cluster_context` to connect to your cluster. -Use value of `redis_host` in `helm/pychunkedgraph/values.yaml` (more info in Helm section). You can also look these up again with `terraform show` from within the `terraform/` directory. @@ -126,6 +127,7 @@ image: ## Helm ([docs](https://helm.sh/docs/)) `helm` is used to run the ingest. The provided chart installs kubernetes resources such as configmaps, secrets, deployments needed to run the ingest. Refer to example `helm/pychunkedgraph/example_values.yaml` file for more information. +Terraform will generate a values file for you that reflects the values you configured in your terraform.tfvars file in helm/config/{common_name}.yaml. IMPORTANT: If you have a large dataset to ingest, it is recommended to do this layer by layer. See [scaling](#scaling). @@ -133,7 +135,7 @@ IMPORTANT: If you have a large dataset to ingest, it is recommended to do this l ### Chart Installation -When all variables are ready, rename your values file to `values.yaml` (ignored by git because it can contain sensitive information). If a different name is preferred (for different datasets/project), use the format `values*.[yml|yaml]` which will also be ignored by git. Then the file name will need to be explicitly passed to `helm install` with `-f `. +When all variables are ready, rename your values file to `values.yaml` (ignored by git because it can contain sensitive information). If a different name is preferred (for different datasets/project), use the format `values*.[yml|yaml]` which will also be ignored by git. Then the file name will need to be explicitly passed to `helm install` with `-f `. Note, if you rerun terraform without changing the common_name then your old values file will get overwritten by terraform. Then run: diff --git a/terraform/helm_config.tf b/terraform/helm_config.tf new file mode 100644 index 0000000..76ebdac --- /dev/null +++ b/terraform/helm_config.tf @@ -0,0 +1,8 @@ +resource "local_file" "helm_values" { + filename = "${path.module}/../helm/config/${var.common_name}.yaml" + content = templatefile("${path.module}/helm_values.tpl", { + redis_host = google_redis_instance.redis.host + google_project = var.project_id + bigtable_instance = google_bigtable_instance.instance.id + }) +} diff --git a/terraform/helm_values.tpl b/terraform/helm_values.tpl new file mode 100644 index 0000000..adc5d77 --- /dev/null +++ b/terraform/helm_values.tpl @@ -0,0 +1,168 @@ +env: + - name: &commonEnvVars "pychunkedgraph" + vars: + REDIS_HOST: "${redis_host}" + REDIS_PORT: 6379 + REDIS_PASSWORD: "" + BIGTABLE_PROJECT: &bt_project "${google_project}" + BIGTABLE_INSTANCE: &bt_instance "${bigtable_instance}" + GOOGLE_APPLICATION_CREDENTIALS: /root/.cloudvolume/secrets/google-secret.json + SHELL: /bin/bash + FLASK_APP: run_dev.py + APP_SETTINGS: pychunkedgraph.app.config.DeploymentWithRedisConfig + +configfiles: +- name: &bashrc "bashrc" + files: + ".bashrc": |- + alias watch='watch ' + alias ingest='flask ingest' + alias rqx='flask rq' + +configyamls: +- name: &dataset test + files: + - name: test.yml + content: + data_source: + EDGES: "" + COMPONENTS: "" + WATERSHED: "" + + graph_config: + CHUNK_SIZE: [] # [X, Y, Z] + FANOUT: 2 + SPATIAL_BITS: 2 + LAYER_ID_BITS: 8 + + backend_client: + TYPE: "bigtable" + CONFIG: + ADMIN: true + READ_ONLY: false + PROJECT: *bt_project + INSTANCE: *bt_instance + +secrets: +- name: &cloudVolumeSecrets cloud-volume-secrets + files: + # these are used by python bigtable client and cloud-files + # must have the following permissions: + # * read gcs objects if edges/component files are stored in google cloud buckets + # if they're stored elsewhere use the secrets with appropriate permissions accordingly + # * bigtable - create and read tables + google-secret.json: |- + { + + } + +deployments: +- enabled: true + name: &name master + nodeSelector: + cloud.google.com/gke-nodepool: master + hpa: + enabled: false + volumes: &commonVolumes + - name: *cloudVolumeSecrets + secret: + secretName: *cloudVolumeSecrets + - name: &datasetsVolume datasets-volume + configMap: + name: *dataset + - name: &bashrcVolume bashrc-volume + configMap: + name: *bashrc + containers: + - name: *name + image: &image + repository: + tag: "" + volumeMounts: &commonVolumeMounts + - name: *cloudVolumeSecrets + mountPath: /root/.cloudvolume/secrets + readOnly: true + - name: *datasetsVolume + mountPath: /app/datasets + - name: *bashrcVolume + mountPath: /root/ + envFromConfigMap: + - *commonEnvVars + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: + requests: + memory: 500M + +workerDeployments: +- enabled: true + name: &name l2 + nodeSelector: + cloud.google.com/gke-nodepool: low + hpa: + enabled: false + volumes: *commonVolumes + containers: + - name: *name + command: [rq, worker, *name] + image: *image + volumeMounts: *commonVolumeMounts + envFromConfigMap: + - *commonEnvVars + resources: + requests: + memory: 1G + +- enabled: true + name: &name l3 + nodeSelector: + cloud.google.com/gke-nodepool: low + hpa: + enabled: false + volumes: *commonVolumes + containers: + - name: *name + command: [rq, worker, *name] + image: *image + volumeMounts: *commonVolumeMounts + envFromConfigMap: + - *commonEnvVars + resources: + requests: + memory: 1.5G + +- enabled: true + name: &name l4 + nodeSelector: + cloud.google.com/gke-nodepool: low + hpa: + enabled: false + volumes: *commonVolumes + containers: + - name: *name + command: [rq, worker, *name] + image: *image + volumeMounts: *commonVolumeMounts + envFromConfigMap: + - *commonEnvVars + resources: + requests: + memory: 2G + +trackerDeployments: + count: 4 # number of layers in the chunkedgraph + enabled: false + volumes: *commonVolumes + hpa: + enabled: false + containers: + - image: *image + volumeMounts: *commonVolumeMounts + env: + - name: *commonEnvVars + resources: + requests: + memory: 100M \ No newline at end of file