feat(DoutD): enable worker to spawn containers on host (ADAnalise-container)

Author: luabida

.github/workflows/build.yaml

@@ -64,16 +64,14 @@ jobs:
       with:
         convention-name: conventionalcommits
 
-    - name: Install Conda Dependencies
-      uses: conda-incubator/setup-miniconda@v3
+    - uses: conda-incubator/setup-miniconda@v3
       with:
-        miniconda-version: "latest"
-        mamba-version: "*"
+        miniforge-version: latest
         environment-file: conda/env.yaml
         channels: conda-forge,nodefaults
         activate-environment: alertflow
-        use-mamba: true
-        miniforge-variant: Mambaforge
+        auto-update-conda: true
+        conda-solver: libmamba
 
     - name: Create environment variables file
       run: |
@@ -93,11 +91,11 @@ jobs:
 
     - name: Build Base Image
       run: |
-        sugar build
+        sugar compose build
 
     - name: Start Airflow
       run: |
-        sugar up --options -d
+        sugar compose up --options -d
 
     - name: Wait Containers
       run: |

.sugar.yaml

@@ -1,13 +1,11 @@
-version: 1.0.0
-compose-app: docker-compose
+backend: compose
 env-file: .env
 defaults:
-  group: {{ env.ENV }}
-
+  group: dev
 groups:
   dev:
     project-name: alertflow
-    compose-path:
+    config-path:
       - docker/compose.yaml
       - docker/compose-dev.yaml
     env-file: .env
@@ -21,7 +19,7 @@ groups:
 
   prod:
     project-name: alertflow
-    compose-path: 
+    config-path:
       - docker/compose.yaml
       - docker/compose-prod.yaml
     env-file: .env

alertflow/dags/satellite-weather/brasil.py

@@ -16,9 +16,9 @@
 """
 
 import os
-from pathlib import Path
 from datetime import date, timedelta
 from itertools import chain
+from pathlib import Path
 
 import pendulum
 from airflow import DAG
@@ -83,7 +83,7 @@ def fetch_ds(locale, dt, uri, api_key):
             locale=locale,
         ) as ds:
             for geocode in geocodes:
-                adm = ADM2.get(code=geocode):
+                adm = ADM2.get(code=geocode)
                 with engine.connect() as conn:
                     ds.cope.to_sql(adm, conn, tablename, "weather")
             file = Path(f"{basename}.zip")

docker/Dockerfile

@@ -44,12 +44,14 @@ RUN addgroup --gid ${HOST_GID} airflow \
   && chmod 0440 /etc/sudoers.d/airflow \
   && chown -R ${HOST_UID}:${HOST_GID} ${AIRFLOW_HOME}/ /opt/airflow/
 
+RUN groupadd docker && usermod -aG docker airflow && newgrp docker
+
 ENV PATH "$PATH:/home/airflow/.local/bin"
 ENV PATH "$PATH:/usr/bin/dirname"
 
 COPY --chown=airflow alertflow/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
 COPY --chown=airflow docker/scripts/entrypoint.sh /entrypoint.sh
-COPY --chown=airflow pyproject.toml ${AIRFLOW_HOME}
+COPY --chown=airflow pyproject.toml README.md ${AIRFLOW_HOME}
 RUN chmod +x /entrypoint.sh
 
 USER airflow
@@ -59,6 +61,6 @@ RUN curl -sSL https://install.python-poetry.org | python3
 WORKDIR ${AIRFLOW_HOME}
 
 RUN poetry config virtualenvs.create false \
-  && poetry install --only main
+  && poetry install --only main --no-root
 
 ENTRYPOINT [ "/entrypoint.sh" ]

docker/compose.yaml

@@ -189,6 +189,8 @@ services:
         condition: service_healthy
       startup:
         condition: service_completed_successfully
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - alertflow
 

poetry.lock

@@ -15,8 +15,7 @@ sqlalchemy = "<2.0"
 flake8 = "^7.0.0"
 black = "^24.2.0"
 isort = "^5.13.2"
-containers-sugar = "1.11.1"
-compose-go = "^2.24.6"
+containers-sugar = "^1.16.1"
 apache-airflow = "^2.10.2"
 
 [build-system]