diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 48a4ee0..2d77de0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,16 +64,14 @@ jobs: with: convention-name: conventionalcommits - - name: Install Conda Dependencies - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@v3 with: - miniconda-version: "latest" - mamba-version: "*" + miniforge-version: latest environment-file: conda/env.yaml channels: conda-forge,nodefaults activate-environment: alertflow - use-mamba: true - miniforge-variant: Mambaforge + auto-update-conda: true + conda-solver: libmamba - name: Create environment variables file run: | @@ -93,11 +91,11 @@ jobs: - name: Build Base Image run: | - sugar build + sugar compose build - name: Start Airflow run: | - sugar up --options -d + sugar compose up --options -d - name: Wait Containers run: | diff --git a/.sugar.yaml b/.sugar.yaml index 3027e29..70b95a8 100644 --- a/.sugar.yaml +++ b/.sugar.yaml @@ -1,13 +1,11 @@ -version: 1.0.0 -compose-app: docker-compose +backend: compose env-file: .env defaults: - group: {{ env.ENV }} - + group: dev groups: dev: project-name: alertflow - compose-path: + config-path: - docker/compose.yaml - docker/compose-dev.yaml env-file: .env @@ -21,7 +19,7 @@ groups: prod: project-name: alertflow - compose-path: + config-path: - docker/compose.yaml - docker/compose-prod.yaml env-file: .env diff --git a/alertflow/dags/satellite-weather/brasil.py b/alertflow/dags/satellite-weather/brasil.py index b280147..146e0ec 100644 --- a/alertflow/dags/satellite-weather/brasil.py +++ b/alertflow/dags/satellite-weather/brasil.py @@ -16,9 +16,9 @@ """ import os -from pathlib import Path from datetime import date, timedelta from itertools import chain +from pathlib import Path import pendulum from airflow import DAG @@ -45,7 +45,7 @@ dag_id="COPERNICUS_BRASIL", description="ETL of weather data for Brazil", tags=["Brasil", "Copernicus"], - schedule="@monthly", + schedule="@daily", default_args=DEFAULT_ARGS, start_date=pendulum.datetime(2000, 1, 1), catchup=True, @@ -70,7 +70,9 @@ def fetch_ds(locale, dt, uri, api_key): ) table_geocodes = set(chain(*cur.fetchall())) - all_geocodes = set([adm.code for adm in ADM2.filter(adm0=locale)]) + all_geocodes = set( + map(int, [adm.code for adm in ADM2.filter(adm0=locale)]) + ) geocodes = all_geocodes.difference(table_geocodes) print("TABLE_GEO ", f"[{len(table_geocodes)}]: ", table_geocodes) print("DIFF_GEO: ", f"[{len(geocodes)}]: ", geocodes) @@ -83,7 +85,7 @@ def fetch_ds(locale, dt, uri, api_key): locale=locale, ) as ds: for geocode in geocodes: - adm = ADM2.get(code=geocode): + adm = ADM2.get(code=geocode) with engine.connect() as conn: ds.cope.to_sql(adm, conn, tablename, "weather") file = Path(f"{basename}.zip") diff --git a/docker/Dockerfile b/docker/Dockerfile index 1655c3e..5c227e1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,12 +44,14 @@ RUN addgroup --gid ${HOST_GID} airflow \ && chmod 0440 /etc/sudoers.d/airflow \ && chown -R ${HOST_UID}:${HOST_GID} ${AIRFLOW_HOME}/ /opt/airflow/ +RUN groupadd docker && usermod -aG docker airflow && newgrp docker + ENV PATH "$PATH:/home/airflow/.local/bin" ENV PATH "$PATH:/usr/bin/dirname" COPY --chown=airflow alertflow/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg COPY --chown=airflow docker/scripts/entrypoint.sh /entrypoint.sh -COPY --chown=airflow pyproject.toml ${AIRFLOW_HOME} +COPY --chown=airflow pyproject.toml README.md ${AIRFLOW_HOME} RUN chmod +x /entrypoint.sh USER airflow @@ -59,6 +61,6 @@ RUN curl -sSL https://install.python-poetry.org | python3 WORKDIR ${AIRFLOW_HOME} RUN poetry config virtualenvs.create false \ - && poetry install --only main + && poetry install --only main --no-root ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/compose.yaml b/docker/compose.yaml index 1c26355..46a9ed0 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -189,6 +189,8 @@ services: condition: service_healthy startup: condition: service_completed_successfully + volumes: + - /var/run/docker.sock:/var/run/docker.sock networks: - alertflow diff --git a/pyproject.toml b/pyproject.toml index d0d20e7..db20e3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,7 @@ sqlalchemy = "<2.0" flake8 = "^7.0.0" black = "^24.2.0" isort = "^5.13.2" -containers-sugar = "1.11.1" -compose-go = "^2.24.6" +containers-sugar = "^1.16.1" apache-airflow = "^2.10.2" [build-system]