fix(satellite): use latest version of satellite (read-only db)

Author: luabida

alertflow/airflow.cfg

@@ -534,7 +534,7 @@ default_hive_mapred_queue =
 # The base url of your website as airflow cannot guess what domain or
 # cname you are using. This is used in automated emails that
 # airflow sends to point links to the right web server
-base_url = http://localhost:8080
+base_url = http://localhost:8080/alertflow
 
 # Default timezone to display all dates in the UI, can be UTC, system, or
 # any IANA timezone string (e.g. Europe/Amsterdam). If left empty the
@@ -585,7 +585,7 @@ reload_on_plugin_change = False
 # The token generated using the secret key has a short expiry time though - make sure that time on
 # ALL the machines that you run airflow components on is synchronized (for example using ntpd)
 # otherwise you might get "forbidden" errors when the logs are accessed.
-# secret_key =
+secret_key = $AIRFLOW__WEBSERVER__SECRET_KEY
 
 # Number of workers to run the Gunicorn web server
 workers = 4

alertflow/dags/satellite-weather/brasil.py

@@ -45,7 +45,7 @@
     dag_id="COPERNICUS_BRASIL",
     description="ETL of weather data for Brazil",
     tags=["Brasil", "Copernicus"],
-    schedule="@monthly",
+    schedule="@daily",
     default_args=DEFAULT_ARGS,
     start_date=pendulum.datetime(2000, 1, 1),
     catchup=True,
@@ -75,6 +75,9 @@ def fetch_ds(locale, dt, uri, api_key):
         print("TABLE_GEO ", f"[{len(table_geocodes)}]: ", table_geocodes)
         print("DIFF_GEO: ", f"[{len(geocodes)}]: ", geocodes)
 
+        if not geocodes:
+            return
+
         basename = str(dt).replace("-", "_") + locale
         with request.reanalysis_era5_land(
             basename,
@@ -83,12 +86,13 @@ def fetch_ds(locale, dt, uri, api_key):
             locale=locale,
         ) as ds:
             for geocode in geocodes:
-                adm = ADM2.get(code=geocode):
+                adm = ADM2.get(code=geocode, adm0=locale)
                 with engine.connect() as conn:
                     ds.cope.to_sql(adm, conn, tablename, "weather")
-            file = Path(f"{basename}.zip")
-            if file.exists():
-                file.unlink()
-                print(f"{file} removed")
+
+        file = Path(f"{basename}.zip")
+        if file.exists():
+            file.unlink()
+            print(f"{file} removed")
 
     fetch_ds("BRA", DATE, URI["PSQL_MAIN_URI"], KEY["CDSAPI_KEY"])

conda/env.yaml

@@ -7,3 +7,4 @@ dependencies:
   - pip
   - pre-commit
   - poetry >= 1.7.1
+  - docker-compose

docker/compose.yaml

@@ -28,10 +28,12 @@ x-airflow-common:
   environment:
     &airflow-common-env
     AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
+    AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
     AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
     AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow}
     AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}'
     AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}'
+
     HOST_UID: ${HOST_UID}
     HOST_GID: ${HOST_GID}
   volumes:
@@ -170,17 +172,17 @@ services:
       timeout: 10s
       retries: 5
       start_period: 30s
-    environment:
-      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
-      AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
-      AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow}
-      AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}'
-      AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}'
-      HOST_UID: ${HOST_UID}
-      HOST_GID: ${HOST_GID}
-      # Required to handle warm shutdown of the celery workers properly
-      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
-      DUMB_INIT_SETSID: "0"
+    # environment:
+    #   AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
+    #   AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
+    #   AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow}
+    #   AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}'
+    #   AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}'
+    #   HOST_UID: ${HOST_UID}
+    #   HOST_GID: ${HOST_GID}
+    #   # Required to handle warm shutdown of the celery workers properly
+    #   # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
+    #   DUMB_INIT_SETSID: "0"
     restart: always
     depends_on:
       redis: