feat(DENG-9188) Add BigConfig checks to targeted stable tables

bqetl_project.yaml

@@ -26,6 +26,9 @@ monitoring:
   - 463
   - 508
   - 509
+  stable_tables_targeted:
+    mozilla_vpn_external:
+    - waitlist_v1
 
 dry_run:
   function: https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/bigquery-etl-dryrun

sql_generators/stable_tables_targeted/README.md

@@ -0,0 +1,22 @@
+# Stable Tables Targeted
+
+This generator builds the BigEye `biconfig.yml` and `metadata.yaml` files for targeted stable tables. The reason for this appproach is to mitigate any potential negative impacts should there be increased future costs associated with freshness and volume checks.
+
+## Conifguration
+
+Modify the `bqetl_project.yaml` file. Example:
+
+```yaml
+monitoring:
+  stable_tables_targeted:
+    mozilla_vpn_external:
+    - waitlist_v1
+```
+
+## Running the generator
+
+To test this generator run `./bqetl generate stable_tables_targeted`.
+
+Output:
+
+<img width="442" height="258" alt="Screenshot 2025-11-17 at 10 46 34 AM" src="https://github.com/user-attachments/assets/13218167-fcd1-4fed-bbf2-d32537aad872" />

sql_generators/stable_tables_targeted/__init__.py

@@ -0,0 +1,27 @@
+"""Generate metadata and bigconfig files for targeted stable tables."""
+
+import click
+
+from bigquery_etl.cli.utils import is_valid_project
+from sql_generators.stable_tables_targeted.stable_tables_targeted import (
+    generate_stable_table_bigconfig_files,
+)
+
+
+@click.command()
+@click.option(
+    "--target-project",
+    "--target_project",
+    help="GCP project ID",
+    default="moz-fx-data-shared-prod",
+    callback=is_valid_project,
+)
+@click.option(
+    "--enable-monitoring",
+    "--enable_monitoring",
+    help="Monitoring enabled true or false",
+    default=True,
+)
+def generate(target_project, enable_monitoring, **kwargs):
+    """Call generate_mobile_kpi_support_metrics function."""
+    generate_stable_table_bigconfig_files(target_project, enable_monitoring)

sql_generators/stable_tables_targeted/stable_tables_targeted.py

@@ -0,0 +1,76 @@
+"""Generate metadata and bigconfig files for targeted stable tables."""
+
+from pathlib import Path
+
+import yaml
+from jinja2 import Environment, FileSystemLoader
+
+
+def load_yaml_file(file_path):
+    """Load bqetl project yaml file to get stable table datasets and table names."""
+    with open(file_path, "r") as file:
+        return yaml.safe_load(file)
+
+
+def write_file(content, file_path):
+    """Write metadata.yaml and bigconfig.yml to correct directories."""
+    with open(file_path, "w") as file:
+        file.write(content)
+
+
+def parse_config_name(config):
+    """Parse config name into name and version parts."""
+    if "_" in config:
+        return config.rsplit("_", 1)
+    return config, "v1"
+
+
+def generate_stable_table_bigconfig_files(target_project, enable_monitoring):
+    """Generate the metadata and bigconfig files and write to correct directories."""
+    THIS_PATH = Path(__file__).parent
+    TEMPLATES_DIR = THIS_PATH / "templates"
+    SQL_BASE_DIR = THIS_PATH.parent.parent / "sql" / target_project
+
+    BIGEYE_COLLECTION = "Operational Checks"
+    BIGEYE_SLACK_CHANNEL = "#de-bigeye-triage"
+
+    env = Environment(loader=FileSystemLoader(str(TEMPLATES_DIR)))
+    bigconfig_template = env.get_template("stable_tables_targeted.bigconfig.yml")
+    metadata_template = env.get_template("stable_tables_targeted.metadata.yaml")
+
+    project_config = load_yaml_file("bqetl_project.yaml")
+    stable_table_bigconfigs = project_config.get("monitoring", {}).get(
+        "stable_tables_targeted", {}
+    )
+
+    common_metadata_vars = {
+        "bigeye_collection": BIGEYE_COLLECTION,
+        "enable_monitoring": enable_monitoring,
+    }
+
+    # Create directory for each dataset
+    for dataset_name, table_name in stable_table_bigconfigs.items():
+
+        target_dir = SQL_BASE_DIR / dataset_name
+        target_dir.mkdir(parents=True, exist_ok=True)
+
+        # Create directory for each table each with a metadata.yaml and bigconfig.yml file
+        for table in table_name:
+            name_part, version_part = parse_config_name(table)
+
+            rendered_content = bigconfig_template.render(
+                project_id=target_project,
+                dataset=dataset_name,
+                name=name_part,
+                version=version_part,
+                bigeye_collection=BIGEYE_COLLECTION,
+                bigeye_notification_slack_channel=BIGEYE_SLACK_CHANNEL,
+            )
+
+            metadata_rendered = metadata_template.render(**common_metadata_vars)
+
+            stable_table_bigconfig_dir = target_dir / table
+            stable_table_bigconfig_dir.mkdir(exist_ok=True)
+
+            write_file(metadata_rendered, stable_table_bigconfig_dir / "metadata.yaml")
+            write_file(rendered_content, stable_table_bigconfig_dir / "bigconfig.yml")

sql_generators/stable_tables_targeted/templates/stable_tables_targeted.bigconfig.yml

@@ -0,0 +1,13 @@
+type: BIGCONFIG_FILE
+
+tag_deployments:
+  - collection:
+      name: {{ bigeye_collection }}
+      notification_channels:
+        - slack: '{{ bigeye_notification_slack_channel }}'
+    deployments:
+      - column_selectors:
+        - name: {{ project_id }}.{{ project_id }}.{{ dataset }}_derived.{{ name }}_{{ version }}.*
+        metrics:
+          - saved_metric_id: volume
+          - saved_metric_id: freshness

sql_generators/stable_tables_targeted/templates/stable_tables_targeted.metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Stable Table BigEye Checks
+description: |-
+  There are two main namespaces in BigQuery for used during telemetry ingestion process.
+  Those namespaces end with _live and _stable . _live is the initial landing zone in BigQuery for our ping data
+  coming from our ingestion-sink which uses a smaller time window to batch telemetry data together.
+  However, there is a potential for duplicate entries in this dataset which is why once a day copy_deduplicate DAG performs de-duplication of this data
+  and saves the result in the corresponding _stable namespace (which also acts as long term storage of this raw data).
+  These BigEye checks add freshness and volume checks to these stable tables.
+
+monitoring:
+  enabled: {% if enable_monitoring %}true{% else %}false{% endif %}
+  collection: {{ bigeye_collection }}