From ae2696bffa3cf9d7f73ca73f3d13c541a7cd9932 Mon Sep 17 00:00:00 2001 From: Macpie Date: Fri, 3 Oct 2025 15:38:36 -0700 Subject: [PATCH 01/35] Create migration test --- Cargo.lock | 1 + mobile_config/Cargo.toml | 1 + .../20251003000000_gateways_historical.sql | 24 ++++ mobile_config/src/gateway/db.rs | 25 ++-- mobile_config/src/gateway/metadata_db.rs | 2 +- .../tests/integrations/common/gateway_db.rs | 71 ++++++++++++ .../tests/integrations/common/mod.rs | 2 + .../integrations/common/partial_migrator.rs | 108 ++++++++++++++++++ .../tests/integrations/gateway_db.rs | 27 +++-- .../tests/integrations/gateway_service.rs | 96 ++++++++-------- .../tests/integrations/gateway_service_v3.rs | 34 +++--- mobile_config/tests/integrations/main.rs | 1 + .../tests/integrations/migrations.rs | 61 ++++++++++ 13 files changed, 361 insertions(+), 92 deletions(-) create mode 100644 mobile_config/migrations/20251003000000_gateways_historical.sql create mode 100644 mobile_config/tests/integrations/common/gateway_db.rs create mode 100644 mobile_config/tests/integrations/common/partial_migrator.rs create mode 100644 mobile_config/tests/integrations/migrations.rs diff --git a/Cargo.lock b/Cargo.lock index 4b7399e57..6d5de88ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4995,6 +4995,7 @@ dependencies = [ "strum", "strum_macros", "task-manager", + "tempfile", "thiserror 1.0.69", "tls-init", "tokio", diff --git a/mobile_config/Cargo.toml b/mobile_config/Cargo.toml index 27a6e6197..8d08d76c6 100644 --- a/mobile_config/Cargo.toml +++ b/mobile_config/Cargo.toml @@ -60,3 +60,4 @@ tls-init = { path = "../tls_init" } [dev-dependencies] rand = { workspace = true } tokio-stream = { workspace = true, features = ["net"] } +tempfile = "3" diff --git a/mobile_config/migrations/20251003000000_gateways_historical.sql b/mobile_config/migrations/20251003000000_gateways_historical.sql new file mode 100644 index 000000000..ada65eee2 --- /dev/null +++ b/mobile_config/migrations/20251003000000_gateways_historical.sql @@ -0,0 +1,24 @@ +-- 1. Drop the primary key constraint on address +ALTER TABLE + gateways DROP CONSTRAINT IF EXISTS gateways_pkey; + +-- 2. Rename column updated_at -> inserted_at +ALTER TABLE + gateways RENAME COLUMN updated_at TO inserted_at; + +-- 3. Backfill inserted_at with created_at values +UPDATE + gateways +SET + inserted_at = created_at; + +-- 4. Ensure inserted_at is NOT NULL +ALTER TABLE + gateways +ALTER COLUMN + inserted_at +SET + NOT NULL; + +-- 5. Create an index on (address, inserted_at DESC) +CREATE INDEX IF NOT EXISTS gateways_address_inserted_idx ON gateways (address, inserted_at DESC); \ No newline at end of file diff --git a/mobile_config/src/gateway/db.rs b/mobile_config/src/gateway/db.rs index c7a4c039a..d721b4e86 100644 --- a/mobile_config/src/gateway/db.rs +++ b/mobile_config/src/gateway/db.rs @@ -81,8 +81,8 @@ pub struct Gateway { pub gateway_type: GatewayType, // When the record was first created from metadata DB pub created_at: DateTime, - // When record was last updated - pub updated_at: DateTime, + // When record was inserted + pub inserted_at: DateTime, // When record was last updated from metadata DB (could be set to now if no metadata DB info) pub refreshed_at: DateTime, // When location or hash last changed, set to refreshed_at (updated via SQL query see Gateway::insert) @@ -96,6 +96,7 @@ pub struct Gateway { pub location_changed_at: Option>, pub location_asserts: Option, } + #[derive(Debug)] pub struct LocationChangedAtUpdate { pub address: PublicKeyBinary, @@ -113,7 +114,7 @@ impl Gateway { address, gateway_type, created_at, - updated_at, + inserted_at, refreshed_at, last_changed_at, hash, @@ -130,7 +131,7 @@ impl Gateway { b.push_bind(g.address.as_ref()) .push_bind(g.gateway_type) .push_bind(g.created_at) - .push_bind(g.updated_at) + .push_bind(g.inserted_at) .push_bind(g.refreshed_at) .push_bind(g.last_changed_at) .push_bind(g.hash.as_str()) @@ -146,7 +147,7 @@ impl Gateway { " ON CONFLICT (address) DO UPDATE SET gateway_type = EXCLUDED.gateway_type, created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, + inserted_at = EXCLUDED.inserted_at, refreshed_at = EXCLUDED.refreshed_at, last_changed_at = CASE WHEN gateways.location IS DISTINCT FROM EXCLUDED.location @@ -178,7 +179,7 @@ impl Gateway { address, gateway_type, created_at, - updated_at, + inserted_at, refreshed_at, last_changed_at, hash, @@ -197,7 +198,7 @@ impl Gateway { DO UPDATE SET gateway_type = EXCLUDED.gateway_type, created_at = EXCLUDED.created_at, - updated_at = EXCLUDED.updated_at, + inserted_at = EXCLUDED.inserted_at, refreshed_at = EXCLUDED.refreshed_at, last_changed_at = CASE WHEN gateways.location IS DISTINCT FROM EXCLUDED.location @@ -221,7 +222,7 @@ impl Gateway { .bind(self.address.as_ref()) .bind(self.gateway_type) .bind(self.created_at) - .bind(self.updated_at) + .bind(self.inserted_at) .bind(self.refreshed_at) .bind(self.last_changed_at) .bind(self.hash.as_str()) @@ -247,7 +248,7 @@ impl Gateway { address, gateway_type, created_at, - updated_at, + inserted_at, refreshed_at, last_changed_at, hash, @@ -281,7 +282,7 @@ impl Gateway { address, gateway_type, created_at, - updated_at, + inserted_at, refreshed_at, last_changed_at, hash, @@ -315,7 +316,7 @@ impl Gateway { address, gateway_type, created_at, - updated_at, + inserted_at, refreshed_at, last_changed_at, hash, @@ -395,7 +396,7 @@ impl FromRow<'_, PgRow> for Gateway { address: PublicKeyBinary::from(row.try_get::, _>("address")?), gateway_type: row.try_get("gateway_type")?, created_at: row.try_get("created_at")?, - updated_at: row.try_get("updated_at")?, + inserted_at: row.try_get("inserted_at")?, refreshed_at: row.try_get("refreshed_at")?, last_changed_at: row.try_get("last_changed_at")?, hash: row.try_get("hash")?, diff --git a/mobile_config/src/gateway/metadata_db.rs b/mobile_config/src/gateway/metadata_db.rs index 7987579cd..badb7fc9e 100644 --- a/mobile_config/src/gateway/metadata_db.rs +++ b/mobile_config/src/gateway/metadata_db.rs @@ -123,7 +123,7 @@ impl MobileHotspotInfo { address: self.entity_key.clone(), gateway_type: GatewayType::try_from(self.device_type.clone())?, created_at: self.created_at, - updated_at: Utc::now(), + inserted_at: Utc::now(), refreshed_at: self.refreshed_at.unwrap_or_else(Utc::now), // Updated via SQL query see Gateway::insert last_changed_at: Utc::now(), diff --git a/mobile_config/tests/integrations/common/gateway_db.rs b/mobile_config/tests/integrations/common/gateway_db.rs new file mode 100644 index 000000000..7ca9dd2d0 --- /dev/null +++ b/mobile_config/tests/integrations/common/gateway_db.rs @@ -0,0 +1,71 @@ +use chrono::{DateTime, Utc}; +use helium_crypto::PublicKeyBinary; +use mobile_config::gateway::db::GatewayType; +use sqlx::PgPool; + +#[derive(Debug, Clone)] +pub struct PreHistoricalGateway { + pub address: PublicKeyBinary, + pub gateway_type: GatewayType, + // When the record was first created from metadata DB + pub created_at: DateTime, + // When record was last updated + pub updated_at: DateTime, + // When record was last updated from metadata DB (could be set to now if no metadata DB info) + pub refreshed_at: DateTime, + // When location or hash last changed, set to refreshed_at (updated via SQL query see Gateway::insert) + pub last_changed_at: DateTime, + pub hash: String, + pub antenna: Option, + pub elevation: Option, + pub azimuth: Option, + pub location: Option, + // When location last changed, set to refreshed_at (updated via SQL query see Gateway::insert) + pub location_changed_at: Option>, + pub location_asserts: Option, +} + +impl PreHistoricalGateway { + pub async fn insert(&self, pool: &PgPool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO gateways ( + address, + gateway_type, + created_at, + updated_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, + $8, $9, $10, $11, $12, $13 + ) + "#, + ) + .bind(self.address.as_ref()) + .bind(self.gateway_type) + .bind(self.created_at) + .bind(self.updated_at) + .bind(self.refreshed_at) + .bind(self.last_changed_at) + .bind(self.hash.as_str()) + .bind(self.antenna.map(|v| v as i64)) + .bind(self.elevation.map(|v| v as i64)) + .bind(self.azimuth.map(|v| v as i64)) + .bind(self.location.map(|v| v as i64)) + .bind(self.location_changed_at) + .bind(self.location_asserts.map(|v| v as i64)) + .execute(pool) + .await?; + + Ok(()) + } +} diff --git a/mobile_config/tests/integrations/common/mod.rs b/mobile_config/tests/integrations/common/mod.rs index 6007db573..c9813fe78 100644 --- a/mobile_config/tests/integrations/common/mod.rs +++ b/mobile_config/tests/integrations/common/mod.rs @@ -12,7 +12,9 @@ use std::sync::Arc; use tokio::net::TcpListener; use tonic::transport; +pub mod gateway_db; pub mod gateway_metadata_db; +pub mod partial_migrator; pub fn make_keypair() -> Keypair { Keypair::generate(KeyTag::default(), &mut rand::rngs::OsRng) diff --git a/mobile_config/tests/integrations/common/partial_migrator.rs b/mobile_config/tests/integrations/common/partial_migrator.rs new file mode 100644 index 000000000..3d9c38a8a --- /dev/null +++ b/mobile_config/tests/integrations/common/partial_migrator.rs @@ -0,0 +1,108 @@ +use chrono::Utc; +use sqlx::{ + migrate::{Migration, Migrator}, + PgPool, +}; +use std::path::Path; + +pub struct PartialMigrator { + pool: PgPool, + versions: Vec, + path: String, +} + +impl PartialMigrator { + pub async fn new( + pool: PgPool, + versions: Vec, + path: Option, + ) -> anyhow::Result { + let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM _sqlx_migrations") + .fetch_one(&pool) + .await + .unwrap_or((0,)); + + if count.0 > 0 { + anyhow::bail!("PartialMigrator: database already has applied migrations. Did you forget `migrations = false`?"); + } + + Ok(Self { + pool, + versions, + path: path.unwrap_or_else(|| "./migrations".to_string()), + }) + } + + pub async fn run_partial(&self) -> anyhow::Result<()> { + { + // Run tmp_migrator to create _sqlx_migrations table + let tmp_dir = tempfile::tempdir()?; + let tmp_migrator = Migrator::new(tmp_dir.path()).await?; + tmp_migrator.run(&self.pool).await?; + } + + let migrator = Migrator::new(Path::new(&self.path)).await?; + + // Mark skipped migrations as applied first + for m in migrator.iter() { + if self.versions.contains(&m.version) { + println!("⏭️ Skipping migration {} {}", m.version, m.description); + self.skip_migration(m).await?; + } + } + + // Now run the migrator normally + migrator.run(&self.pool).await?; + + Ok(()) + } + + pub async fn run_skipped(&self) -> anyhow::Result<()> { + let migrator = Migrator::new(Path::new(&self.path)).await?; + + println!("Re applaying skipped migrations... {:?}", self.versions); + + // Delete skipped migrations first + self.delete_skipped().await?; + + // Now run the migrator normally + migrator.run(&self.pool).await?; + + Ok(()) + } + + async fn skip_migration(&self, migration: &Migration) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO _sqlx_migrations + (version, description, installed_on, success, checksum, execution_time) + VALUES ($1, $2, $3, TRUE, $4, 0) + ON CONFLICT (version) DO NOTHING + "#, + ) + .bind(migration.version) + .bind(format!("SKIPPED - {}", migration.description.clone())) + .bind(Utc::now()) + .bind(migration.checksum.as_ref()) + .execute(&self.pool) + .await?; + + Ok(()) + } + + async fn delete_skipped(&self) -> anyhow::Result<()> { + for version in &self.versions { + sqlx::query( + r#" + DELETE FROM _sqlx_migrations + WHERE version = $1 + "#, + ) + .bind(version) + .execute(&self.pool) + .await?; + } + + Ok(()) + } +} diff --git a/mobile_config/tests/integrations/gateway_db.rs b/mobile_config/tests/integrations/gateway_db.rs index de7f8cb43..3ba000d78 100644 --- a/mobile_config/tests/integrations/gateway_db.rs +++ b/mobile_config/tests/integrations/gateway_db.rs @@ -1,11 +1,10 @@ +use crate::common; use chrono::{TimeZone, Utc}; use futures::{pin_mut, StreamExt}; use helium_crypto::PublicKeyBinary; use mobile_config::gateway::db::{Gateway, GatewayType}; use sqlx::PgPool; -use crate::common; - #[sqlx::test] async fn gateway_insert_and_get_by_address(pool: PgPool) -> anyhow::Result<()> { let addr = pk_binary(); @@ -20,7 +19,7 @@ async fn gateway_insert_and_get_by_address(pool: PgPool) -> anyhow::Result<()> { assert_eq!(gateway.gateway_type, GatewayType::WifiIndoor); assert_eq!(gateway.created_at, common::nanos_trunc(now)); - assert_eq!(gateway.updated_at, common::nanos_trunc(now)); + assert_eq!(gateway.inserted_at, common::nanos_trunc(now)); assert_eq!(gateway.refreshed_at, common::nanos_trunc(now)); assert_eq!(gateway.last_changed_at, common::nanos_trunc(now)); // first insert: equals refreshed_at assert_eq!(gateway.location, Some(123)); @@ -45,7 +44,7 @@ async fn gateway_upsert_last_changed_at_on_location_or_hash_change( // upsert with no change (only timestamps move) let mut same = gw(addr.clone(), GatewayType::WifiOutdoor, t0); - same.updated_at = t1; + same.inserted_at = t1; same.refreshed_at = t1; same.last_changed_at = t1; // should be ignored by SQL if no change same.insert(&pool).await?; @@ -56,7 +55,7 @@ async fn gateway_upsert_last_changed_at_on_location_or_hash_change( // upsert with location change -> last_changed_at bumps to refreshed_at (t2) let mut loc = after_same.clone(); - loc.updated_at = t2; + loc.inserted_at = t2; loc.refreshed_at = t2; loc.location = Some(456); loc.insert(&pool).await?; @@ -67,7 +66,7 @@ async fn gateway_upsert_last_changed_at_on_location_or_hash_change( // upsert with hash change (location same) -> last_changed_at bumps again let mut h = after_loc.clone(); - h.updated_at = t3; + h.inserted_at = t3; h.refreshed_at = t3; h.hash = "h1".into(); h.insert(&pool).await?; @@ -99,7 +98,7 @@ async fn gateway_bulk_insert_and_get(pool: PgPool) -> anyhow::Result<()> { .await? .expect("row should exist"); assert_eq!(got.created_at, common::nanos_trunc(now)); - assert_eq!(got.updated_at, common::nanos_trunc(now)); + assert_eq!(got.inserted_at, common::nanos_trunc(now)); assert_eq!(got.refreshed_at, common::nanos_trunc(now)); assert_eq!(got.last_changed_at, common::nanos_trunc(now)); assert_eq!(got.location, Some(123)); @@ -124,11 +123,11 @@ async fn gateway_bulk_upsert_updates_and_change(pool: PgPool) -> anyhow::Result< // and change location for g2 (should bump last_changed_at) let t1 = Utc::now(); - g1.updated_at = t1; + g1.inserted_at = t1; g1.refreshed_at = t1; // leave g1.location / g1.hash the same - g2.updated_at = t1; + g2.inserted_at = t1; g2.refreshed_at = t1; g2.location = Some(456); // change => last_changed_at should bump to t1 // g2.hash unchanged @@ -141,7 +140,7 @@ async fn gateway_bulk_upsert_updates_and_change(pool: PgPool) -> anyhow::Result< let got1 = Gateway::get_by_address(&pool, &a1) .await? .expect("row should exist"); - assert_eq!(got1.updated_at, common::nanos_trunc(t1)); + assert_eq!(got1.inserted_at, common::nanos_trunc(t1)); assert_eq!(got1.refreshed_at, common::nanos_trunc(t1)); assert_eq!( got1.last_changed_at, @@ -155,7 +154,7 @@ async fn gateway_bulk_upsert_updates_and_change(pool: PgPool) -> anyhow::Result< let got2 = Gateway::get_by_address(&pool, &a2) .await? .expect("row should exist"); - assert_eq!(got2.updated_at, common::nanos_trunc(t1)); + assert_eq!(got2.inserted_at, common::nanos_trunc(t1)); assert_eq!(got2.refreshed_at, common::nanos_trunc(t1)); assert_eq!( got2.last_changed_at, @@ -168,7 +167,7 @@ async fn gateway_bulk_upsert_updates_and_change(pool: PgPool) -> anyhow::Result< // second upsert at t2: change hash only for g1, ensure bump let t2 = Utc::now(); - g1.updated_at = t2; + g1.inserted_at = t2; g1.refreshed_at = t2; g1.hash = "h1".into(); // change ⇒ bump last_changed_at @@ -178,7 +177,7 @@ async fn gateway_bulk_upsert_updates_and_change(pool: PgPool) -> anyhow::Result< let got1b = Gateway::get_by_address(&pool, &a1) .await? .expect("row should exist"); - assert_eq!(got1b.updated_at, common::nanos_trunc(t2)); + assert_eq!(got1b.inserted_at, common::nanos_trunc(t2)); assert_eq!(got1b.refreshed_at, common::nanos_trunc(t2)); assert_eq!( got1b.last_changed_at, @@ -292,7 +291,7 @@ fn gw(address: PublicKeyBinary, gateway_type: GatewayType, t: chrono::DateTime anyhow::Result<()> { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now, - updated_at: now, + inserted_at: now, refreshed_at: now, last_changed_at: now, hash: "".to_string(), @@ -134,7 +134,7 @@ async fn gateway_stream_info_v1(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at: now_plus_10, - updated_at: now_plus_10, + inserted_at: now_plus_10, refreshed_at: now_plus_10, last_changed_at: now_plus_10, hash: "".to_string(), @@ -182,7 +182,7 @@ async fn gateway_stream_info_v2_by_type(pool: PgPool) -> anyhow::Result<()> { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now, - updated_at: now, + inserted_at: now, refreshed_at: now, last_changed_at: now, hash: "".to_string(), @@ -199,7 +199,7 @@ async fn gateway_stream_info_v2_by_type(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at: now_plus_10, - updated_at: now_plus_10, + inserted_at: now_plus_10, refreshed_at: now_plus_10, last_changed_at: now_plus_10, hash: "".to_string(), @@ -247,21 +247,21 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { let loc4 = 0x8c44a82aed527ff_u64; let created_at = Utc::now() - Duration::hours(5); - let updated_at = Utc::now() - Duration::hours(3); + let inserted_at = Utc::now() - Duration::hours(3); let gateway1 = Gateway { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at, - refreshed_at: updated_at, - last_changed_at: updated_at, + inserted_at, + refreshed_at: inserted_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: None, elevation: None, azimuth: None, location: Some(loc1), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway1.insert(&pool).await?; @@ -270,15 +270,15 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at, - refreshed_at: updated_at, - last_changed_at: updated_at, + inserted_at, + refreshed_at: inserted_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(1), elevation: None, azimuth: None, location: Some(loc2), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway2.insert(&pool).await?; @@ -287,15 +287,15 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { address: address3.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at, - updated_at, - refreshed_at: updated_at, - last_changed_at: updated_at, + inserted_at, + refreshed_at: inserted_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(1), elevation: Some(2), azimuth: Some(3), location: Some(loc3), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway3.insert(&pool).await?; @@ -304,7 +304,7 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { address: address4.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at, - updated_at: created_at, + inserted_at: created_at, refreshed_at: created_at, last_changed_at: created_at, hash: "".to_string(), @@ -320,8 +320,8 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { let (addr, _handle) = spawn_gateway_service(pool.clone(), admin_key.public_key().clone()).await; let mut client = GatewayClient::connect(addr).await?; - let res = - gateway_info_stream_v2(&mut client, &admin_key, &[], updated_at.timestamp() as u64).await?; + let res = gateway_info_stream_v2(&mut client, &admin_key, &[], inserted_at.timestamp() as u64) + .await?; assert_eq!(res.gateways.len(), 3); let gateways = res.gateways; @@ -335,7 +335,7 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { u64::from_str_radix(&gw1.metadata.clone().unwrap().location, 16).unwrap(), loc1 ); - assert_eq!(gw1.updated_at, updated_at.timestamp() as u64); + assert_eq!(gw1.updated_at, inserted_at.timestamp() as u64); assert_eq!(gw1.metadata.clone().unwrap().deployment_info, None); let gw2 = gateways @@ -347,7 +347,7 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { u64::from_str_radix(&gw2.metadata.clone().unwrap().location, 16).unwrap(), loc2 ); - assert_eq!(gw2.updated_at, updated_at.timestamp() as u64); + assert_eq!(gw2.updated_at, inserted_at.timestamp() as u64); let deployment_info = gw2.metadata.clone().unwrap().deployment_info.unwrap(); match deployment_info { DeploymentInfo::WifiDeploymentInfo(v) => { @@ -369,7 +369,7 @@ async fn gateway_stream_info_v2(pool: PgPool) -> anyhow::Result<()> { u64::from_str_radix(&gw3.metadata.clone().unwrap().location, 16).unwrap(), loc3 ); - assert_eq!(gw3.updated_at, updated_at.timestamp() as u64); + assert_eq!(gw3.updated_at, inserted_at.timestamp() as u64); let deployment_info = gw3.metadata.clone().unwrap().deployment_info.unwrap(); match deployment_info { DeploymentInfo::WifiDeploymentInfo(v) => { @@ -399,21 +399,21 @@ async fn gateway_info_batch_v1(pool: PgPool) -> anyhow::Result<()> { let loc2 = 631711286145955327_u64; let created_at = Utc::now() - Duration::hours(5); - let updated_at = Utc::now() - Duration::hours(3); + let inserted_at = Utc::now() - Duration::hours(3); let gateway1 = Gateway { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at, - refreshed_at: updated_at, - last_changed_at: updated_at, + inserted_at, + refreshed_at: inserted_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(18), elevation: Some(2), azimuth: Some(161), location: Some(loc1), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway1.insert(&pool).await?; @@ -422,7 +422,7 @@ async fn gateway_info_batch_v1(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at, - updated_at: created_at, + inserted_at: created_at, refreshed_at: created_at, last_changed_at: created_at, hash: "".to_string(), @@ -485,21 +485,21 @@ async fn gateway_info_batch_v2(pool: PgPool) -> anyhow::Result<()> { let loc2 = 631711286145955327_u64; let created_at = Utc::now() - Duration::hours(5); - let updated_at = Utc::now() - Duration::hours(3); + let inserted_at = Utc::now() - Duration::hours(3); let gateway1 = Gateway { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at, - refreshed_at: updated_at, - last_changed_at: updated_at, + inserted_at, + refreshed_at: inserted_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(18), elevation: Some(2), azimuth: Some(161), location: Some(loc1), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway1.insert(&pool).await?; @@ -508,7 +508,7 @@ async fn gateway_info_batch_v2(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at, - updated_at: created_at, + inserted_at: created_at, refreshed_at: created_at, last_changed_at: created_at, hash: "".to_string(), @@ -594,13 +594,13 @@ async fn gateway_info_batch_v2_updated_at_check(pool: PgPool) -> anyhow::Result< let created_at = Utc::now() - Duration::hours(5); let refreshed_at = Utc::now() - Duration::hours(3); - let updated_at = Utc::now() - Duration::hours(4); + let inserted_at = Utc::now() - Duration::hours(4); let gateway1 = Gateway { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at: refreshed_at, + inserted_at: refreshed_at, refreshed_at, last_changed_at: refreshed_at, hash: "".to_string(), @@ -617,7 +617,7 @@ async fn gateway_info_batch_v2_updated_at_check(pool: PgPool) -> anyhow::Result< address: address2.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at: created_at, + inserted_at: created_at, refreshed_at: created_at, last_changed_at: created_at, hash: "".to_string(), @@ -634,15 +634,15 @@ async fn gateway_info_batch_v2_updated_at_check(pool: PgPool) -> anyhow::Result< address: address3.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at, - updated_at, + inserted_at, refreshed_at, - last_changed_at: updated_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(18), elevation: Some(2), azimuth: Some(161), location: Some(loc3), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway3.insert(&pool).await?; @@ -651,9 +651,9 @@ async fn gateway_info_batch_v2_updated_at_check(pool: PgPool) -> anyhow::Result< address: address4.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at, + inserted_at, refreshed_at: created_at, - last_changed_at: updated_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(18), elevation: Some(2), @@ -707,7 +707,7 @@ async fn gateway_info_batch_v2_updated_at_check(pool: PgPool) -> anyhow::Result< .find(|v| v.address == address3.to_vec()) .unwrap() .updated_at, - updated_at.timestamp() as u64 + inserted_at.timestamp() as u64 ); Ok(()) @@ -730,7 +730,7 @@ async fn gateway_info_v2(pool: PgPool) -> anyhow::Result<()> { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at: refreshed_at, + inserted_at: refreshed_at, refreshed_at, last_changed_at: refreshed_at, hash: "".to_string(), @@ -747,7 +747,7 @@ async fn gateway_info_v2(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at: created_at, + inserted_at: created_at, refreshed_at: created_at, last_changed_at: created_at, hash: "".to_string(), @@ -866,14 +866,14 @@ async fn gateway_info_stream_v2( client: &mut GatewayClient, signer: &Keypair, device_types: &[DeviceType], - min_updated_at: u64, + min_inserted_at: u64, ) -> anyhow::Result { let mut req = GatewayInfoStreamReqV2 { batch_size: 10000, signer: signer.public_key().to_vec(), signature: vec![], device_types: device_types.iter().map(|v| DeviceType::into(*v)).collect(), - min_updated_at, + min_updated_at: min_inserted_at, }; req.signature = signer.sign(&req.encode_to_vec()).unwrap(); diff --git a/mobile_config/tests/integrations/gateway_service_v3.rs b/mobile_config/tests/integrations/gateway_service_v3.rs index c0457ec90..1ee5b311e 100644 --- a/mobile_config/tests/integrations/gateway_service_v3.rs +++ b/mobile_config/tests/integrations/gateway_service_v3.rs @@ -29,7 +29,7 @@ async fn gateway_stream_info_v3_basic(pool: PgPool) -> anyhow::Result<()> { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now, - updated_at: now, + inserted_at: now, refreshed_at: now, last_changed_at: now_plus_10, hash: "".to_string(), @@ -46,7 +46,7 @@ async fn gateway_stream_info_v3_basic(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiOutdoor, created_at: now_plus_10, - updated_at: now_plus_10, + inserted_at: now_plus_10, refreshed_at: now_plus_10, last_changed_at: now_plus_10, hash: "".to_string(), @@ -107,7 +107,7 @@ async fn gateway_stream_info_v3_no_metadata(pool: PgPool) -> anyhow::Result<()> address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now, - updated_at: now, + inserted_at: now, refreshed_at: now, last_changed_at: now_plus_10, hash: "".to_string(), @@ -153,7 +153,7 @@ async fn gateway_stream_info_v3_no_deployment_info(pool: PgPool) -> anyhow::Resu address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now, - updated_at: now, + inserted_at: now, refreshed_at: now, last_changed_at: now_plus_10, hash: "".to_string(), @@ -205,21 +205,21 @@ async fn gateway_stream_info_v3_updated_at(pool: PgPool) -> anyhow::Result<()> { let loc2 = 631711286145955327_u64; let created_at = Utc::now() - Duration::hours(5); - let updated_at = Utc::now() - Duration::hours(3); + let inserted_at = Utc::now() - Duration::hours(3); let gateway1 = Gateway { address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, - updated_at, - refreshed_at: updated_at, - last_changed_at: updated_at, + inserted_at, + refreshed_at: inserted_at, + last_changed_at: inserted_at, hash: "".to_string(), antenna: Some(18), elevation: Some(2), azimuth: Some(161), location: Some(loc1), - location_changed_at: Some(updated_at), + location_changed_at: Some(inserted_at), location_asserts: Some(1), }; gateway1.insert(&pool).await?; @@ -228,7 +228,7 @@ async fn gateway_stream_info_v3_updated_at(pool: PgPool) -> anyhow::Result<()> { address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at, - updated_at: created_at, + inserted_at: created_at, refreshed_at: created_at, last_changed_at: created_at, hash: "".to_string(), @@ -248,7 +248,7 @@ async fn gateway_stream_info_v3_updated_at(pool: PgPool) -> anyhow::Result<()> { &mut client, &admin_key, &[], - updated_at.timestamp() as u64, + inserted_at.timestamp() as u64, 0, ) .await?; @@ -299,7 +299,7 @@ async fn gateway_stream_info_v3_min_location_changed_at_zero(pool: PgPool) -> an address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now_minus_six, - updated_at: now_minus_six, + inserted_at: now_minus_six, refreshed_at: now_minus_six, last_changed_at: now_minus_three, hash: "".to_string(), @@ -316,7 +316,7 @@ async fn gateway_stream_info_v3_min_location_changed_at_zero(pool: PgPool) -> an address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at: now_minus_six, - updated_at: now_minus_six, + inserted_at: now_minus_six, refreshed_at: now_minus_six, last_changed_at: now_minus_three, hash: "".to_string(), @@ -370,7 +370,7 @@ async fn gateway_stream_info_v3_location_changed_at(pool: PgPool) -> anyhow::Res address: address1.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at: now_minus_six, - updated_at: now_minus_six, + inserted_at: now_minus_six, refreshed_at: now, last_changed_at: now_minus_three, hash: "".to_string(), @@ -387,7 +387,7 @@ async fn gateway_stream_info_v3_location_changed_at(pool: PgPool) -> anyhow::Res address: address2.clone().into(), gateway_type: GatewayType::WifiDataOnly, created_at: now_minus_six, - updated_at: now_minus_six, + inserted_at: now_minus_six, refreshed_at: now, last_changed_at: now_minus_three, hash: "".to_string(), @@ -449,7 +449,7 @@ async fn gateway_info_stream_v3( client: &mut GatewayClient, signer: &Keypair, device_types: &[DeviceTypeV2], - min_updated_at: u64, + min_inserted_at: u64, min_location_changed_at: u64, ) -> anyhow::Result { let mut req = GatewayInfoStreamReqV3 { @@ -460,7 +460,7 @@ async fn gateway_info_stream_v3( .iter() .map(|v| DeviceTypeV2::into(*v)) .collect(), - min_updated_at, + min_updated_at: min_inserted_at, min_location_changed_at, }; req.signature = signer.sign(&req.encode_to_vec())?; diff --git a/mobile_config/tests/integrations/main.rs b/mobile_config/tests/integrations/main.rs index 358b16a15..318f93eb7 100644 --- a/mobile_config/tests/integrations/main.rs +++ b/mobile_config/tests/integrations/main.rs @@ -5,3 +5,4 @@ mod gateway_db; mod gateway_service; mod gateway_service_v3; mod gateway_tracker; +mod migrations; diff --git a/mobile_config/tests/integrations/migrations.rs b/mobile_config/tests/integrations/migrations.rs new file mode 100644 index 000000000..12d0adbb6 --- /dev/null +++ b/mobile_config/tests/integrations/migrations.rs @@ -0,0 +1,61 @@ +use crate::common::{self, gateway_db::PreHistoricalGateway, partial_migrator::PartialMigrator}; +use chrono::{Duration, Utc}; +use helium_crypto::PublicKeyBinary; +use mobile_config::gateway::db::{Gateway, GatewayType}; +use sqlx::PgPool; + +#[sqlx::test(migrations = false)] +async fn gateways_historical(pool: PgPool) -> anyhow::Result<()> { + let partial_migrator = PartialMigrator::new(pool.clone(), vec![20251003000000], None).await?; + + partial_migrator.run_partial().await?; + + let address = pk_binary(); + let now = Utc::now(); + let one_min_ago = now - Duration::minutes(1); + + let pre_gw = PreHistoricalGateway { + address: address.clone(), + gateway_type: GatewayType::WifiIndoor, + created_at: one_min_ago, + updated_at: now, + refreshed_at: now, + last_changed_at: now, + hash: "h0".to_string(), + antenna: Some(1), + elevation: Some(2), + azimuth: Some(3), + location: Some(123), + location_changed_at: Some(now), + location_asserts: Some(1), + }; + + pre_gw.insert(&pool).await?; + + partial_migrator.run_skipped().await?; + + let gw = Gateway::get_by_address(&pool, &address) + .await? + .expect("should find gateway"); + + assert!(pre_gw.address == gw.address); + assert!(pre_gw.gateway_type == gw.gateway_type); + assert!(pre_gw.created_at == gw.created_at); + // The real change is updated_at renamed to inserted_at AND inserted_at = created_at; + assert!(pre_gw.created_at == gw.inserted_at); + assert!(pre_gw.refreshed_at == gw.refreshed_at); + assert!(pre_gw.last_changed_at == gw.last_changed_at); + assert!(pre_gw.hash == gw.hash); + assert!(pre_gw.antenna == gw.antenna); + assert!(pre_gw.elevation == gw.elevation); + assert!(pre_gw.azimuth == gw.azimuth); + assert!(pre_gw.location == gw.location); + assert!(pre_gw.location_changed_at == gw.location_changed_at); + assert!(pre_gw.location_asserts == gw.location_asserts); + + Ok(()) +} + +fn pk_binary() -> PublicKeyBinary { + common::make_keypair().public_key().clone().into() +} From 98f8b4f200e7e1679c7ec86d4bf61980c4a7008e Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 6 Oct 2025 13:55:39 -0700 Subject: [PATCH 02/35] Update tracker --- mobile_config/src/gateway/db.rs | 84 +++++----- mobile_config/src/gateway/metadata_db.rs | 11 +- mobile_config/src/gateway/tracker.rs | 58 ++++++- .../common/gateway_metadata_db.rs | 152 +++++++++++------- .../tests/integrations/gateway_db.rs | 51 ------ .../tests/integrations/gateway_tracker.rs | 134 +++++++++++---- 6 files changed, 294 insertions(+), 196 deletions(-) diff --git a/mobile_config/src/gateway/db.rs b/mobile_config/src/gateway/db.rs index d721b4e86..6f21ffe0a 100644 --- a/mobile_config/src/gateway/db.rs +++ b/mobile_config/src/gateway/db.rs @@ -143,31 +143,6 @@ impl Gateway { .push_bind(g.location_asserts.map(|v| v as i64)); }); - qb.push( - " ON CONFLICT (address) DO UPDATE SET - gateway_type = EXCLUDED.gateway_type, - created_at = EXCLUDED.created_at, - inserted_at = EXCLUDED.inserted_at, - refreshed_at = EXCLUDED.refreshed_at, - last_changed_at = CASE - WHEN gateways.location IS DISTINCT FROM EXCLUDED.location - OR gateways.hash IS DISTINCT FROM EXCLUDED.hash - THEN EXCLUDED.refreshed_at - ELSE gateways.last_changed_at - END, - hash = EXCLUDED.hash, - antenna = EXCLUDED.antenna, - elevation = EXCLUDED.elevation, - azimuth = EXCLUDED.azimuth, - location = EXCLUDED.location, - location_changed_at = CASE - WHEN gateways.location IS DISTINCT FROM EXCLUDED.location - THEN EXCLUDED.refreshed_at - ELSE gateways.location_changed_at - END, - location_asserts = EXCLUDED.location_asserts", - ); - let res = qb.build().execute(pool).await?; Ok(res.rows_affected()) } @@ -194,29 +169,6 @@ impl Gateway { $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 ) - ON CONFLICT (address) - DO UPDATE SET - gateway_type = EXCLUDED.gateway_type, - created_at = EXCLUDED.created_at, - inserted_at = EXCLUDED.inserted_at, - refreshed_at = EXCLUDED.refreshed_at, - last_changed_at = CASE - WHEN gateways.location IS DISTINCT FROM EXCLUDED.location - OR gateways.hash IS DISTINCT FROM EXCLUDED.hash - THEN EXCLUDED.refreshed_at - ELSE gateways.last_changed_at - END, - hash = EXCLUDED.hash, - antenna = EXCLUDED.antenna, - elevation = EXCLUDED.elevation, - azimuth = EXCLUDED.azimuth, - location = EXCLUDED.location, - location_changed_at = CASE - WHEN gateways.location IS DISTINCT FROM EXCLUDED.location - THEN EXCLUDED.refreshed_at - ELSE gateways.location_changed_at - END, - location_asserts = EXCLUDED.location_asserts "#, ) .bind(self.address.as_ref()) @@ -260,6 +212,8 @@ impl Gateway { location_asserts FROM gateways WHERE address = $1 + ORDER BY inserted_at DESC + LIMIT 1 "#, ) .bind(address.as_ref()) @@ -269,6 +223,40 @@ impl Gateway { Ok(gateway) } + pub async fn get_by_addresses<'a>( + db: impl PgExecutor<'a>, + addresses: Vec, + ) -> anyhow::Result> { + let addr_array: Vec> = addresses.iter().map(|a| a.as_ref().to_vec()).collect(); + + let rows = sqlx::query_as::<_, Self>( + r#" + SELECT DISTINCT ON (address) + address, + gateway_type, + created_at, + inserted_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts + FROM gateways + WHERE address = ANY($1) + ORDER BY address, inserted_at DESC + "#, + ) + .bind(addr_array) + .fetch_all(db) + .await?; + + Ok(rows) + } + pub fn stream_by_addresses<'a>( db: impl PgExecutor<'a> + 'a, addresses: Vec, diff --git a/mobile_config/src/gateway/metadata_db.rs b/mobile_config/src/gateway/metadata_db.rs index badb7fc9e..983382532 100644 --- a/mobile_config/src/gateway/metadata_db.rs +++ b/mobile_config/src/gateway/metadata_db.rs @@ -119,14 +119,15 @@ impl MobileHotspotInfo { None => (None, None, None), }; + let refreshed_at = self.refreshed_at.unwrap_or_else(Utc::now); + Ok(Some(Gateway { address: self.entity_key.clone(), gateway_type: GatewayType::try_from(self.device_type.clone())?, created_at: self.created_at, inserted_at: Utc::now(), - refreshed_at: self.refreshed_at.unwrap_or_else(Utc::now), - // Updated via SQL query see Gateway::insert - last_changed_at: Utc::now(), + refreshed_at, + last_changed_at: refreshed_at, hash: self.compute_hash(), antenna, elevation, @@ -134,7 +135,7 @@ impl MobileHotspotInfo { location, // Set to refreshed_at when hotspot has a location, None otherwise location_changed_at: if location.is_some() { - Some(self.refreshed_at.unwrap_or_else(Utc::now)) + Some(refreshed_at) } else { None }, @@ -164,7 +165,7 @@ impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for MobileHotspotInfo { ) .map_err(|err| sqlx::Error::Decode(Box::new(err)))?, refreshed_at: row.get::>, &str>("refreshed_at"), - created_at: row.get::, &str>("refreshed_at"), + created_at: row.get::, &str>("created_at"), location: row.get::, &str>("location"), is_full_hotspot: row.get::, &str>("is_full_hotspot"), num_location_asserts: row.get::, &str>("num_location_asserts"), diff --git a/mobile_config/src/gateway/tracker.rs b/mobile_config/src/gateway/tracker.rs index 88503dbb5..579f4d660 100644 --- a/mobile_config/src/gateway/tracker.rs +++ b/mobile_config/src/gateway/tracker.rs @@ -2,7 +2,10 @@ use crate::gateway::{db::Gateway, metadata_db::MobileHotspotInfo}; use futures::{stream::TryChunksError, TryFutureExt}; use futures_util::TryStreamExt; use sqlx::{Pool, Postgres}; -use std::time::{Duration, Instant}; +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; use task_manager::ManagedTask; const EXECUTE_DURATION_METRIC: &str = @@ -67,11 +70,60 @@ pub async fn execute(pool: &Pool, metadata: &Pool) -> anyhow let total: u64 = MobileHotspotInfo::stream(metadata) .map_err(anyhow::Error::from) - .try_filter_map(|mhi| async move { mhi.to_gateway() }) + .try_filter_map(|mhi| async move { + match mhi.to_gateway() { + Ok(Some(gw)) => Ok(Some(gw)), + Ok(None) => Ok(None), + Err(e) => { + tracing::error!(?e, "error converting gateway"); + Err(e) + } + } + }) .try_chunks(BATCH_SIZE) .map_err(|TryChunksError(_gateways, err)| err) .try_fold(0, |total, batch| async move { - let affected = Gateway::insert_bulk(pool, &batch).await?; + let addresses: Vec<_> = batch.iter().map(|gw| gw.address.clone()).collect(); + let existing_gateways = Gateway::get_by_addresses(pool, addresses).await?; + let mut existing_map = existing_gateways + .into_iter() + .map(|gw| (gw.address.clone(), gw)) + .collect::>(); + + let mut to_insert = Vec::with_capacity(batch.len()); + + for mut gw in batch { + match existing_map.remove(&gw.address) { + None => { + // New gateway + to_insert.push(gw); + } + Some(last_gw) => { + // FYI hash includes location + let loc_changed = gw.location != last_gw.location; + let hash_changed = gw.hash != last_gw.hash; + + gw.last_changed_at = if hash_changed { + gw.refreshed_at + } else { + last_gw.last_changed_at + }; + + gw.location_changed_at = if loc_changed { + Some(gw.refreshed_at) + } else { + last_gw.location_changed_at + }; + + // We only add record if something changed + if hash_changed { + to_insert.push(gw); + } + } + } + } + + let affected = Gateway::insert_bulk(pool, &to_insert).await?; Ok(total + affected) }) .await?; diff --git a/mobile_config/tests/integrations/common/gateway_metadata_db.rs b/mobile_config/tests/integrations/common/gateway_metadata_db.rs index e92d4259a..fc1fe9564 100644 --- a/mobile_config/tests/integrations/common/gateway_metadata_db.rs +++ b/mobile_config/tests/integrations/common/gateway_metadata_db.rs @@ -1,76 +1,114 @@ use bs58; use chrono::{DateTime, Utc}; +use futures::{stream, StreamExt, TryStreamExt}; use helium_crypto::PublicKeyBinary; -use sqlx::PgPool; +use sqlx::{PgPool, Postgres, QueryBuilder}; -#[allow(clippy::too_many_arguments)] -pub async fn insert_gateway( +pub struct GatewayInsert { + pub asset: String, + pub location: Option, + pub device_type: String, + pub key: PublicKeyBinary, + pub created_at: DateTime, + pub refreshed_at: Option>, + pub deployment_info: Option, +} + +pub async fn insert_gateway_bulk( pool: &PgPool, - asset: &str, - location: Option, - device_type: &str, - key: PublicKeyBinary, - created_at: DateTime, - refreshed_at: Option>, - deployment_info: Option<&str>, -) { - insert_mobile_hotspot_infos( - pool, - asset, - location, - device_type, - created_at, - refreshed_at, - deployment_info, - ) - .await; - insert_asset_key(pool, asset, key).await; + gateways: &[GatewayInsert], + chunk_size: usize, +) -> anyhow::Result<()> { + stream::iter(gateways.chunks(chunk_size)) + .map(Ok) // convert chunks to a Result for try_for_each_concurrent + .try_for_each_concurrent(Some(20), |chunk| { + let pool = pool.clone(); + async move { + let mut tx = pool.begin().await?; + + // insert mobile_hotspot_infos + let mut qb = QueryBuilder::::new( + r#" + INSERT INTO mobile_hotspot_infos ( + asset, location, device_type, created_at, + refreshed_at, deployment_info, num_location_asserts + ) + "#, + ); + + qb.push_values(chunk, |mut b, gw| { + let num_locations = if gw.location.is_some() { + Some(1) + } else { + Some(0) + }; + + let device_type_json: serde_json::Value = + serde_json::from_str(&gw.device_type).unwrap(); + let deployment_info_json: serde_json::Value = + serde_json::from_str(&gw.deployment_info.as_deref().unwrap_or("null")) + .unwrap(); + + b.push_bind(&gw.asset) + .push_bind(gw.location) + .push_bind(device_type_json) + .push_bind(gw.created_at) + .push_bind(gw.refreshed_at) + .push_bind(deployment_info_json) + .push_bind(num_locations); + }); + + qb.build().execute(&mut *tx).await?; + + // insert key_to_assets + let mut qb1 = QueryBuilder::::new( + r#" + INSERT INTO key_to_assets ( + asset, entity_key + ) + "#, + ); + + qb1.push_values(chunk, |mut b, gw| { + let b58 = bs58::decode(gw.key.to_string()).into_vec().unwrap(); + b.push_bind(&gw.asset).push_bind(b58); + }); + + qb1.build().execute(&mut *tx).await?; + + tx.commit().await?; + + Ok::<_, anyhow::Error>(()) + } + }) + .await?; + + Ok(()) } -async fn insert_mobile_hotspot_infos( +pub async fn update_gateway( pool: &PgPool, asset: &str, - location: Option, - device_type: &str, - created_at: DateTime, - refreshed_at: Option>, - deployment_info: Option<&str>, -) { - let num_locations = if location.is_some() { Some(1) } else { Some(0) }; + location: i64, + refreshed_at: DateTime, +) -> anyhow::Result<()> { sqlx::query( r#" - INSERT INTO -"mobile_hotspot_infos" ("asset", "location", "device_type", "created_at", "refreshed_at", "deployment_info", "num_location_asserts") - VALUES -($1, $2, $3::jsonb, $4, $5, $6::jsonb, $7); - "#, + UPDATE mobile_hotspot_infos + SET location = $1, + num_location_asserts = $2, + refreshed_at = $3 + WHERE asset = $4 + "#, ) - .bind(asset) .bind(location) - .bind(device_type) - .bind(created_at) + .bind(2) .bind(refreshed_at) - .bind(deployment_info) - .bind(num_locations) - .execute(pool) - .await - .unwrap(); -} - -async fn insert_asset_key(pool: &PgPool, asset: &str, key: PublicKeyBinary) { - let b58 = bs58::decode(key.to_string()).into_vec().unwrap(); - sqlx::query( - r#" - INSERT INTO - "key_to_assets" ("asset", "entity_key") - VALUES ($1, $2); - "#, - ) .bind(asset) - .bind(b58) .execute(pool) - .await - .unwrap(); + .await?; + + Ok(()) } pub async fn create_tables(pool: &PgPool) { diff --git a/mobile_config/tests/integrations/gateway_db.rs b/mobile_config/tests/integrations/gateway_db.rs index 3ba000d78..516df582a 100644 --- a/mobile_config/tests/integrations/gateway_db.rs +++ b/mobile_config/tests/integrations/gateway_db.rs @@ -27,57 +27,6 @@ async fn gateway_insert_and_get_by_address(pool: PgPool) -> anyhow::Result<()> { Ok(()) } -#[sqlx::test] -async fn gateway_upsert_last_changed_at_on_location_or_hash_change( - pool: PgPool, -) -> anyhow::Result<()> { - let addr = pk_binary(); - let t0 = Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap(); - let t1 = Utc.with_ymd_and_hms(2025, 1, 2, 0, 0, 0).unwrap(); - let t2 = Utc.with_ymd_and_hms(2025, 1, 3, 0, 0, 0).unwrap(); - let t3 = Utc.with_ymd_and_hms(2025, 1, 4, 0, 0, 0).unwrap(); - - // insert baseline - gw(addr.clone(), GatewayType::WifiOutdoor, t0) - .insert(&pool) - .await?; - - // upsert with no change (only timestamps move) - let mut same = gw(addr.clone(), GatewayType::WifiOutdoor, t0); - same.inserted_at = t1; - same.refreshed_at = t1; - same.last_changed_at = t1; // should be ignored by SQL if no change - same.insert(&pool).await?; - - let after_same = Gateway::get_by_address(&pool, &addr).await?.unwrap(); - assert_eq!(after_same.refreshed_at, t1); - assert_eq!(after_same.last_changed_at, t0); // unchanged - - // upsert with location change -> last_changed_at bumps to refreshed_at (t2) - let mut loc = after_same.clone(); - loc.inserted_at = t2; - loc.refreshed_at = t2; - loc.location = Some(456); - loc.insert(&pool).await?; - - let after_loc = Gateway::get_by_address(&pool, &addr).await?.unwrap(); - assert_eq!(after_loc.location, Some(456)); - assert_eq!(after_loc.last_changed_at, t2); - - // upsert with hash change (location same) -> last_changed_at bumps again - let mut h = after_loc.clone(); - h.inserted_at = t3; - h.refreshed_at = t3; - h.hash = "h1".into(); - h.insert(&pool).await?; - - let after_hash = Gateway::get_by_address(&pool, &addr).await?.unwrap(); - assert_eq!(after_hash.hash, "h1"); - assert_eq!(after_hash.last_changed_at, t3); - - Ok(()) -} - #[sqlx::test] async fn gateway_bulk_insert_and_get(pool: PgPool) -> anyhow::Result<()> { let now = Utc::now(); diff --git a/mobile_config/tests/integrations/gateway_tracker.rs b/mobile_config/tests/integrations/gateway_tracker.rs index 79a0e7aa4..384603a3a 100644 --- a/mobile_config/tests/integrations/gateway_tracker.rs +++ b/mobile_config/tests/integrations/gateway_tracker.rs @@ -1,53 +1,123 @@ -use crate::common::{ - gateway_metadata_db::{create_tables, insert_gateway}, - make_keypair, -}; +use crate::common::{gateway_metadata_db, make_keypair}; use chrono::{Timelike, Utc}; use custom_tracing::Settings; use mobile_config::gateway::{ db::{Gateway, GatewayType}, tracker, }; +use rand::{seq::SliceRandom, thread_rng}; use sqlx::PgPool; #[sqlx::test] async fn execute_test(pool: PgPool) -> anyhow::Result<()> { + const TOTAL: usize = 10_000; + custom_tracing::init("mobile_config=debug,info".to_string(), Settings::default()).await?; - let pubkey1 = make_keypair().public_key().clone(); - let hex1 = 631711281837647359_i64; let now = Utc::now() .with_nanosecond(Utc::now().timestamp_subsec_micros() * 1000) .unwrap(); - create_tables(&pool).await; - insert_gateway( - &pool, - "asset1", - Some(hex1), - "\"wifiIndoor\"", - pubkey1.clone().into(), - now, - Some(now), - None, - ) - .await; + // ensure tables exist + gateway_metadata_db::create_tables(&pool).await; + + // Prepare the bulk insert data + let gateways: Vec = (0..TOTAL) + .map(|i| { + let pubkey = make_keypair().public_key().clone(); + let hex_val = 631_711_281_837_647_359_i64 + i as i64; + + gateway_metadata_db::GatewayInsert { + asset: format!("asset{}", i), + location: Some(hex_val), + device_type: "\"wifiIndoor\"".to_string(), + key: pubkey.into(), + created_at: now, + refreshed_at: Some(now), + deployment_info: None, + } + }) + .collect(); + + // Bulk insert all gateways in one shot + gateway_metadata_db::insert_gateway_bulk(&pool, &gateways, 1000).await?; + + tracing::info!("inserted {} gateways, running tracker", TOTAL); + + // now run the tracker execute function tracker::execute(&pool, &pool).await?; - let gateway1 = Gateway::get_by_address(&pool, &pubkey1.clone().into()) - .await? - .expect("asset1 gateway not found"); - - assert_eq!(gateway1.address, pubkey1.clone().into()); - assert_eq!(gateway1.gateway_type, GatewayType::WifiIndoor); - assert_eq!(gateway1.created_at, now); - assert_eq!(gateway1.refreshed_at, now); - assert_eq!(gateway1.antenna, None); - assert_eq!(gateway1.elevation, None); - assert_eq!(gateway1.azimuth, None); - assert_eq!(gateway1.location, Some(hex1 as u64)); - assert_eq!(gateway1.location_changed_at, Some(now)); - assert_eq!(gateway1.location_asserts, Some(1)); + // Check that we have TOTAL gateways in the DB + let total = count_gateways(&pool).await?; + assert_eq!(TOTAL as i64, total); + + // Sample 100 gateways to verify + let mut rng = thread_rng(); + let sample_size = 100; + let sample: Vec<_> = gateways.choose_multiple(&mut rng, sample_size).collect(); + + let new_loc = 0_i64; + let now = Utc::now() + .with_nanosecond(Utc::now().timestamp_subsec_micros() * 1000) + .unwrap(); + + for gw_insert in sample.clone() { + let gateway = Gateway::get_by_address(&pool, &gw_insert.key) + .await? + .expect("gateway not found"); + + assert_eq!(gateway.address, gw_insert.key.clone()); + assert_eq!(gateway.gateway_type, GatewayType::WifiIndoor); + assert_eq!(gateway.created_at, gw_insert.created_at); + assert_eq!(Some(gateway.refreshed_at), gw_insert.refreshed_at); + assert_eq!(Some(gateway.last_changed_at), gw_insert.refreshed_at); + assert_eq!(gateway.antenna, None); + assert_eq!(gateway.elevation, None); + assert_eq!(gateway.azimuth, None); + assert_eq!(gateway.location, gw_insert.location.map(|v| v as u64)); + assert_eq!(gateway.location_changed_at, gw_insert.refreshed_at); // matches logic in tracker + assert_eq!(gateway.location_asserts, gw_insert.location.map(|_| 1)); + + // Update sample gateways + gateway_metadata_db::update_gateway(&pool, &gw_insert.asset, new_loc, now).await?; + } + + // now run the tracker again after updates + tracker::execute(&pool, &pool).await?; + + // We should have TOTAL + sample_size gateways in the DB + let total = count_gateways(&pool).await?; + assert_eq!(TOTAL as i64 + sample_size as i64, total); + + for gw_insert in sample.clone() { + let gateway = Gateway::get_by_address(&pool, &gw_insert.key) + .await? + .expect("gateway not found"); + + assert_eq!(gateway.address, gw_insert.key.clone()); + assert_eq!(gateway.gateway_type, GatewayType::WifiIndoor); + assert_eq!(gateway.created_at, gw_insert.created_at); + assert_eq!(gateway.refreshed_at, now); + assert_eq!(gateway.last_changed_at, now); + assert_eq!(gateway.antenna, None); + assert_eq!(gateway.elevation, None); + assert_eq!(gateway.azimuth, None); + assert_eq!(gateway.location, Some(0)); + assert_eq!(gateway.location_changed_at, Some(now)); + assert_eq!(gateway.location_asserts, Some(2)); + } Ok(()) } + +async fn count_gateways(pool: &PgPool) -> anyhow::Result { + let count: (i64,) = sqlx::query_as( + r#" + SELECT COUNT(*) FROM gateways; + "#, + ) + .fetch_one(pool) + .await?; + + Ok(count.0) +} From 029dd398712ec93102ee62ee5aba1d6662de7fc6 Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 6 Oct 2025 16:18:03 -0700 Subject: [PATCH 03/35] Fix gateway DB calls --- mobile_config/src/gateway/db.rs | 62 ++----------- .../tests/integrations/gateway_db.rs | 92 ++----------------- .../tests/integrations/gateway_tracker.rs | 1 + 3 files changed, 13 insertions(+), 142 deletions(-) diff --git a/mobile_config/src/gateway/db.rs b/mobile_config/src/gateway/db.rs index 6f21ffe0a..2ebc4cbbe 100644 --- a/mobile_config/src/gateway/db.rs +++ b/mobile_config/src/gateway/db.rs @@ -196,20 +196,7 @@ impl Gateway { ) -> anyhow::Result> { let gateway = sqlx::query_as::<_, Self>( r#" - SELECT - address, - gateway_type, - created_at, - inserted_at, - refreshed_at, - last_changed_at, - hash, - antenna, - elevation, - azimuth, - location, - location_changed_at, - location_asserts + SELECT * FROM gateways WHERE address = $1 ORDER BY inserted_at DESC @@ -231,20 +218,7 @@ impl Gateway { let rows = sqlx::query_as::<_, Self>( r#" - SELECT DISTINCT ON (address) - address, - gateway_type, - created_at, - inserted_at, - refreshed_at, - last_changed_at, - hash, - antenna, - elevation, - azimuth, - location, - location_changed_at, - location_asserts + SELECT DISTINCT ON (address) * FROM gateways WHERE address = ANY($1) ORDER BY address, inserted_at DESC @@ -266,23 +240,11 @@ impl Gateway { sqlx::query_as::<_, Self>( r#" - SELECT - address, - gateway_type, - created_at, - inserted_at, - refreshed_at, - last_changed_at, - hash, - antenna, - elevation, - azimuth, - location, - location_changed_at, - location_asserts + SELECT DISTINCT ON (address) * FROM gateways WHERE address = ANY($1) AND last_changed_at >= $2 + ORDER BY address, inserted_at DESC "#, ) .bind(addr_array) @@ -300,20 +262,7 @@ impl Gateway { ) -> impl Stream + 'a { sqlx::query_as::<_, Self>( r#" - SELECT - address, - gateway_type, - created_at, - inserted_at, - refreshed_at, - last_changed_at, - hash, - antenna, - elevation, - azimuth, - location, - location_changed_at, - location_asserts + SELECT DISTINCT ON (address) * FROM gateways WHERE gateway_type = ANY($1) AND last_changed_at >= $2 @@ -321,6 +270,7 @@ impl Gateway { $3::timestamptz IS NULL OR (location IS NOT NULL AND location_changed_at >= $3) ) + ORDER BY address, inserted_at DESC "#, ) .bind(types) diff --git a/mobile_config/tests/integrations/gateway_db.rs b/mobile_config/tests/integrations/gateway_db.rs index 516df582a..9c56b102a 100644 --- a/mobile_config/tests/integrations/gateway_db.rs +++ b/mobile_config/tests/integrations/gateway_db.rs @@ -57,87 +57,6 @@ async fn gateway_bulk_insert_and_get(pool: PgPool) -> anyhow::Result<()> { Ok(()) } -#[sqlx::test] -async fn gateway_bulk_upsert_updates_and_change(pool: PgPool) -> anyhow::Result<()> { - // seed two rows at t0 - let t0 = Utc::now(); - let a1 = pk_binary(); - let a2 = pk_binary(); - - let mut g1 = gw(a1.clone(), GatewayType::WifiIndoor, t0); - let mut g2 = gw(a2.clone(), GatewayType::WifiOutdoor, t0); - let _ = Gateway::insert_bulk(&pool, &[g1.clone(), g2.clone()]).await?; - - // upsert at t1: change only timestamps for g1 (no loc/hash change) - // and change location for g2 (should bump last_changed_at) - let t1 = Utc::now(); - - g1.inserted_at = t1; - g1.refreshed_at = t1; - // leave g1.location / g1.hash the same - - g2.inserted_at = t1; - g2.refreshed_at = t1; - g2.location = Some(456); // change => last_changed_at should bump to t1 - // g2.hash unchanged - - let affected = Gateway::insert_bulk(&pool, &[g1.clone(), g2.clone()]).await?; - // 2 rows should be affected (both upserted) - assert_eq!(affected, 2); - - // verify g1: timestamps updated, last_changed_at unchanged (no relevant change) - let got1 = Gateway::get_by_address(&pool, &a1) - .await? - .expect("row should exist"); - assert_eq!(got1.inserted_at, common::nanos_trunc(t1)); - assert_eq!(got1.refreshed_at, common::nanos_trunc(t1)); - assert_eq!( - got1.last_changed_at, - common::nanos_trunc(t0), - "no loc/hash change ⇒ last_changed_at stays t0" - ); - assert_eq!(got1.location, Some(123)); - assert_eq!(got1.hash, "h0"); - - // verify g2: timestamps updated, last_changed_at bumped due to location change - let got2 = Gateway::get_by_address(&pool, &a2) - .await? - .expect("row should exist"); - assert_eq!(got2.inserted_at, common::nanos_trunc(t1)); - assert_eq!(got2.refreshed_at, common::nanos_trunc(t1)); - assert_eq!( - got2.last_changed_at, - common::nanos_trunc(t1), - "location changed ⇒ last_changed_at = refreshed_at" - ); - assert_eq!(got2.location, Some(456)); - assert_eq!(got2.hash, "h0"); - - // second upsert at t2: change hash only for g1, ensure bump - let t2 = Utc::now(); - - g1.inserted_at = t2; - g1.refreshed_at = t2; - g1.hash = "h1".into(); // change ⇒ bump last_changed_at - - let affected2 = Gateway::insert_bulk(&pool, &[g1.clone()]).await?; - assert_eq!(affected2, 1); - - let got1b = Gateway::get_by_address(&pool, &a1) - .await? - .expect("row should exist"); - assert_eq!(got1b.inserted_at, common::nanos_trunc(t2)); - assert_eq!(got1b.refreshed_at, common::nanos_trunc(t2)); - assert_eq!( - got1b.last_changed_at, - common::nanos_trunc(t2), - "hash changed ⇒ last_changed_at = refreshed_at" - ); - assert_eq!(got1b.hash, "h1"); - - Ok(()) -} - #[sqlx::test] async fn stream_by_addresses_filters_by_min_last_changed_at(pool: PgPool) -> anyhow::Result<()> { let a1 = pk_binary(); @@ -161,8 +80,8 @@ async fn stream_by_addresses_filters_by_min_last_changed_at(pool: PgPool) -> any // bump g1.last_changed_at to t2 let mut g1b = g1.clone(); - g1b.hash = "h1".to_string(); - g1b.refreshed_at = t2; + g1b.hash = "x1".to_string(); + g1b.last_changed_at = t2; g1b.insert(&pool).await?; // now we should see g1 only @@ -188,7 +107,8 @@ async fn stream_by_types_filters_by_min_date(pool: PgPool) -> anyhow::Result<()> gw(pk_binary(), GatewayType::WifiOutdoor, t1) .insert(&pool) .await?; - gw(pk_binary(), GatewayType::WifiIndoor, t2) + let key = pk_binary(); + gw(key.clone(), GatewayType::WifiIndoor, t2) .insert(&pool) .await?; @@ -197,7 +117,7 @@ async fn stream_by_types_filters_by_min_date(pool: PgPool) -> anyhow::Result<()> pin_mut!(s); let first = s.next().await.expect("row expected"); assert_eq!(first.gateway_type, GatewayType::WifiIndoor); - assert_eq!(first.created_at, t2); + assert_eq!(first.address, key); assert!(s.next().await.is_none()); Ok(()) @@ -242,7 +162,7 @@ fn gw(address: PublicKeyBinary, gateway_type: GatewayType, t: chrono::DateTime anyhow::Result<()> { + // Tested with 100k const TOTAL: usize = 10_000; custom_tracing::init("mobile_config=debug,info".to_string(), Settings::default()).await?; From 6240c9d84b1dc5a76cee6aa47e363e5c63368cae Mon Sep 17 00:00:00 2001 From: Macpie Date: Mon, 6 Oct 2025 16:23:15 -0700 Subject: [PATCH 04/35] Fix Clippy --- mobile_config/tests/integrations/common/gateway_metadata_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile_config/tests/integrations/common/gateway_metadata_db.rs b/mobile_config/tests/integrations/common/gateway_metadata_db.rs index fc1fe9564..0d07c77a0 100644 --- a/mobile_config/tests/integrations/common/gateway_metadata_db.rs +++ b/mobile_config/tests/integrations/common/gateway_metadata_db.rs @@ -46,7 +46,7 @@ pub async fn insert_gateway_bulk( let device_type_json: serde_json::Value = serde_json::from_str(&gw.device_type).unwrap(); let deployment_info_json: serde_json::Value = - serde_json::from_str(&gw.deployment_info.as_deref().unwrap_or("null")) + serde_json::from_str(gw.deployment_info.as_deref().unwrap_or("null")) .unwrap(); b.push_bind(&gw.asset) From eb1498f87bac7780d6cb9ea758378cedc8e1504c Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 7 Oct 2025 14:53:28 -0700 Subject: [PATCH 05/35] PR comments --- .../20251003000000_gateways_historical.sql | 8 ++- mobile_config/src/gateway/db.rs | 64 +++++++++++++++++-- mobile_config/src/gateway/tracker.rs | 3 +- .../tests/integrations/gateway_db.rs | 4 +- .../tests/integrations/migrations.rs | 8 +-- 5 files changed, 73 insertions(+), 14 deletions(-) diff --git a/mobile_config/migrations/20251003000000_gateways_historical.sql b/mobile_config/migrations/20251003000000_gateways_historical.sql index ada65eee2..996a19b5f 100644 --- a/mobile_config/migrations/20251003000000_gateways_historical.sql +++ b/mobile_config/migrations/20251003000000_gateways_historical.sql @@ -21,4 +21,10 @@ SET NOT NULL; -- 5. Create an index on (address, inserted_at DESC) -CREATE INDEX IF NOT EXISTS gateways_address_inserted_idx ON gateways (address, inserted_at DESC); \ No newline at end of file +CREATE INDEX IF NOT EXISTS gateways_address_inserted_idx ON gateways (address, inserted_at DESC); + +-- 6. Create an PK on (address, inserted_at DESC) +ALTER TABLE + gateways +ADD + CONSTRAINT gateways_pkey PRIMARY KEY (address, inserted_at); \ No newline at end of file diff --git a/mobile_config/src/gateway/db.rs b/mobile_config/src/gateway/db.rs index 2ebc4cbbe..d4e599843 100644 --- a/mobile_config/src/gateway/db.rs +++ b/mobile_config/src/gateway/db.rs @@ -131,7 +131,7 @@ impl Gateway { b.push_bind(g.address.as_ref()) .push_bind(g.gateway_type) .push_bind(g.created_at) - .push_bind(g.inserted_at) + .push_bind(Utc::now()) .push_bind(g.refreshed_at) .push_bind(g.last_changed_at) .push_bind(g.hash.as_str()) @@ -174,7 +174,7 @@ impl Gateway { .bind(self.address.as_ref()) .bind(self.gateway_type) .bind(self.created_at) - .bind(self.inserted_at) + .bind(Utc::now()) .bind(self.refreshed_at) .bind(self.last_changed_at) .bind(self.hash.as_str()) @@ -196,7 +196,20 @@ impl Gateway { ) -> anyhow::Result> { let gateway = sqlx::query_as::<_, Self>( r#" - SELECT * + SELECT + address, + gateway_type, + created_at, + inserted_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts FROM gateways WHERE address = $1 ORDER BY inserted_at DESC @@ -218,7 +231,20 @@ impl Gateway { let rows = sqlx::query_as::<_, Self>( r#" - SELECT DISTINCT ON (address) * + SELECT DISTINCT ON (address) + address, + gateway_type, + created_at, + inserted_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts FROM gateways WHERE address = ANY($1) ORDER BY address, inserted_at DESC @@ -240,7 +266,20 @@ impl Gateway { sqlx::query_as::<_, Self>( r#" - SELECT DISTINCT ON (address) * + SELECT DISTINCT ON (address) + address, + gateway_type, + created_at, + inserted_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts FROM gateways WHERE address = ANY($1) AND last_changed_at >= $2 @@ -262,7 +301,20 @@ impl Gateway { ) -> impl Stream + 'a { sqlx::query_as::<_, Self>( r#" - SELECT DISTINCT ON (address) * + SELECT DISTINCT ON (address) + address, + gateway_type, + created_at, + inserted_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts FROM gateways WHERE gateway_type = ANY($1) AND last_changed_at >= $2 diff --git a/mobile_config/src/gateway/tracker.rs b/mobile_config/src/gateway/tracker.rs index 579f4d660..07b14c4d3 100644 --- a/mobile_config/src/gateway/tracker.rs +++ b/mobile_config/src/gateway/tracker.rs @@ -99,10 +99,10 @@ pub async fn execute(pool: &Pool, metadata: &Pool) -> anyhow to_insert.push(gw); } Some(last_gw) => { - // FYI hash includes location let loc_changed = gw.location != last_gw.location; let hash_changed = gw.hash != last_gw.hash; + // FYI hash includes location gw.last_changed_at = if hash_changed { gw.refreshed_at } else { @@ -116,6 +116,7 @@ pub async fn execute(pool: &Pool, metadata: &Pool) -> anyhow }; // We only add record if something changed + // FYI hash includes location if hash_changed { to_insert.push(gw); } diff --git a/mobile_config/tests/integrations/gateway_db.rs b/mobile_config/tests/integrations/gateway_db.rs index 9c56b102a..daae41730 100644 --- a/mobile_config/tests/integrations/gateway_db.rs +++ b/mobile_config/tests/integrations/gateway_db.rs @@ -19,7 +19,7 @@ async fn gateway_insert_and_get_by_address(pool: PgPool) -> anyhow::Result<()> { assert_eq!(gateway.gateway_type, GatewayType::WifiIndoor); assert_eq!(gateway.created_at, common::nanos_trunc(now)); - assert_eq!(gateway.inserted_at, common::nanos_trunc(now)); + assert!(gateway.inserted_at > now); assert_eq!(gateway.refreshed_at, common::nanos_trunc(now)); assert_eq!(gateway.last_changed_at, common::nanos_trunc(now)); // first insert: equals refreshed_at assert_eq!(gateway.location, Some(123)); @@ -47,7 +47,7 @@ async fn gateway_bulk_insert_and_get(pool: PgPool) -> anyhow::Result<()> { .await? .expect("row should exist"); assert_eq!(got.created_at, common::nanos_trunc(now)); - assert_eq!(got.inserted_at, common::nanos_trunc(now)); + assert!(got.inserted_at > now); assert_eq!(got.refreshed_at, common::nanos_trunc(now)); assert_eq!(got.last_changed_at, common::nanos_trunc(now)); assert_eq!(got.location, Some(123)); diff --git a/mobile_config/tests/integrations/migrations.rs b/mobile_config/tests/integrations/migrations.rs index 12d0adbb6..6c5324994 100644 --- a/mobile_config/tests/integrations/migrations.rs +++ b/mobile_config/tests/integrations/migrations.rs @@ -40,11 +40,11 @@ async fn gateways_historical(pool: PgPool) -> anyhow::Result<()> { assert!(pre_gw.address == gw.address); assert!(pre_gw.gateway_type == gw.gateway_type); - assert!(pre_gw.created_at == gw.created_at); + assert!(pre_gw.created_at == common::nanos_trunc(gw.created_at)); // The real change is updated_at renamed to inserted_at AND inserted_at = created_at; - assert!(pre_gw.created_at == gw.inserted_at); - assert!(pre_gw.refreshed_at == gw.refreshed_at); - assert!(pre_gw.last_changed_at == gw.last_changed_at); + assert!(pre_gw.created_at == common::nanos_trunc(gw.inserted_at)); + assert!(pre_gw.refreshed_at == common::nanos_trunc(gw.refreshed_at)); + assert!(pre_gw.last_changed_at == common::nanos_trunc(gw.last_changed_at)); assert!(pre_gw.hash == gw.hash); assert!(pre_gw.antenna == gw.antenna); assert!(pre_gw.elevation == gw.elevation); From e0531afdaf5eeca109a93e010700ca5a5d893c4d Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 7 Oct 2025 15:05:11 -0700 Subject: [PATCH 06/35] Add some print to debug --- mobile_config/tests/integrations/migrations.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile_config/tests/integrations/migrations.rs b/mobile_config/tests/integrations/migrations.rs index 6c5324994..66c5d6e6b 100644 --- a/mobile_config/tests/integrations/migrations.rs +++ b/mobile_config/tests/integrations/migrations.rs @@ -38,6 +38,9 @@ async fn gateways_historical(pool: PgPool) -> anyhow::Result<()> { .await? .expect("should find gateway"); + println!("pre_gw: {:?}", pre_gw); + println!("gw: {:?}", gw); + assert!(pre_gw.address == gw.address); assert!(pre_gw.gateway_type == gw.gateway_type); assert!(pre_gw.created_at == common::nanos_trunc(gw.created_at)); From 497e0e40312cd0716cfb2372355fe84155cf20a3 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 7 Oct 2025 15:15:26 -0700 Subject: [PATCH 07/35] Fix timestamp check --- mobile_config/tests/integrations/migrations.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mobile_config/tests/integrations/migrations.rs b/mobile_config/tests/integrations/migrations.rs index 66c5d6e6b..b0cfacd78 100644 --- a/mobile_config/tests/integrations/migrations.rs +++ b/mobile_config/tests/integrations/migrations.rs @@ -43,17 +43,20 @@ async fn gateways_historical(pool: PgPool) -> anyhow::Result<()> { assert!(pre_gw.address == gw.address); assert!(pre_gw.gateway_type == gw.gateway_type); - assert!(pre_gw.created_at == common::nanos_trunc(gw.created_at)); + assert!(common::nanos_trunc(pre_gw.created_at) == common::nanos_trunc(gw.created_at)); // The real change is updated_at renamed to inserted_at AND inserted_at = created_at; - assert!(pre_gw.created_at == common::nanos_trunc(gw.inserted_at)); - assert!(pre_gw.refreshed_at == common::nanos_trunc(gw.refreshed_at)); - assert!(pre_gw.last_changed_at == common::nanos_trunc(gw.last_changed_at)); + assert!(common::nanos_trunc(pre_gw.created_at) == common::nanos_trunc(gw.inserted_at)); + assert!(common::nanos_trunc(pre_gw.refreshed_at) == common::nanos_trunc(gw.refreshed_at)); + assert!(common::nanos_trunc(pre_gw.last_changed_at) == common::nanos_trunc(gw.last_changed_at)); assert!(pre_gw.hash == gw.hash); assert!(pre_gw.antenna == gw.antenna); assert!(pre_gw.elevation == gw.elevation); assert!(pre_gw.azimuth == gw.azimuth); assert!(pre_gw.location == gw.location); - assert!(pre_gw.location_changed_at == gw.location_changed_at); + assert!( + common::nanos_trunc(pre_gw.location_changed_at.unwrap()) + == common::nanos_trunc(gw.location_changed_at.unwrap()) + ); assert!(pre_gw.location_asserts == gw.location_asserts); Ok(()) From ca0c79f10d2818d5f8cc8bb02ff2ca620cdc164a Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Fri, 17 Oct 2025 14:25:54 +0100 Subject: [PATCH 08/35] Added new API for retrieving historical gateway info --- file_store/src/traits/msg_verify.rs | 1 + .../20251003000000_gateways_historical.sql | 8 +- mobile_config/src/gateway/db.rs | 42 +++++- mobile_config/src/gateway/service/info.rs | 9 ++ mobile_config/src/gateway/service/mod.rs | 64 +++++++++ .../tests/integrations/gateway_db.rs | 31 +++++ .../tests/integrations/gateway_service.rs | 121 +++++++++++++++++- 7 files changed, 263 insertions(+), 13 deletions(-) diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 42e04b0a2..0e2fb1b3d 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -102,6 +102,7 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamResV3, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); +impl_msg_verify!(mobile_config::GatewayInfoHistoricalReq, signature); impl_msg_verify!(sub_dao::SubDaoEpochRewardInfoReqV1, signature); impl_msg_verify!(sub_dao::SubDaoEpochRewardInfoResV1, signature); impl_msg_verify!(poc_mobile::SubscriberVerifiedMappingEventReqV1, signature); diff --git a/mobile_config/migrations/20251003000000_gateways_historical.sql b/mobile_config/migrations/20251003000000_gateways_historical.sql index 996a19b5f..5fb7aa774 100644 --- a/mobile_config/migrations/20251003000000_gateways_historical.sql +++ b/mobile_config/migrations/20251003000000_gateways_historical.sql @@ -12,13 +12,11 @@ UPDATE SET inserted_at = created_at; --- 4. Ensure inserted_at is NOT NULL +-- 4. Ensure inserted_at is NOT NULL and has a default value of now() ALTER TABLE gateways -ALTER COLUMN - inserted_at -SET - NOT NULL; +ALTER COLUMN inserted_at SET DEFAULT now(), +ALTER COLUMN inserted_at SET NOT NULL; -- 5. Create an index on (address, inserted_at DESC) CREATE INDEX IF NOT EXISTS gateways_address_inserted_idx ON gateways (address, inserted_at DESC); diff --git a/mobile_config/src/gateway/db.rs b/mobile_config/src/gateway/db.rs index d4e599843..994676a8d 100644 --- a/mobile_config/src/gateway/db.rs +++ b/mobile_config/src/gateway/db.rs @@ -114,7 +114,6 @@ impl Gateway { address, gateway_type, created_at, - inserted_at, refreshed_at, last_changed_at, hash, @@ -131,7 +130,6 @@ impl Gateway { b.push_bind(g.address.as_ref()) .push_bind(g.gateway_type) .push_bind(g.created_at) - .push_bind(Utc::now()) .push_bind(g.refreshed_at) .push_bind(g.last_changed_at) .push_bind(g.hash.as_str()) @@ -154,7 +152,6 @@ impl Gateway { address, gateway_type, created_at, - inserted_at, refreshed_at, last_changed_at, hash, @@ -167,14 +164,13 @@ impl Gateway { ) VALUES ( $1, $2, $3, $4, $5, $6, $7, - $8, $9, $10, $11, $12, $13 + $8, $9, $10, $11, $12 ) "#, ) .bind(self.address.as_ref()) .bind(self.gateway_type) .bind(self.created_at) - .bind(Utc::now()) .bind(self.refreshed_at) .bind(self.last_changed_at) .bind(self.hash.as_str()) @@ -257,6 +253,42 @@ impl Gateway { Ok(rows) } + pub async fn get_by_address_and_inserted_at<'a>( + db: impl PgExecutor<'a>, + address: &PublicKeyBinary, + inserted_at_max: &DateTime, + ) -> anyhow::Result> { + let gateway = sqlx::query_as::<_, Self>( + r#" + SELECT + address, + gateway_type, + created_at, + inserted_at, + refreshed_at, + last_changed_at, + hash, + antenna, + elevation, + azimuth, + location, + location_changed_at, + location_asserts + FROM gateways + WHERE address = $1 + AND inserted_at <= $2 + ORDER BY inserted_at DESC + LIMIT 1 + "#, + ) + .bind(address.as_ref()) + .bind(inserted_at_max) + .fetch_optional(db) + .await?; + + Ok(gateway) + } + pub fn stream_by_addresses<'a>( db: impl PgExecutor<'a> + 'a, addresses: Vec, diff --git a/mobile_config/src/gateway/service/info.rs b/mobile_config/src/gateway/service/info.rs index f64828e96..16c8ba973 100644 --- a/mobile_config/src/gateway/service/info.rs +++ b/mobile_config/src/gateway/service/info.rs @@ -453,3 +453,12 @@ pub fn stream_by_types<'a>( Gateway::stream_by_types(db, gateway_types, min_date, None).map(|gateway| gateway.into()) } + +pub async fn get_by_address_and_inserted_at( + db: impl PgExecutor<'_>, + pubkey_bin: &PublicKeyBinary, + inserted_at_max: &DateTime, +) -> anyhow::Result> { + let gateway = Gateway::get_by_address_and_inserted_at(db, pubkey_bin, inserted_at_max).await?; + Ok(gateway.map(|g| g.into())) +} diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index 5e65d278e..3696c3f34 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -15,6 +15,7 @@ use helium_proto::{ self, GatewayInfoBatchReqV1, GatewayInfoReqV1, GatewayInfoResV1, GatewayInfoResV2, GatewayInfoStreamReqV1, GatewayInfoStreamReqV2, GatewayInfoStreamReqV3, GatewayInfoStreamResV1, GatewayInfoStreamResV2, GatewayInfoStreamResV3, GatewayInfoV2, + GatewayInfoHistoricalReq }, Message, }; @@ -63,6 +64,18 @@ impl GatewayService { self.verify_request_signature(&signer, request) } + fn verify_request_signature_for_historical_info(&self, request: &GatewayInfoHistoricalReq) -> Result<(), Status> { + let signer = verify_public_key(&request.signer)?; + let address = verify_public_key(&request.address)?; + + if address == signer && request.verify(&signer).is_ok() { + tracing::debug!(%signer, "self authorized"); + return Ok(()); + } + + self.verify_request_signature(&signer, request) + } + fn sign_response(&self, response: &[u8]) -> Result, Status> { self.signing_key .sign(response) @@ -155,6 +168,57 @@ impl mobile_config::Gateway for GatewayService { ) } + async fn info_historical(&self, request: Request) -> GrpcResult { + let request = request.into_inner(); + telemetry::count_request("gateway", "info-v2"); + custom_tracing::record_b58("pub_key", &request.address); + custom_tracing::record_b58("signer", &request.signer); + + self.verify_request_signature_for_historical_info(&request)?; + + let pubkey: PublicKeyBinary = request.address.into(); + tracing::debug!(pubkey = pubkey.to_string(), "fetching historical gateway info (v2)"); + + let query_time = Utc + .timestamp_opt(request.query_time as i64, 0) + .single() + .ok_or(Status::invalid_argument("Invalid query_time argument"))?; + + info::get_by_address_and_inserted_at(&self.pool, &pubkey, &query_time) + .await + .map_err(|_| { + println!("error fetching historical gateway info (v2)"); + Status::internal("error fetching historical gateway info") + })? + .map_or_else( + || { + telemetry::count_gateway_chain_lookup("not-found"); + println!("Could not find in db"); + Err(Status::not_found(pubkey.to_string())) + }, + |info| { + if info.metadata.is_some() { + telemetry::count_gateway_chain_lookup("asserted"); + } else { + telemetry::count_gateway_chain_lookup("not-asserted"); + }; + + let info: GatewayInfoV2 = info + .try_into() + .map_err(|_| Status::internal("error serializing historical gateway info (v2)"))?; + + let mut res = GatewayInfoResV2 { + info: Some(info), + timestamp: Utc::now().encode_timestamp(), + signer: self.signing_key.public_key().into(), + signature: vec![], + }; + res.signature = self.sign_response(&res.encode_to_vec())?; + Ok(Response::new(res)) + }, + ) + } + // Deprecated type info_batchStream = GrpcStreamResult; async fn info_batch( diff --git a/mobile_config/tests/integrations/gateway_db.rs b/mobile_config/tests/integrations/gateway_db.rs index daae41730..4df7844fa 100644 --- a/mobile_config/tests/integrations/gateway_db.rs +++ b/mobile_config/tests/integrations/gateway_db.rs @@ -27,6 +27,37 @@ async fn gateway_insert_and_get_by_address(pool: PgPool) -> anyhow::Result<()> { Ok(()) } +#[sqlx::test] +async fn gateway_get_by_address_and_inserted_at(pool: PgPool) -> anyhow::Result<()> { + let addr = pk_binary(); + let now = Utc::now(); + + // Insert gateway first time + let gateway = gw(addr.clone(), GatewayType::WifiIndoor, now); + gateway.insert(&pool).await?; + + // Insert gateway second time with different type + let gateway = gw(addr.clone(), GatewayType::WifiDataOnly, now); + gateway.insert(&pool).await?; + + let later = now + chrono::Duration::minutes(10); + + let gateway = Gateway::get_by_address_and_inserted_at(&pool, &addr, &later) + .await? + .expect("gateway should exist"); + + // Assert most recent gateway was returned + assert_eq!(gateway.gateway_type, GatewayType::WifiDataOnly); + assert_eq!(gateway.created_at, common::nanos_trunc(now)); + assert!(gateway.inserted_at > now); + assert_eq!(gateway.refreshed_at, common::nanos_trunc(now)); + assert_eq!(gateway.last_changed_at, common::nanos_trunc(now)); + assert_eq!(gateway.location, Some(123)); + assert_eq!(gateway.hash, "h0"); + + Ok(()) +} + #[sqlx::test] async fn gateway_bulk_insert_and_get(pool: PgPool) -> anyhow::Result<()> { let now = Utc::now(); diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index 839c29e41..f9d132e41 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -1,5 +1,5 @@ use crate::common::{make_keypair, spawn_gateway_service}; -use chrono::{Duration, Utc}; +use chrono::{DateTime, Duration, Utc}; use futures::stream::StreamExt; use helium_crypto::{Keypair, PublicKey, Sign}; use helium_proto::services::mobile_config::{ @@ -16,14 +16,14 @@ use mobile_config::{ }; use prost::Message; use sqlx::PgPool; -use std::{sync::Arc, vec}; +use std::{sync::Arc, time, vec}; use tokio::net::TcpListener; use tonic::{transport, Code}; #[sqlx::test] async fn gateway_info_authorization_errors(pool: PgPool) -> anyhow::Result<()> { // NOTE(mj): The information we're requesting does not exist in the DB for - // this test. But we're only interested in Authization Errors. + // this test. But we're only interested in Authorization Errors. let admin_key = make_keypair(); // unlimited access let gw_key = make_keypair(); // access to self @@ -812,6 +812,104 @@ async fn gateway_info_v2(pool: PgPool) -> anyhow::Result<()> { Ok(()) } +#[sqlx::test] +async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { + let admin_key = make_keypair(); + + let address = make_keypair().public_key().clone(); + let loc_original = 631711281837647359_u64; + + let created_at = Utc::now() - Duration::hours(5); + let refreshed_at = Utc::now() - Duration::hours(3); + + let gateway_original = Gateway { + address: address.clone().into(), + gateway_type: GatewayType::WifiIndoor, + created_at, + inserted_at: refreshed_at, + refreshed_at, + last_changed_at: refreshed_at, + hash: "".to_string(), + antenna: Some(18), + elevation: Some(2), + azimuth: Some(161), + location: Some(loc_original), + location_changed_at: Some(refreshed_at), + location_asserts: Some(1), + }; + gateway_original.insert(&pool).await?; + + let query_time_original = Utc::now() + Duration::milliseconds(800); + tokio::time::sleep(time::Duration::from_millis(800)).await; + + let loc_recent = 631711281837647358_u64; + + let gateway_recent = Gateway { + address: address.clone().into(), + gateway_type: GatewayType::WifiIndoor, + created_at, + inserted_at: created_at, + refreshed_at: created_at, + last_changed_at: created_at, + hash: "".to_string(), + antenna: Some(18), + elevation: Some(2), + azimuth: Some(161), + location: Some(loc_recent), + location_changed_at: Some(created_at), + location_asserts: Some(1), + }; + gateway_recent.insert(&pool).await?; + + let (addr, _handle) = spawn_gateway_service(pool.clone(), admin_key.public_key().clone()).await; + let mut client = GatewayClient::connect(addr).await?; + + // Get most recent gateway info + let query_time = Utc::now() + Duration::minutes(10); + let res = info_historical_request(&mut client, &address, &admin_key, &query_time).await; + + let gw_info = res?.info.unwrap(); + assert_eq!(gw_info.address, address.to_vec()); + let deployment_info = gw_info.metadata.clone().unwrap().deployment_info.unwrap(); + match deployment_info { + DeploymentInfo::WifiDeploymentInfo(v) => { + assert_eq!(v.antenna, 18); + assert_eq!(v.azimuth, 161); + assert_eq!(v.elevation, 2); + } + DeploymentInfo::CbrsDeploymentInfo(_) => panic!(), + }; + + // Assert that recent gateway was returned + assert_eq!( + u64::from_str_radix(&gw_info.metadata.clone().unwrap().location, 16).unwrap(), + loc_recent + ); + + // Get original gateway info by using an earlier inserted_at condition + let res = info_historical_request(&mut client, &address, &admin_key, &query_time_original).await; + + let gw_info = res?.info.unwrap(); + assert_eq!(gw_info.address, address.to_vec()); + let deployment_info = gw_info.metadata.clone().unwrap().deployment_info.unwrap(); + match deployment_info { + DeploymentInfo::WifiDeploymentInfo(v) => { + assert_eq!(v.antenna, 18); + assert_eq!(v.azimuth, 161); + assert_eq!(v.elevation, 2); + } + DeploymentInfo::CbrsDeploymentInfo(_) => panic!(), + }; + + // Assert that original gateway was returned + assert_eq!( + u64::from_str_radix(&gw_info.metadata.clone().unwrap().location, 16).unwrap(), + loc_original + ); + + Ok(()) +} + fn make_signed_info_request(address: &PublicKey, signer: &Keypair) -> proto::GatewayInfoReqV1 { let mut req = proto::GatewayInfoReqV1 { address: address.to_vec(), @@ -837,6 +935,23 @@ async fn info_request_v2( Ok(res) } +async fn info_historical_request( + client: &mut GatewayClient, + address: &PublicKey, + signer: &Keypair, + query_time: &DateTime +) -> anyhow::Result { + let mut req = proto::GatewayInfoHistoricalReq { + address: address.to_vec(), + signer: signer.public_key().to_vec(), + signature: vec![], + query_time: query_time.timestamp() as u64, + }; + req.signature = signer.sign(&req.encode_to_vec()).unwrap(); + let res = client.info_historical(req).await?.into_inner(); + Ok(res) +} + async fn gateway_info_stream_v1( client: &mut GatewayClient, signer: &Keypair, From 4dfc96ca02a7f160d130bcd87704b75711e646f4 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Fri, 17 Oct 2025 15:01:11 +0100 Subject: [PATCH 09/35] Add v1 to historical gateway info protobuf --- file_store/src/traits/msg_verify.rs | 2 +- mobile_config/src/gateway/service/mod.rs | 6 +++--- mobile_config/tests/integrations/gateway_service.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 0e2fb1b3d..59937268d 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -102,7 +102,7 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamResV3, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); -impl_msg_verify!(mobile_config::GatewayInfoHistoricalReq, signature); +impl_msg_verify!(mobile_config::GatewayInfoHistoricalReqV1, signature); impl_msg_verify!(sub_dao::SubDaoEpochRewardInfoReqV1, signature); impl_msg_verify!(sub_dao::SubDaoEpochRewardInfoResV1, signature); impl_msg_verify!(poc_mobile::SubscriberVerifiedMappingEventReqV1, signature); diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index 3696c3f34..a27ff7a24 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -15,7 +15,7 @@ use helium_proto::{ self, GatewayInfoBatchReqV1, GatewayInfoReqV1, GatewayInfoResV1, GatewayInfoResV2, GatewayInfoStreamReqV1, GatewayInfoStreamReqV2, GatewayInfoStreamReqV3, GatewayInfoStreamResV1, GatewayInfoStreamResV2, GatewayInfoStreamResV3, GatewayInfoV2, - GatewayInfoHistoricalReq + GatewayInfoHistoricalReqV1 }, Message, }; @@ -64,7 +64,7 @@ impl GatewayService { self.verify_request_signature(&signer, request) } - fn verify_request_signature_for_historical_info(&self, request: &GatewayInfoHistoricalReq) -> Result<(), Status> { + fn verify_request_signature_for_historical_info(&self, request: &GatewayInfoHistoricalReqV1) -> Result<(), Status> { let signer = verify_public_key(&request.signer)?; let address = verify_public_key(&request.address)?; @@ -168,7 +168,7 @@ impl mobile_config::Gateway for GatewayService { ) } - async fn info_historical(&self, request: Request) -> GrpcResult { + async fn info_historical(&self, request: Request) -> GrpcResult { let request = request.into_inner(); telemetry::count_request("gateway", "info-v2"); custom_tracing::record_b58("pub_key", &request.address); diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index f9d132e41..1ebca1f09 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -941,7 +941,7 @@ async fn info_historical_request( signer: &Keypair, query_time: &DateTime ) -> anyhow::Result { - let mut req = proto::GatewayInfoHistoricalReq { + let mut req = proto::GatewayInfoHistoricalReqV1 { address: address.to_vec(), signer: signer.public_key().to_vec(), signature: vec![], From e7def30976a236f69b64ce91c285871d96132d71 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Fri, 17 Oct 2025 15:34:28 +0100 Subject: [PATCH 10/35] Refactored gateway service reducing code dupe --- mobile_config/src/gateway/service/mod.rs | 96 +++++++++--------------- 1 file changed, 37 insertions(+), 59 deletions(-) diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index a27ff7a24..f506e375f 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -22,6 +22,7 @@ use helium_proto::{ use sqlx::{Pool, Postgres}; use std::sync::Arc; use tonic::{Request, Response, Status}; +use crate::gateway::service::info::GatewayInfo; pub mod info; pub mod info_v3; @@ -52,21 +53,12 @@ impl GatewayService { Err(Status::permission_denied("unauthorized request signature")) } - fn verify_request_signature_for_info(&self, request: &GatewayInfoReqV1) -> Result<(), Status> { - let signer = verify_public_key(&request.signer)?; - let address = verify_public_key(&request.address)?; - - if address == signer && request.verify(&signer).is_ok() { - tracing::debug!(%signer, "self authorized"); - return Ok(()); - } - - self.verify_request_signature(&signer, request) - } - - fn verify_request_signature_for_historical_info(&self, request: &GatewayInfoHistoricalReqV1) -> Result<(), Status> { - let signer = verify_public_key(&request.signer)?; - let address = verify_public_key(&request.address)?; + fn verify_request_signature_for_info(&self, request: &R, signer: &Vec, address: &Vec) -> Result<(), Status> + where + R: MsgVerify, + { + let signer = verify_public_key(signer)?; + let address = verify_public_key(address)?; if address == signer && request.verify(&signer).is_ok() { tracing::debug!(%signer, "self authorized"); @@ -81,6 +73,30 @@ impl GatewayService { .sign(response) .map_err(|_| Status::internal("response signing error")) } + + fn map_info_v2_response( + &self, + info: GatewayInfo + ) -> GrpcResult { + if info.metadata.is_some() { + telemetry::count_gateway_chain_lookup("asserted"); + } else { + telemetry::count_gateway_chain_lookup("not-asserted"); + }; + + let info: GatewayInfoV2 = info + .try_into() + .map_err(|_| Status::internal("error serializing historical gateway info (v2)"))?; + + let mut res = GatewayInfoResV2 { + info: Some(info), + timestamp: Utc::now().encode_timestamp(), + signer: self.signing_key.public_key().into(), + signature: vec![], + }; + res.signature = self.sign_response(&res.encode_to_vec())?; + Ok(Response::new(res)) + } } #[tonic::async_trait] @@ -92,7 +108,7 @@ impl mobile_config::Gateway for GatewayService { custom_tracing::record_b58("pub_key", &request.address); custom_tracing::record_b58("signer", &request.signer); - self.verify_request_signature_for_info(&request)?; + self.verify_request_signature_for_info(&request, &request.signer, &request.address)?; let pubkey: PublicKeyBinary = request.address.into(); tracing::debug!(pubkey = pubkey.to_string(), "fetching gateway info"); @@ -132,7 +148,7 @@ impl mobile_config::Gateway for GatewayService { custom_tracing::record_b58("pub_key", &request.address); custom_tracing::record_b58("signer", &request.signer); - self.verify_request_signature_for_info(&request)?; + self.verify_request_signature_for_info(&request, &request.signer, &request.address)?; let pubkey: PublicKeyBinary = request.address.into(); tracing::debug!(pubkey = pubkey.to_string(), "fetching gateway info (v2)"); @@ -145,26 +161,7 @@ impl mobile_config::Gateway for GatewayService { telemetry::count_gateway_chain_lookup("not-found"); Err(Status::not_found(pubkey.to_string())) }, - |info| { - if info.metadata.is_some() { - telemetry::count_gateway_chain_lookup("asserted"); - } else { - telemetry::count_gateway_chain_lookup("not-asserted"); - }; - - let info: GatewayInfoV2 = info - .try_into() - .map_err(|_| Status::internal("error serializing gateway info (v2)"))?; - - let mut res = GatewayInfoResV2 { - info: Some(info), - timestamp: Utc::now().encode_timestamp(), - signer: self.signing_key.public_key().into(), - signature: vec![], - }; - res.signature = self.sign_response(&res.encode_to_vec())?; - Ok(Response::new(res)) - }, + |info| self.map_info_v2_response(info), ) } @@ -174,7 +171,7 @@ impl mobile_config::Gateway for GatewayService { custom_tracing::record_b58("pub_key", &request.address); custom_tracing::record_b58("signer", &request.signer); - self.verify_request_signature_for_historical_info(&request)?; + self.verify_request_signature_for_info(&request, &request.signer, &request.address)?; let pubkey: PublicKeyBinary = request.address.into(); tracing::debug!(pubkey = pubkey.to_string(), "fetching historical gateway info (v2)"); @@ -196,26 +193,7 @@ impl mobile_config::Gateway for GatewayService { println!("Could not find in db"); Err(Status::not_found(pubkey.to_string())) }, - |info| { - if info.metadata.is_some() { - telemetry::count_gateway_chain_lookup("asserted"); - } else { - telemetry::count_gateway_chain_lookup("not-asserted"); - }; - - let info: GatewayInfoV2 = info - .try_into() - .map_err(|_| Status::internal("error serializing historical gateway info (v2)"))?; - - let mut res = GatewayInfoResV2 { - info: Some(info), - timestamp: Utc::now().encode_timestamp(), - signer: self.signing_key.public_key().into(), - signature: vec![], - }; - res.signature = self.sign_response(&res.encode_to_vec())?; - Ok(Response::new(res)) - }, + |info| self.map_info_v2_response(info), ) } @@ -522,4 +500,4 @@ where )))) }) .await?) -} +} \ No newline at end of file From bb648435aef59f696652372013c1839af670c3ea Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Wed, 22 Oct 2025 11:25:33 +0100 Subject: [PATCH 11/35] Changed proto branch --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e67d19e14..9f023af23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,10 +125,10 @@ aws-smithy-types-convert = { version = "0.60.9", features = [ url = "2.5.4" ### Protobuf -helium-proto = { git = "https://github.com/helium/proto", branch = "master", features = [ +helium-proto = { git = "https://github.com/helium/proto", branch = "connor/historical", features = [ "services", ] } -beacon = { git = "https://github.com/helium/proto", branch = "master" } +beacon = { git = "https://github.com/helium/proto", branch = "connor/historical" } # Pickup versions from above prost = "*" tonic = { version = "*", features = ["tls-aws-lc", "tls-native-roots"] } From bda0b40d134f65e8b768d5a0b3343393263a1490 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Wed, 22 Oct 2025 11:55:36 +0100 Subject: [PATCH 12/35] Fixed formatting errors --- mobile_config/src/gateway/db.rs | 8 +++--- mobile_config/src/gateway/service/mod.rs | 27 +++++++++---------- .../tests/integrations/gateway_service.rs | 11 +++++--- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/mobile_config/src/gateway/db.rs b/mobile_config/src/gateway/db.rs index 994676a8d..7821f752e 100644 --- a/mobile_config/src/gateway/db.rs +++ b/mobile_config/src/gateway/db.rs @@ -281,10 +281,10 @@ impl Gateway { LIMIT 1 "#, ) - .bind(address.as_ref()) - .bind(inserted_at_max) - .fetch_optional(db) - .await?; + .bind(address.as_ref()) + .bind(inserted_at_max) + .fetch_optional(db) + .await?; Ok(gateway) } diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index f506e375f..5f88b95de 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -1,5 +1,5 @@ use crate::{ - gateway::service::{info::DeviceType, info_v3::DeviceTypeV2}, + gateway::service::{info::DeviceType, info_v3::DeviceTypeV2, info::GatewayInfo}, key_cache::KeyCache, telemetry, verify_public_key, GrpcResult, GrpcStreamResult, }; @@ -12,17 +12,16 @@ use futures::{ use helium_crypto::{Keypair, PublicKey, PublicKeyBinary, Sign}; use helium_proto::{ services::mobile_config::{ - self, GatewayInfoBatchReqV1, GatewayInfoReqV1, GatewayInfoResV1, GatewayInfoResV2, - GatewayInfoStreamReqV1, GatewayInfoStreamReqV2, GatewayInfoStreamReqV3, - GatewayInfoStreamResV1, GatewayInfoStreamResV2, GatewayInfoStreamResV3, GatewayInfoV2, - GatewayInfoHistoricalReqV1 + self, GatewayInfoBatchReqV1, GatewayInfoHistoricalReqV1, GatewayInfoReqV1, + GatewayInfoResV1, GatewayInfoResV2, GatewayInfoStreamReqV1, GatewayInfoStreamReqV2, + GatewayInfoStreamReqV3, GatewayInfoStreamResV1, GatewayInfoStreamResV2, + GatewayInfoStreamResV3, GatewayInfoV2, }, Message, }; use sqlx::{Pool, Postgres}; use std::sync::Arc; use tonic::{Request, Response, Status}; -use crate::gateway::service::info::GatewayInfo; pub mod info; pub mod info_v3; @@ -53,7 +52,12 @@ impl GatewayService { Err(Status::permission_denied("unauthorized request signature")) } - fn verify_request_signature_for_info(&self, request: &R, signer: &Vec, address: &Vec) -> Result<(), Status> + fn verify_request_signature_for_info( + &self, + request: &R, + signer: &Vec, + address: &[u8] + ) -> Result<(), Status> where R: MsgVerify, { @@ -74,10 +78,7 @@ impl GatewayService { .map_err(|_| Status::internal("response signing error")) } - fn map_info_v2_response( - &self, - info: GatewayInfo - ) -> GrpcResult { + fn map_info_v2_response(&self, info: GatewayInfo) -> GrpcResult { if info.metadata.is_some() { telemetry::count_gateway_chain_lookup("asserted"); } else { @@ -184,13 +185,11 @@ impl mobile_config::Gateway for GatewayService { info::get_by_address_and_inserted_at(&self.pool, &pubkey, &query_time) .await .map_err(|_| { - println!("error fetching historical gateway info (v2)"); Status::internal("error fetching historical gateway info") })? .map_or_else( || { telemetry::count_gateway_chain_lookup("not-found"); - println!("Could not find in db"); Err(Status::not_found(pubkey.to_string())) }, |info| self.map_info_v2_response(info), @@ -500,4 +499,4 @@ where )))) }) .await?) -} \ No newline at end of file +} diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index 1ebca1f09..ec9ec14e6 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -861,12 +861,14 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { }; gateway_recent.insert(&pool).await?; - let (addr, _handle) = spawn_gateway_service(pool.clone(), admin_key.public_key().clone()).await; + let (addr, _handle) = + spawn_gateway_service(pool.clone(), admin_key.public_key().clone()).await; let mut client = GatewayClient::connect(addr).await?; // Get most recent gateway info let query_time = Utc::now() + Duration::minutes(10); - let res = info_historical_request(&mut client, &address, &admin_key, &query_time).await; + let res = + info_historical_request(&mut client, &address, &admin_key, &query_time).await; let gw_info = res?.info.unwrap(); assert_eq!(gw_info.address, address.to_vec()); @@ -887,7 +889,8 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { ); // Get original gateway info by using an earlier inserted_at condition - let res = info_historical_request(&mut client, &address, &admin_key, &query_time_original).await; + let res = + info_historical_request(&mut client, &address, &admin_key, &query_time_original).await; let gw_info = res?.info.unwrap(); assert_eq!(gw_info.address, address.to_vec()); @@ -939,7 +942,7 @@ async fn info_historical_request( client: &mut GatewayClient, address: &PublicKey, signer: &Keypair, - query_time: &DateTime + query_time: &DateTime, ) -> anyhow::Result { let mut req = proto::GatewayInfoHistoricalReqV1 { address: address.to_vec(), From 968b6bedba21e8ee80890ab2f1a3027f65e034df Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Wed, 22 Oct 2025 13:12:39 +0100 Subject: [PATCH 13/35] Gateway historical info test assertions updated --- .../tests/integrations/gateway_service.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index ec9ec14e6..d46898983 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -816,23 +816,23 @@ async fn gateway_info_v2(pool: PgPool) -> anyhow::Result<()> { async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { let admin_key = make_keypair(); - let address = make_keypair().public_key().clone(); + let address_original = make_keypair().public_key().clone(); let loc_original = 631711281837647359_u64; let created_at = Utc::now() - Duration::hours(5); let refreshed_at = Utc::now() - Duration::hours(3); let gateway_original = Gateway { - address: address.clone().into(), + address: address_original.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, inserted_at: refreshed_at, refreshed_at, last_changed_at: refreshed_at, hash: "".to_string(), - antenna: Some(18), - elevation: Some(2), - azimuth: Some(161), + antenna: Some(10), + elevation: Some(4), + azimuth: Some(168), location: Some(loc_original), location_changed_at: Some(refreshed_at), location_asserts: Some(1), @@ -842,10 +842,11 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { let query_time_original = Utc::now() + Duration::milliseconds(800); tokio::time::sleep(time::Duration::from_millis(800)).await; + let address_recent = make_keypair().public_key().clone(); let loc_recent = 631711281837647358_u64; let gateway_recent = Gateway { - address: address.clone().into(), + address: address_recent.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, inserted_at: created_at, @@ -868,10 +869,11 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Get most recent gateway info let query_time = Utc::now() + Duration::minutes(10); let res = - info_historical_request(&mut client, &address, &admin_key, &query_time).await; + info_historical_request(&mut client, &address_recent, &admin_key, &query_time).await; + // Assert that recent gateway was returned let gw_info = res?.info.unwrap(); - assert_eq!(gw_info.address, address.to_vec()); + assert_eq!(gw_info.address, address_recent.to_vec()); let deployment_info = gw_info.metadata.clone().unwrap().deployment_info.unwrap(); match deployment_info { DeploymentInfo::WifiDeploymentInfo(v) => { @@ -881,8 +883,6 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { } DeploymentInfo::CbrsDeploymentInfo(_) => panic!(), }; - - // Assert that recent gateway was returned assert_eq!( u64::from_str_radix(&gw_info.metadata.clone().unwrap().location, 16).unwrap(), loc_recent @@ -890,21 +890,21 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Get original gateway info by using an earlier inserted_at condition let res = - info_historical_request(&mut client, &address, &admin_key, &query_time_original).await; + info_historical_request(&mut client, &address_original, &admin_key, &query_time_original).await; + // Assert that original gateway was returned let gw_info = res?.info.unwrap(); - assert_eq!(gw_info.address, address.to_vec()); + assert_eq!(gw_info.address, address_original.to_vec()); let deployment_info = gw_info.metadata.clone().unwrap().deployment_info.unwrap(); match deployment_info { DeploymentInfo::WifiDeploymentInfo(v) => { - assert_eq!(v.antenna, 18); - assert_eq!(v.azimuth, 161); - assert_eq!(v.elevation, 2); + assert_eq!(v.antenna, 10); + assert_eq!(v.azimuth, 168); + assert_eq!(v.elevation, 4); } DeploymentInfo::CbrsDeploymentInfo(_) => panic!(), }; - // Assert that original gateway was returned assert_eq!( u64::from_str_radix(&gw_info.metadata.clone().unwrap().location, 16).unwrap(), loc_original From 507a18d8eb9ecfde77d762ef71bb4850ed6d6d17 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Wed, 22 Oct 2025 13:22:43 +0100 Subject: [PATCH 14/35] Fixed formatting errors --- mobile_config/src/gateway/service/mod.rs | 18 +++++++++++------- .../tests/integrations/gateway_service.rs | 14 ++++++++------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index 5f88b95de..07dd0b69f 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -1,5 +1,5 @@ use crate::{ - gateway::service::{info::DeviceType, info_v3::DeviceTypeV2, info::GatewayInfo}, + gateway::service::{info::DeviceType, info::GatewayInfo, info_v3::DeviceTypeV2}, key_cache::KeyCache, telemetry, verify_public_key, GrpcResult, GrpcStreamResult, }; @@ -56,7 +56,7 @@ impl GatewayService { &self, request: &R, signer: &Vec, - address: &[u8] + address: &[u8], ) -> Result<(), Status> where R: MsgVerify, @@ -166,7 +166,10 @@ impl mobile_config::Gateway for GatewayService { ) } - async fn info_historical(&self, request: Request) -> GrpcResult { + async fn info_historical( + &self, + request: Request, + ) -> GrpcResult { let request = request.into_inner(); telemetry::count_request("gateway", "info-v2"); custom_tracing::record_b58("pub_key", &request.address); @@ -175,7 +178,10 @@ impl mobile_config::Gateway for GatewayService { self.verify_request_signature_for_info(&request, &request.signer, &request.address)?; let pubkey: PublicKeyBinary = request.address.into(); - tracing::debug!(pubkey = pubkey.to_string(), "fetching historical gateway info (v2)"); + tracing::debug!( + pubkey = pubkey.to_string(), + "fetching historical gateway info (v2)" + ); let query_time = Utc .timestamp_opt(request.query_time as i64, 0) @@ -184,9 +190,7 @@ impl mobile_config::Gateway for GatewayService { info::get_by_address_and_inserted_at(&self.pool, &pubkey, &query_time) .await - .map_err(|_| { - Status::internal("error fetching historical gateway info") - })? + .map_err(|_| Status::internal("error fetching historical gateway info"))? .map_or_else( || { telemetry::count_gateway_chain_lookup("not-found"); diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index d46898983..be40ffd98 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -862,14 +862,12 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { }; gateway_recent.insert(&pool).await?; - let (addr, _handle) = - spawn_gateway_service(pool.clone(), admin_key.public_key().clone()).await; + let (addr, _handle) = spawn_gateway_service(pool.clone(), admin_key.public_key().clone()).await; let mut client = GatewayClient::connect(addr).await?; // Get most recent gateway info let query_time = Utc::now() + Duration::minutes(10); - let res = - info_historical_request(&mut client, &address_recent, &admin_key, &query_time).await; + let res = info_historical_request(&mut client, &address_recent, &admin_key, &query_time).await; // Assert that recent gateway was returned let gw_info = res?.info.unwrap(); @@ -889,8 +887,12 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { ); // Get original gateway info by using an earlier inserted_at condition - let res = - info_historical_request(&mut client, &address_original, &admin_key, &query_time_original).await; + let res = info_historical_request( + &mut client, + &address_original, + &admin_key, + &query_time_original, + ).await; // Assert that original gateway was returned let gw_info = res?.info.unwrap(); From 8c8f3747de993b0b1d22f9e55d41cd0094fc2235 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Wed, 22 Oct 2025 13:25:22 +0100 Subject: [PATCH 15/35] Fixed formatting errors --- mobile_config/tests/integrations/gateway_service.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index be40ffd98..810902ca3 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -892,7 +892,8 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { &address_original, &admin_key, &query_time_original, - ).await; + ) + .await; // Assert that original gateway was returned let gw_info = res?.info.unwrap(); From 9cdf3aff1ec3a536089a15bd47fd91c6e09236a2 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Wed, 22 Oct 2025 13:45:06 +0100 Subject: [PATCH 16/35] Increased sleep in historical info test --- mobile_config/src/gateway/service/mod.rs | 2 +- mobile_config/tests/integrations/gateway_service.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index 07dd0b69f..a18cf8ccd 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -55,7 +55,7 @@ impl GatewayService { fn verify_request_signature_for_info( &self, request: &R, - signer: &Vec, + signer: &[u8], address: &[u8], ) -> Result<(), Status> where diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index 810902ca3..2e720275a 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -839,8 +839,8 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { }; gateway_original.insert(&pool).await?; - let query_time_original = Utc::now() + Duration::milliseconds(800); - tokio::time::sleep(time::Duration::from_millis(800)).await; + let query_time_original = Utc::now() + Duration::milliseconds(1200); + tokio::time::sleep(time::Duration::from_millis(1500)).await; let address_recent = make_keypair().public_key().clone(); let loc_recent = 631711281837647358_u64; From 7f6003d34dc642c0a393c542b1b2f7e74467ed0a Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 23 Oct 2025 12:01:15 +0100 Subject: [PATCH 17/35] Tests cleanup, removed println, changed assert! to assert_eq --- .../common/gateway_metadata_db.rs | 13 ++--- .../integrations/common/partial_migrator.rs | 31 +++++------- .../tests/integrations/gateway_tracker.rs | 11 ++--- .../tests/integrations/migrations.rs | 47 +++++++++++-------- 4 files changed, 48 insertions(+), 54 deletions(-) diff --git a/mobile_config/tests/integrations/common/gateway_metadata_db.rs b/mobile_config/tests/integrations/common/gateway_metadata_db.rs index 0d07c77a0..40ea78a5e 100644 --- a/mobile_config/tests/integrations/common/gateway_metadata_db.rs +++ b/mobile_config/tests/integrations/common/gateway_metadata_db.rs @@ -21,11 +21,9 @@ pub async fn insert_gateway_bulk( ) -> anyhow::Result<()> { stream::iter(gateways.chunks(chunk_size)) .map(Ok) // convert chunks to a Result for try_for_each_concurrent - .try_for_each_concurrent(Some(20), |chunk| { + .try_for_each_concurrent(20, |chunk| { let pool = pool.clone(); async move { - let mut tx = pool.begin().await?; - // insert mobile_hotspot_infos let mut qb = QueryBuilder::::new( r#" @@ -58,7 +56,7 @@ pub async fn insert_gateway_bulk( .push_bind(num_locations); }); - qb.build().execute(&mut *tx).await?; + qb.build().execute(&pool).await?; // insert key_to_assets let mut qb1 = QueryBuilder::::new( @@ -74,9 +72,7 @@ pub async fn insert_gateway_bulk( b.push_bind(&gw.asset).push_bind(b58); }); - qb1.build().execute(&mut *tx).await?; - - tx.commit().await?; + qb1.build().execute(&pool).await?; Ok::<_, anyhow::Error>(()) } @@ -91,6 +87,7 @@ pub async fn update_gateway( asset: &str, location: i64, refreshed_at: DateTime, + num_location_asserts: i32, ) -> anyhow::Result<()> { sqlx::query( r#" @@ -102,7 +99,7 @@ pub async fn update_gateway( "#, ) .bind(location) - .bind(2) + .bind(num_location_asserts) .bind(refreshed_at) .bind(asset) .execute(pool) diff --git a/mobile_config/tests/integrations/common/partial_migrator.rs b/mobile_config/tests/integrations/common/partial_migrator.rs index 3d9c38a8a..3c3b9bacc 100644 --- a/mobile_config/tests/integrations/common/partial_migrator.rs +++ b/mobile_config/tests/integrations/common/partial_migrator.rs @@ -3,33 +3,28 @@ use sqlx::{ migrate::{Migration, Migrator}, PgPool, }; -use std::path::Path; pub struct PartialMigrator { pool: PgPool, versions: Vec, - path: String, + migrator: Migrator, } impl PartialMigrator { - pub async fn new( - pool: PgPool, - versions: Vec, - path: Option, - ) -> anyhow::Result { - let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM _sqlx_migrations") + pub async fn new(pool: PgPool, versions: Vec) -> anyhow::Result { + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM _sqlx_migrations") .fetch_one(&pool) .await - .unwrap_or((0,)); + .unwrap_or(0); - if count.0 > 0 { + if count > 0 { anyhow::bail!("PartialMigrator: database already has applied migrations. Did you forget `migrations = false`?"); } Ok(Self { pool, versions, - path: path.unwrap_or_else(|| "./migrations".to_string()), + migrator: sqlx::migrate!(), }) } @@ -41,32 +36,28 @@ impl PartialMigrator { tmp_migrator.run(&self.pool).await?; } - let migrator = Migrator::new(Path::new(&self.path)).await?; - // Mark skipped migrations as applied first - for m in migrator.iter() { + for m in self.migrator.iter() { if self.versions.contains(&m.version) { - println!("⏭️ Skipping migration {} {}", m.version, m.description); + tracing::info!("⏭️ Skipping migration {} {}", m.version, m.description); self.skip_migration(m).await?; } } // Now run the migrator normally - migrator.run(&self.pool).await?; + self.migrator.run(&self.pool).await?; Ok(()) } pub async fn run_skipped(&self) -> anyhow::Result<()> { - let migrator = Migrator::new(Path::new(&self.path)).await?; - - println!("Re applaying skipped migrations... {:?}", self.versions); + tracing::info!("Re applaying skipped migrations... {:?}", self.versions); // Delete skipped migrations first self.delete_skipped().await?; // Now run the migrator normally - migrator.run(&self.pool).await?; + self.migrator.run(&self.pool).await?; Ok(()) } diff --git a/mobile_config/tests/integrations/gateway_tracker.rs b/mobile_config/tests/integrations/gateway_tracker.rs index 82057f97f..bd9db2284 100644 --- a/mobile_config/tests/integrations/gateway_tracker.rs +++ b/mobile_config/tests/integrations/gateway_tracker.rs @@ -1,6 +1,5 @@ use crate::common::{gateway_metadata_db, make_keypair}; use chrono::{Timelike, Utc}; -use custom_tracing::Settings; use mobile_config::gateway::{ db::{Gateway, GatewayType}, tracker, @@ -9,12 +8,10 @@ use rand::{seq::SliceRandom, thread_rng}; use sqlx::PgPool; #[sqlx::test] -async fn execute_test(pool: PgPool) -> anyhow::Result<()> { +async fn gateway_tracker_test(pool: PgPool) -> anyhow::Result<()> { // Tested with 100k const TOTAL: usize = 10_000; - custom_tracing::init("mobile_config=debug,info".to_string(), Settings::default()).await?; - let now = Utc::now() .with_nanosecond(Utc::now().timestamp_subsec_micros() * 1000) .unwrap(); @@ -80,7 +77,7 @@ async fn execute_test(pool: PgPool) -> anyhow::Result<()> { assert_eq!(gateway.location_asserts, gw_insert.location.map(|_| 1)); // Update sample gateways - gateway_metadata_db::update_gateway(&pool, &gw_insert.asset, new_loc, now).await?; + gateway_metadata_db::update_gateway(&pool, &gw_insert.asset, new_loc, now, 2).await?; } // now run the tracker again after updates @@ -112,7 +109,7 @@ async fn execute_test(pool: PgPool) -> anyhow::Result<()> { } async fn count_gateways(pool: &PgPool) -> anyhow::Result { - let count: (i64,) = sqlx::query_as( + let count = sqlx::query_scalar( r#" SELECT COUNT(*) FROM gateways; "#, @@ -120,5 +117,5 @@ async fn count_gateways(pool: &PgPool) -> anyhow::Result { .fetch_one(pool) .await?; - Ok(count.0) + Ok(count) } diff --git a/mobile_config/tests/integrations/migrations.rs b/mobile_config/tests/integrations/migrations.rs index b0cfacd78..d09fb7642 100644 --- a/mobile_config/tests/integrations/migrations.rs +++ b/mobile_config/tests/integrations/migrations.rs @@ -6,7 +6,7 @@ use sqlx::PgPool; #[sqlx::test(migrations = false)] async fn gateways_historical(pool: PgPool) -> anyhow::Result<()> { - let partial_migrator = PartialMigrator::new(pool.clone(), vec![20251003000000], None).await?; + let partial_migrator = PartialMigrator::new(pool.clone(), vec![20251003000000]).await?; partial_migrator.run_partial().await?; @@ -38,26 +38,35 @@ async fn gateways_historical(pool: PgPool) -> anyhow::Result<()> { .await? .expect("should find gateway"); - println!("pre_gw: {:?}", pre_gw); - println!("gw: {:?}", gw); - - assert!(pre_gw.address == gw.address); - assert!(pre_gw.gateway_type == gw.gateway_type); - assert!(common::nanos_trunc(pre_gw.created_at) == common::nanos_trunc(gw.created_at)); + assert_eq!(pre_gw.address, gw.address); + assert_eq!(pre_gw.gateway_type, gw.gateway_type); + assert_eq!( + common::nanos_trunc(pre_gw.created_at), + common::nanos_trunc(gw.created_at) + ); // The real change is updated_at renamed to inserted_at AND inserted_at = created_at; - assert!(common::nanos_trunc(pre_gw.created_at) == common::nanos_trunc(gw.inserted_at)); - assert!(common::nanos_trunc(pre_gw.refreshed_at) == common::nanos_trunc(gw.refreshed_at)); - assert!(common::nanos_trunc(pre_gw.last_changed_at) == common::nanos_trunc(gw.last_changed_at)); - assert!(pre_gw.hash == gw.hash); - assert!(pre_gw.antenna == gw.antenna); - assert!(pre_gw.elevation == gw.elevation); - assert!(pre_gw.azimuth == gw.azimuth); - assert!(pre_gw.location == gw.location); - assert!( - common::nanos_trunc(pre_gw.location_changed_at.unwrap()) - == common::nanos_trunc(gw.location_changed_at.unwrap()) + assert_eq!( + common::nanos_trunc(pre_gw.created_at), + common::nanos_trunc(gw.inserted_at) + ); + assert_eq!( + common::nanos_trunc(pre_gw.refreshed_at), + common::nanos_trunc(gw.refreshed_at) + ); + assert_eq!( + common::nanos_trunc(pre_gw.last_changed_at), + common::nanos_trunc(gw.last_changed_at) + ); + assert_eq!(pre_gw.hash, gw.hash); + assert_eq!(pre_gw.antenna, gw.antenna); + assert_eq!(pre_gw.elevation, gw.elevation); + assert_eq!(pre_gw.azimuth, gw.azimuth); + assert_eq!(pre_gw.location, gw.location); + assert_eq!( + common::nanos_trunc(pre_gw.location_changed_at.unwrap()), + common::nanos_trunc(gw.location_changed_at.unwrap()) ); - assert!(pre_gw.location_asserts == gw.location_asserts); + assert_eq!(pre_gw.location_asserts, gw.location_asserts); Ok(()) } From 01376f0a96237e922d8ef3de5a9b6d39ae086257 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 23 Oct 2025 21:46:51 +0100 Subject: [PATCH 18/35] Removed sleep from test --- .../tests/integrations/gateway_service.rs | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index 2e720275a..941bfc038 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -1,7 +1,7 @@ use crate::common::{make_keypair, spawn_gateway_service}; use chrono::{DateTime, Duration, Utc}; use futures::stream::StreamExt; -use helium_crypto::{Keypair, PublicKey, Sign}; +use helium_crypto::{Keypair, PublicKey, PublicKeyBinary, Sign}; use helium_proto::services::mobile_config::{ self as proto, gateway_metadata_v2::DeploymentInfo, DeviceType, GatewayClient, GatewayInfoStreamReqV1, GatewayInfoStreamReqV2, GatewayInfoStreamResV1, GatewayInfoStreamResV2, @@ -816,14 +816,15 @@ async fn gateway_info_v2(pool: PgPool) -> anyhow::Result<()> { async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { let admin_key = make_keypair(); - let address_original = make_keypair().public_key().clone(); + let address = make_keypair().public_key().clone(); let loc_original = 631711281837647359_u64; + let loc_recent = 631711281837647358_u64; let created_at = Utc::now() - Duration::hours(5); let refreshed_at = Utc::now() - Duration::hours(3); let gateway_original = Gateway { - address: address_original.clone().into(), + address: address.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, inserted_at: refreshed_at, @@ -839,14 +840,17 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { }; gateway_original.insert(&pool).await?; - let query_time_original = Utc::now() + Duration::milliseconds(1200); - tokio::time::sleep(time::Duration::from_millis(1500)).await; + let pubkey = address.clone().into(); - let address_recent = make_keypair().public_key().clone(); - let loc_recent = 631711281837647358_u64; + // Change original gateway's inserted_at value to 10 minutes ago + let new_inserted_at = Utc::now() - Duration::minutes(10); + update_gateway_inserted_at(&pool, &pubkey, &new_inserted_at).await?; + + let query_time_original = Utc::now(); + println!("query time original: {:?}", query_time_original); let gateway_recent = Gateway { - address: address_recent.clone().into(), + address: address.clone().into(), gateway_type: GatewayType::WifiIndoor, created_at, inserted_at: created_at, @@ -866,12 +870,13 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { let mut client = GatewayClient::connect(addr).await?; // Get most recent gateway info - let query_time = Utc::now() + Duration::minutes(10); - let res = info_historical_request(&mut client, &address_recent, &admin_key, &query_time).await; + let query_time_recent = Utc::now() + Duration::minutes(10); + println!("query_time_recent: {:?}", query_time_recent); + let res = info_historical_request(&mut client, &address, &admin_key, &query_time_recent).await; // Assert that recent gateway was returned let gw_info = res?.info.unwrap(); - assert_eq!(gw_info.address, address_recent.to_vec()); + assert_eq!(gw_info.address, address.to_vec()); let deployment_info = gw_info.metadata.clone().unwrap().deployment_info.unwrap(); match deployment_info { DeploymentInfo::WifiDeploymentInfo(v) => { @@ -889,7 +894,7 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Get original gateway info by using an earlier inserted_at condition let res = info_historical_request( &mut client, - &address_original, + &address, &admin_key, &query_time_original, ) @@ -897,7 +902,7 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Assert that original gateway was returned let gw_info = res?.info.unwrap(); - assert_eq!(gw_info.address, address_original.to_vec()); + assert_eq!(gw_info.address, address.to_vec()); let deployment_info = gw_info.metadata.clone().unwrap().deployment_info.unwrap(); match deployment_info { DeploymentInfo::WifiDeploymentInfo(v) => { @@ -1043,3 +1048,24 @@ async fn info_batch_v2( Ok(stream) } + + +async fn update_gateway_inserted_at( + pool: &PgPool, + address: &PublicKeyBinary, + new_inserted_at: &DateTime +) -> anyhow::Result<()> { + sqlx::query( + r#" + UPDATE gateways + SET inserted_at = $1 + WHERE address = $2; + "# + ) + .bind(new_inserted_at) + .bind(address.as_ref()) + .fetch_optional(pool) + .await?; + + Ok(()) +} \ No newline at end of file From 5c7eb381d94d227c8b305a2e98beed7d03c354f6 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 23 Oct 2025 22:10:21 +0100 Subject: [PATCH 19/35] Updated gateway tracker test name & reduced test size --- .../tests/integrations/gateway_service.rs | 18 ++++++------------ .../tests/integrations/gateway_tracker.rs | 5 ++--- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index 941bfc038..bc9efbba3 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -16,7 +16,7 @@ use mobile_config::{ }; use prost::Message; use sqlx::PgPool; -use std::{sync::Arc, time, vec}; +use std::{sync::Arc, vec}; use tokio::net::TcpListener; use tonic::{transport, Code}; @@ -892,13 +892,8 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { ); // Get original gateway info by using an earlier inserted_at condition - let res = info_historical_request( - &mut client, - &address, - &admin_key, - &query_time_original, - ) - .await; + let res = + info_historical_request(&mut client, &address, &admin_key, &query_time_original).await; // Assert that original gateway was returned let gw_info = res?.info.unwrap(); @@ -1049,18 +1044,17 @@ async fn info_batch_v2( Ok(stream) } - async fn update_gateway_inserted_at( pool: &PgPool, address: &PublicKeyBinary, - new_inserted_at: &DateTime + new_inserted_at: &DateTime, ) -> anyhow::Result<()> { sqlx::query( r#" UPDATE gateways SET inserted_at = $1 WHERE address = $2; - "# + "#, ) .bind(new_inserted_at) .bind(address.as_ref()) @@ -1068,4 +1062,4 @@ async fn update_gateway_inserted_at( .await?; Ok(()) -} \ No newline at end of file +} diff --git a/mobile_config/tests/integrations/gateway_tracker.rs b/mobile_config/tests/integrations/gateway_tracker.rs index bd9db2284..c356553ec 100644 --- a/mobile_config/tests/integrations/gateway_tracker.rs +++ b/mobile_config/tests/integrations/gateway_tracker.rs @@ -8,9 +8,8 @@ use rand::{seq::SliceRandom, thread_rng}; use sqlx::PgPool; #[sqlx::test] -async fn gateway_tracker_test(pool: PgPool) -> anyhow::Result<()> { - // Tested with 100k - const TOTAL: usize = 10_000; +async fn test_gateway_tracker_updates_changed_gateways(pool: PgPool) -> anyhow::Result<()> { + const TOTAL: usize = 2_000; let now = Utc::now() .with_nanosecond(Utc::now().timestamp_subsec_micros() * 1000) From 7556b567ebb70c231dfa6ba5148498dba6e772b7 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 23 Oct 2025 22:43:12 +0100 Subject: [PATCH 20/35] Removed println in tests --- mobile_config/tests/integrations/common/partial_migrator.rs | 2 +- mobile_config/tests/integrations/gateway_service.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mobile_config/tests/integrations/common/partial_migrator.rs b/mobile_config/tests/integrations/common/partial_migrator.rs index 3c3b9bacc..de90e5788 100644 --- a/mobile_config/tests/integrations/common/partial_migrator.rs +++ b/mobile_config/tests/integrations/common/partial_migrator.rs @@ -51,7 +51,7 @@ impl PartialMigrator { } pub async fn run_skipped(&self) -> anyhow::Result<()> { - tracing::info!("Re applaying skipped migrations... {:?}", self.versions); + tracing::info!("Re applying skipped migrations... {:?}", self.versions); // Delete skipped migrations first self.delete_skipped().await?; diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index bc9efbba3..6b1d51abd 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -847,7 +847,6 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { update_gateway_inserted_at(&pool, &pubkey, &new_inserted_at).await?; let query_time_original = Utc::now(); - println!("query time original: {:?}", query_time_original); let gateway_recent = Gateway { address: address.clone().into(), @@ -871,7 +870,6 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Get most recent gateway info let query_time_recent = Utc::now() + Duration::minutes(10); - println!("query_time_recent: {:?}", query_time_recent); let res = info_historical_request(&mut client, &address, &admin_key, &query_time_recent).await; // Assert that recent gateway was returned @@ -1058,7 +1056,7 @@ async fn update_gateway_inserted_at( ) .bind(new_inserted_at) .bind(address.as_ref()) - .fetch_optional(pool) + .execute(pool) .await?; Ok(()) From f7bec6ab12db99ddb22097b59d3263d80a2d5c55 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Fri, 24 Oct 2025 14:32:57 +0100 Subject: [PATCH 21/35] Add info_historical command to mobile_config_cli --- mobile_config_cli/src/client.rs | 26 ++++++++++++++++++++++++-- mobile_config_cli/src/cmds/gateway.rs | 16 +++++++++++++++- mobile_config_cli/src/cmds/mod.rs | 16 ++++++++++++++++ mobile_config_cli/src/main.rs | 1 + 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/mobile_config_cli/src/client.rs b/mobile_config_cli/src/client.rs index 7a74a66ab..9fa8cc5a1 100644 --- a/mobile_config_cli/src/client.rs +++ b/mobile_config_cli/src/client.rs @@ -12,8 +12,8 @@ use helium_proto::{ AdminAddKeyReqV1, AdminKeyResV1, AdminRemoveKeyReqV1, AuthorizationListReqV1, AuthorizationListResV1, AuthorizationVerifyReqV1, AuthorizationVerifyResV1, CarrierIncentivePromotionListReqV1, CarrierIncentivePromotionListResV1, EntityVerifyReqV1, - EntityVerifyResV1, GatewayInfoBatchReqV1, GatewayInfoReqV1, GatewayInfoResV2, - GatewayInfoStreamResV2, + EntityVerifyResV1, GatewayInfoBatchReqV1, GatewayInfoHistoricalReqV1, GatewayInfoReqV1, + GatewayInfoResV2, GatewayInfoStreamResV2, }, Message, }; @@ -269,6 +269,27 @@ impl GatewayClient { Ok(stream) } + + pub async fn info_historical( + &mut self, + gateway: &PublicKey, + query_time: u64, + keypair: &Keypair, + ) -> Result { + let mut request = GatewayInfoHistoricalReqV1 { + address: gateway.into(), + query_time, + signer: keypair.public_key().into(), + signature: vec![], + }; + request.signature = request.sign(keypair)?; + let response = self.client.info_historical(request).await?.into_inner(); + response.verify(&self.server_pubkey)?; + let info = response + .info + .ok_or_else(|| anyhow::anyhow!("gateway not found"))?; + GatewayInfo::try_from(info) + } } pub trait MsgSign: Message + std::clone::Clone { @@ -296,6 +317,7 @@ impl_sign!(AuthorizationListReqV1, signature); impl_sign!(EntityVerifyReqV1, signature); impl_sign!(GatewayInfoReqV1, signature); impl_sign!(GatewayInfoBatchReqV1, signature); +impl_sign!(GatewayInfoHistoricalReqV1, signature); impl_sign!(CarrierIncentivePromotionListReqV1, signature); pub trait MsgVerify: Message + std::clone::Clone { diff --git a/mobile_config_cli/src/cmds/gateway.rs b/mobile_config_cli/src/cmds/gateway.rs index 3f74e4a7e..2ec05225b 100644 --- a/mobile_config_cli/src/cmds/gateway.rs +++ b/mobile_config_cli/src/cmds/gateway.rs @@ -1,4 +1,4 @@ -use super::{GetHotspot, GetHotspotBatch, PathBufKeypair}; +use super::{GetHotspot, GetHotspotBatch, GetHotspotHistorical, PathBufKeypair}; use crate::{client, Msg, PrettyJson, Result}; use angry_purple_tiger::AnimalName; use futures::StreamExt; @@ -59,6 +59,20 @@ pub async fn info_batch(args: GetHotspotBatch) -> Result { } } +pub async fn info_historical(args: GetHotspotHistorical) -> Result { + let mut client = client::GatewayClient::new(&args.config_host, &args.config_pubkey).await?; + match client + .info_historical(&args.hotspot, args.query_time, &args.keypair.to_keypair()?) + .await + { + Ok(info) => Msg::ok(info.pretty_json()?), + Err(err) => Msg::err(format!( + "failed to retrieve {} info: {}", + &args.hotspot, err + )), + } +} + impl TryFrom for GatewayInfo { type Error = anyhow::Error; diff --git a/mobile_config_cli/src/cmds/mod.rs b/mobile_config_cli/src/cmds/mod.rs index 65baf823c..bb7e948bc 100644 --- a/mobile_config_cli/src/cmds/mod.rs +++ b/mobile_config_cli/src/cmds/mod.rs @@ -167,6 +167,8 @@ pub enum GatewayCommands { /// Retrieve the on-chain registered info for the batch of hotspots /// requested by list of Public Key Binaries InfoBatch(GetHotspotBatch), + /// Retrieve the on-chain registered info for the hotspot at a timestamp + InfoHistorical(GetHotspotHistorical), } #[derive(Debug, Args)] @@ -195,6 +197,20 @@ pub struct GetHotspotBatch { pub config_pubkey: String, } +#[derive(Debug, Args)] +pub struct GetHotspotHistorical { + #[arg(long)] + pub hotspot: PublicKey, + #[arg(long)] + pub query_time: u64, + #[arg(from_global)] + pub keypair: PathBuf, + #[arg(from_global)] + pub config_host: String, + #[arg(from_global)] + pub config_pubkey: String, +} + #[derive(Debug, Subcommand)] pub enum EnvCommands { /// Make Environment variable to ease use diff --git a/mobile_config_cli/src/main.rs b/mobile_config_cli/src/main.rs index a219dfbdb..c0aabd325 100644 --- a/mobile_config_cli/src/main.rs +++ b/mobile_config_cli/src/main.rs @@ -46,6 +46,7 @@ pub async fn handle_cli(cli: Cli) -> Result { Commands::Gateway { command } => match command { cmds::GatewayCommands::Info(args) => gateway::info(args).await, cmds::GatewayCommands::InfoBatch(args) => gateway::info_batch(args).await, + cmds::GatewayCommands::InfoHistorical(args) => gateway::info_historical(args).await, }, } } From 53078d03efba859ba52da0f82d65016a8067d34d Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Fri, 24 Oct 2025 17:12:39 +0100 Subject: [PATCH 22/35] Changed historical info to info_at_timestamp --- file_store/src/traits/msg_verify.rs | 2 +- mobile_config/src/gateway/service/mod.rs | 10 +++++----- mobile_config/tests/integrations/gateway_service.rs | 13 +++++++------ mobile_config_cli/src/client.rs | 10 +++++----- mobile_config_cli/src/cmds/gateway.rs | 6 +++--- mobile_config_cli/src/cmds/mod.rs | 4 ++-- mobile_config_cli/src/main.rs | 2 +- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/file_store/src/traits/msg_verify.rs b/file_store/src/traits/msg_verify.rs index 59937268d..526f1e6a9 100644 --- a/file_store/src/traits/msg_verify.rs +++ b/file_store/src/traits/msg_verify.rs @@ -102,7 +102,7 @@ impl_msg_verify!(mobile_config::GatewayInfoStreamResV3, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexModifiedInfoStreamReqV1, signature); impl_msg_verify!(mobile_config::BoostedHexInfoStreamResV1, signature); -impl_msg_verify!(mobile_config::GatewayInfoHistoricalReqV1, signature); +impl_msg_verify!(mobile_config::GatewayInfoAtTimestampReqV1, signature); impl_msg_verify!(sub_dao::SubDaoEpochRewardInfoReqV1, signature); impl_msg_verify!(sub_dao::SubDaoEpochRewardInfoResV1, signature); impl_msg_verify!(poc_mobile::SubscriberVerifiedMappingEventReqV1, signature); diff --git a/mobile_config/src/gateway/service/mod.rs b/mobile_config/src/gateway/service/mod.rs index a18cf8ccd..8c8173c2a 100644 --- a/mobile_config/src/gateway/service/mod.rs +++ b/mobile_config/src/gateway/service/mod.rs @@ -12,7 +12,7 @@ use futures::{ use helium_crypto::{Keypair, PublicKey, PublicKeyBinary, Sign}; use helium_proto::{ services::mobile_config::{ - self, GatewayInfoBatchReqV1, GatewayInfoHistoricalReqV1, GatewayInfoReqV1, + self, GatewayInfoAtTimestampReqV1, GatewayInfoBatchReqV1, GatewayInfoReqV1, GatewayInfoResV1, GatewayInfoResV2, GatewayInfoStreamReqV1, GatewayInfoStreamReqV2, GatewayInfoStreamReqV3, GatewayInfoStreamResV1, GatewayInfoStreamResV2, GatewayInfoStreamResV3, GatewayInfoV2, @@ -166,9 +166,9 @@ impl mobile_config::Gateway for GatewayService { ) } - async fn info_historical( + async fn info_at_timestamp( &self, - request: Request, + request: Request, ) -> GrpcResult { let request = request.into_inner(); telemetry::count_request("gateway", "info-v2"); @@ -180,7 +180,7 @@ impl mobile_config::Gateway for GatewayService { let pubkey: PublicKeyBinary = request.address.into(); tracing::debug!( pubkey = pubkey.to_string(), - "fetching historical gateway info (v2)" + "fetching gateway info at timestamp" ); let query_time = Utc @@ -190,7 +190,7 @@ impl mobile_config::Gateway for GatewayService { info::get_by_address_and_inserted_at(&self.pool, &pubkey, &query_time) .await - .map_err(|_| Status::internal("error fetching historical gateway info"))? + .map_err(|_| Status::internal("error fetching gateway info at timestamp"))? .map_or_else( || { telemetry::count_gateway_chain_lookup("not-found"); diff --git a/mobile_config/tests/integrations/gateway_service.rs b/mobile_config/tests/integrations/gateway_service.rs index 6b1d51abd..ddfa8399a 100644 --- a/mobile_config/tests/integrations/gateway_service.rs +++ b/mobile_config/tests/integrations/gateway_service.rs @@ -813,7 +813,7 @@ async fn gateway_info_v2(pool: PgPool) -> anyhow::Result<()> { } #[sqlx::test] -async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { +async fn gateway_info_at_timestamp(pool: PgPool) -> anyhow::Result<()> { let admin_key = make_keypair(); let address = make_keypair().public_key().clone(); @@ -870,7 +870,8 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Get most recent gateway info let query_time_recent = Utc::now() + Duration::minutes(10); - let res = info_historical_request(&mut client, &address, &admin_key, &query_time_recent).await; + let res = + info_at_timestamp_request(&mut client, &address, &admin_key, &query_time_recent).await; // Assert that recent gateway was returned let gw_info = res?.info.unwrap(); @@ -891,7 +892,7 @@ async fn gateway_historical_info(pool: PgPool) -> anyhow::Result<()> { // Get original gateway info by using an earlier inserted_at condition let res = - info_historical_request(&mut client, &address, &admin_key, &query_time_original).await; + info_at_timestamp_request(&mut client, &address, &admin_key, &query_time_original).await; // Assert that original gateway was returned let gw_info = res?.info.unwrap(); @@ -939,20 +940,20 @@ async fn info_request_v2( Ok(res) } -async fn info_historical_request( +async fn info_at_timestamp_request( client: &mut GatewayClient, address: &PublicKey, signer: &Keypair, query_time: &DateTime, ) -> anyhow::Result { - let mut req = proto::GatewayInfoHistoricalReqV1 { + let mut req = proto::GatewayInfoAtTimestampReqV1 { address: address.to_vec(), signer: signer.public_key().to_vec(), signature: vec![], query_time: query_time.timestamp() as u64, }; req.signature = signer.sign(&req.encode_to_vec()).unwrap(); - let res = client.info_historical(req).await?.into_inner(); + let res = client.info_at_timestamp(req).await?.into_inner(); Ok(res) } diff --git a/mobile_config_cli/src/client.rs b/mobile_config_cli/src/client.rs index 9fa8cc5a1..b4c093abb 100644 --- a/mobile_config_cli/src/client.rs +++ b/mobile_config_cli/src/client.rs @@ -12,7 +12,7 @@ use helium_proto::{ AdminAddKeyReqV1, AdminKeyResV1, AdminRemoveKeyReqV1, AuthorizationListReqV1, AuthorizationListResV1, AuthorizationVerifyReqV1, AuthorizationVerifyResV1, CarrierIncentivePromotionListReqV1, CarrierIncentivePromotionListResV1, EntityVerifyReqV1, - EntityVerifyResV1, GatewayInfoBatchReqV1, GatewayInfoHistoricalReqV1, GatewayInfoReqV1, + EntityVerifyResV1, GatewayInfoAtTimestampReqV1, GatewayInfoBatchReqV1, GatewayInfoReqV1, GatewayInfoResV2, GatewayInfoStreamResV2, }, Message, @@ -270,20 +270,20 @@ impl GatewayClient { Ok(stream) } - pub async fn info_historical( + pub async fn info_at_timestamp( &mut self, gateway: &PublicKey, query_time: u64, keypair: &Keypair, ) -> Result { - let mut request = GatewayInfoHistoricalReqV1 { + let mut request = GatewayInfoAtTimestampReqV1 { address: gateway.into(), query_time, signer: keypair.public_key().into(), signature: vec![], }; request.signature = request.sign(keypair)?; - let response = self.client.info_historical(request).await?.into_inner(); + let response = self.client.info_at_timestamp(request).await?.into_inner(); response.verify(&self.server_pubkey)?; let info = response .info @@ -317,7 +317,7 @@ impl_sign!(AuthorizationListReqV1, signature); impl_sign!(EntityVerifyReqV1, signature); impl_sign!(GatewayInfoReqV1, signature); impl_sign!(GatewayInfoBatchReqV1, signature); -impl_sign!(GatewayInfoHistoricalReqV1, signature); +impl_sign!(GatewayInfoAtTimestampReqV1, signature); impl_sign!(CarrierIncentivePromotionListReqV1, signature); pub trait MsgVerify: Message + std::clone::Clone { diff --git a/mobile_config_cli/src/cmds/gateway.rs b/mobile_config_cli/src/cmds/gateway.rs index 2ec05225b..8c45a32ce 100644 --- a/mobile_config_cli/src/cmds/gateway.rs +++ b/mobile_config_cli/src/cmds/gateway.rs @@ -1,4 +1,4 @@ -use super::{GetHotspot, GetHotspotBatch, GetHotspotHistorical, PathBufKeypair}; +use super::{GetHotspot, GetHotspotAtTimestamp, GetHotspotBatch, PathBufKeypair}; use crate::{client, Msg, PrettyJson, Result}; use angry_purple_tiger::AnimalName; use futures::StreamExt; @@ -59,10 +59,10 @@ pub async fn info_batch(args: GetHotspotBatch) -> Result { } } -pub async fn info_historical(args: GetHotspotHistorical) -> Result { +pub async fn info_at_timestamp(args: GetHotspotAtTimestamp) -> Result { let mut client = client::GatewayClient::new(&args.config_host, &args.config_pubkey).await?; match client - .info_historical(&args.hotspot, args.query_time, &args.keypair.to_keypair()?) + .info_at_timestamp(&args.hotspot, args.query_time, &args.keypair.to_keypair()?) .await { Ok(info) => Msg::ok(info.pretty_json()?), diff --git a/mobile_config_cli/src/cmds/mod.rs b/mobile_config_cli/src/cmds/mod.rs index bb7e948bc..0c0808a0c 100644 --- a/mobile_config_cli/src/cmds/mod.rs +++ b/mobile_config_cli/src/cmds/mod.rs @@ -168,7 +168,7 @@ pub enum GatewayCommands { /// requested by list of Public Key Binaries InfoBatch(GetHotspotBatch), /// Retrieve the on-chain registered info for the hotspot at a timestamp - InfoHistorical(GetHotspotHistorical), + InfoAtTimestamp(GetHotspotAtTimestamp), } #[derive(Debug, Args)] @@ -198,7 +198,7 @@ pub struct GetHotspotBatch { } #[derive(Debug, Args)] -pub struct GetHotspotHistorical { +pub struct GetHotspotAtTimestamp { #[arg(long)] pub hotspot: PublicKey, #[arg(long)] diff --git a/mobile_config_cli/src/main.rs b/mobile_config_cli/src/main.rs index c0aabd325..3919055c3 100644 --- a/mobile_config_cli/src/main.rs +++ b/mobile_config_cli/src/main.rs @@ -46,7 +46,7 @@ pub async fn handle_cli(cli: Cli) -> Result { Commands::Gateway { command } => match command { cmds::GatewayCommands::Info(args) => gateway::info(args).await, cmds::GatewayCommands::InfoBatch(args) => gateway::info_batch(args).await, - cmds::GatewayCommands::InfoHistorical(args) => gateway::info_historical(args).await, + cmds::GatewayCommands::InfoAtTimestamp(args) => gateway::info_at_timestamp(args).await, }, } } From e83d7b8552e3878c6d1f34fe7cd08c251cc6836c Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Tue, 28 Oct 2025 14:09:07 +0000 Subject: [PATCH 23/35] Update mobile verifiers to use new info_at_timestamp API --- mobile_config/src/gateway/client.rs | 25 +++++++++++-------- mobile_packet_verifier/src/lib.rs | 8 +++++- mobile_verifier/src/heartbeats/mod.rs | 5 +++- mobile_verifier/src/heartbeats/wifi.rs | 2 ++ mobile_verifier/src/lib.rs | 8 +++++- mobile_verifier/src/speedtests.rs | 9 +++++-- .../tests/integrations/boosting_oracles.rs | 2 ++ .../tests/integrations/common/mod.rs | 1 + .../tests/integrations/last_location.rs | 9 +++++++ .../tests/integrations/modeled_coverage.rs | 4 +++ .../tests/integrations/speedtests.rs | 1 + 11 files changed, 59 insertions(+), 15 deletions(-) diff --git a/mobile_config/src/gateway/client.rs b/mobile_config/src/gateway/client.rs index 5c92bedd5..54bfe632b 100644 --- a/mobile_config/src/gateway/client.rs +++ b/mobile_config/src/gateway/client.rs @@ -1,5 +1,6 @@ use crate::client::{call_with_retry, ClientError, Settings}; use crate::gateway::service::info::{GatewayInfo, GatewayInfoStream}; +use chrono::{DateTime, Utc}; use file_store::traits::MsgVerify; use futures::stream::{self, StreamExt}; use helium_crypto::{Keypair, PublicKey, PublicKeyBinary, Sign}; @@ -49,6 +50,7 @@ pub trait GatewayInfoResolver: Clone + Send + Sync + 'static { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result, ClientError>; async fn stream_gateways_info( @@ -62,27 +64,30 @@ impl GatewayInfoResolver for GatewayClient { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result, ClientError> { if let Some(cached_response) = self.cache.get(address).await { return Ok(cached_response.value().clone()); } - let mut request = mobile_config::GatewayInfoReqV1 { + let mut request = mobile_config::GatewayInfoAtTimestampReqV1 { address: address.clone().into(), signer: self.signing_key.public_key().into(), signature: vec![], + query_time: gateway_query_timestamp.clone().timestamp() as u64, }; request.signature = self.signing_key.sign(&request.encode_to_vec())?; tracing::debug!(pubkey = address.to_string(), "fetching gateway info"); - let response = match call_with_retry!(self.client.clone().info_v2(request.clone())) { - Ok(info_res) => { - let response = info_res.into_inner(); - response.verify(&self.config_pubkey)?; - response.info.map(GatewayInfo::try_from).transpose()? - } - Err(status) if status.code() == tonic::Code::NotFound => None, - Err(status) => Err(status)?, - }; + let response = + match call_with_retry!(self.client.clone().info_at_timestamp(request.clone())) { + Ok(info_res) => { + let response = info_res.into_inner(); + response.verify(&self.config_pubkey)?; + response.info.map(GatewayInfo::try_from).transpose()? + } + Err(status) if status.code() == tonic::Code::NotFound => None, + Err(status) => Err(status)?, + }; self.cache .insert(address.clone(), response.clone(), self.cache_ttl) diff --git a/mobile_packet_verifier/src/lib.rs b/mobile_packet_verifier/src/lib.rs index 0a02cd8b9..d96592300 100644 --- a/mobile_packet_verifier/src/lib.rs +++ b/mobile_packet_verifier/src/lib.rs @@ -1,5 +1,6 @@ extern crate tls_init; +use chrono::Utc; use helium_crypto::PublicKeyBinary; use helium_proto::services::mobile_config::NetworkKeyRole; use mobile_config::{ @@ -50,7 +51,12 @@ pub trait MobileConfigResolverExt { #[async_trait::async_trait] impl MobileConfigResolverExt for MobileConfigClients { async fn is_gateway_known(&self, public_key: &PublicKeyBinary) -> bool { - match self.gateway_client.resolve_gateway_info(public_key).await { + let gateway_query_timestamp = Utc::now(); + match self + .gateway_client + .resolve_gateway_info(public_key, &gateway_query_timestamp) + .await + { Ok(res) => res.is_some(), Err(_err) => false, } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 3a3e79c2c..50c32b2f6 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -296,6 +296,7 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &Range>, geofence: &impl GeofenceValidator, + gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let Some(coverage_object) = heartbeat.coverage_object else { return Ok(Self::new( @@ -377,7 +378,7 @@ impl ValidatedHeartbeat { } match gateway_info_resolver - .resolve_gateway(&heartbeat.hotspot_key) + .resolve_gateway(&heartbeat.hotspot_key, gateway_query_timestamp) .await? { GatewayResolution::DataOnly => Ok(Self::new( @@ -488,6 +489,7 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &'a Range>, geofence: &'a impl GeofenceValidator, + gateway_query_timestamp: &'a DateTime, ) -> impl Stream> + 'a { heartbeats.then(move |heartbeat| async move { Self::validate( @@ -498,6 +500,7 @@ impl ValidatedHeartbeat { max_distance_to_coverage, epoch, geofence, + gateway_query_timestamp, ) .await }) diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index 0aa5577be..a867834a4 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -152,6 +152,7 @@ where .into_stream(&mut transaction) .await? .map(Heartbeat::from); + let gateway_query_timestamp = Utc::now(); process_validated_heartbeats( ValidatedHeartbeat::validate_heartbeats( heartbeats, @@ -161,6 +162,7 @@ where self.max_distance_to_coverage, &epoch, &self.geofence, + &gateway_query_timestamp, ), heartbeat_cache, coverage_claim_time_cache, diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index bbdc58c30..2cfb0f1e4 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -22,6 +22,7 @@ pub mod unique_connections; pub use settings::Settings; use async_trait::async_trait; +use chrono::{DateTime, Utc}; use mobile_config::client::ClientError; use rust_decimal::Decimal; use solana::SolPubkey; @@ -44,6 +45,7 @@ pub trait GatewayResolver: Clone + Send + Sync + 'static { async fn resolve_gateway( &self, address: &helium_crypto::PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result; } @@ -52,11 +54,15 @@ impl GatewayResolver for mobile_config::GatewayClient { async fn resolve_gateway( &self, address: &helium_crypto::PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result { use mobile_config::gateway::client::GatewayInfoResolver; use mobile_config::gateway::service::info::{DeviceType, GatewayInfo}; - match self.resolve_gateway_info(address).await? { + match self + .resolve_gateway_info(address, gateway_query_timestamp) + .await? + { None => Ok(GatewayResolution::GatewayNotFound), Some(GatewayInfo { device_type: DeviceType::WifiDataOnly, diff --git a/mobile_verifier/src/speedtests.rs b/mobile_verifier/src/speedtests.rs index f5106595d..25fb0c57f 100644 --- a/mobile_verifier/src/speedtests.rs +++ b/mobile_verifier/src/speedtests.rs @@ -150,8 +150,12 @@ where tracing::info!("Processing speedtest file {}", file.file_info.key); let mut transaction = self.pool.begin().await?; let mut speedtests = file.into_stream(&mut transaction).await?; + let gateway_query_timestamp = Utc::now(); + while let Some(speedtest_report) = speedtests.next().await { - let result = self.validate_speedtest(&speedtest_report).await?; + let result = self + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; if result == SpeedtestResult::SpeedtestValid { save_speedtest(&speedtest_report.report, &mut transaction).await?; let latest_speedtests = get_latest_speedtests_for_pubkey( @@ -176,12 +180,13 @@ where pub async fn validate_speedtest( &self, speedtest: &CellSpeedtestIngestReport, + gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let pubkey = speedtest.report.pubkey.clone(); match self .gateway_info_resolver - .resolve_gateway_info(&pubkey) + .resolve_gateway_info(&pubkey, gateway_query_timestamp) .await? { Some(gw_info) if gw_info.is_data_only() => { diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index b3233bffb..ec31a6ca1 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -478,6 +478,7 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri let location_cache = LocationCache::new(&pool); let epoch = start..end; + let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -486,6 +487,7 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri 2000, &epoch, &MockGeofence, + &gateway_query_timestamp, )); let mut transaction = pool.begin().await?; while let Some(heartbeat) = heartbeats.next().await.transpose()? { diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 292e522a4..4615a6d44 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -216,6 +216,7 @@ impl GatewayResolver for GatewayClientAllOwnersValid { async fn resolve_gateway( &self, _address: &PublicKeyBinary, + _gateway_query_timestamp: &DateTime, ) -> Result { Ok(GatewayResolution::AssertedLocation(0x8c2681a3064d9ff)) } diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index 17912c33c..ab77c0560 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -41,6 +41,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; + let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -52,6 +53,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -70,6 +72,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -103,6 +106,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; + let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -114,6 +118,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -137,6 +142,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -174,6 +180,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( transaction.commit().await?; let location_validation_timestamp = Utc::now(); + let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -187,6 +194,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -207,6 +215,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 9ce9b35ac..9482c3cd2 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -329,6 +329,7 @@ async fn process_wifi_input( .await?; let mut transaction = pool.begin().await?; + let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -337,6 +338,7 @@ async fn process_wifi_input( 2000, epoch, &MockGeofence, + &gateway_query_timestamp, )); while let Some(heartbeat) = heartbeats.next().await.transpose()? { let coverage_claim_time = coverage_claim_time_cache @@ -990,6 +992,7 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow let max_covered_distance = 5_000; let coverage_object_cache = CoverageObjectCache::new(&pool); let location_cache = LocationCache::new(&pool); + let gateway_query_timestamp = Utc::now(); let mk_heartbeat = |latlng: LatLng| WifiHeartbeatIngestReport { report: WifiHeartbeat { @@ -1014,6 +1017,7 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow max_covered_distance, &(DateTime::::MIN_UTC..DateTime::::MAX_UTC), &MockGeofence, + &gateway_query_timestamp, ) }; diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index bb386aec0..768f588a2 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -27,6 +27,7 @@ impl GatewayInfoResolver for MockGatewayInfoResolver { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, + _gateway_query_timestamp: &DateTime, ) -> Result, ClientError> { Ok(Some(GatewayInfo { address: address.clone(), From 924d2d4a9ade0fe96ee415430b242743ea9a0ef2 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Tue, 28 Oct 2025 14:34:44 +0000 Subject: [PATCH 24/35] Revert "Update mobile verifiers to use new info_at_timestamp API" This reverts commit e83d7b8552e3878c6d1f34fe7cd08c251cc6836c. --- mobile_config/src/gateway/client.rs | 25 ++++++++----------- mobile_packet_verifier/src/lib.rs | 8 +----- mobile_verifier/src/heartbeats/mod.rs | 5 +--- mobile_verifier/src/heartbeats/wifi.rs | 2 -- mobile_verifier/src/lib.rs | 8 +----- mobile_verifier/src/speedtests.rs | 9 ++----- .../tests/integrations/boosting_oracles.rs | 2 -- .../tests/integrations/common/mod.rs | 1 - .../tests/integrations/last_location.rs | 9 ------- .../tests/integrations/modeled_coverage.rs | 4 --- .../tests/integrations/speedtests.rs | 1 - 11 files changed, 15 insertions(+), 59 deletions(-) diff --git a/mobile_config/src/gateway/client.rs b/mobile_config/src/gateway/client.rs index 54bfe632b..5c92bedd5 100644 --- a/mobile_config/src/gateway/client.rs +++ b/mobile_config/src/gateway/client.rs @@ -1,6 +1,5 @@ use crate::client::{call_with_retry, ClientError, Settings}; use crate::gateway::service::info::{GatewayInfo, GatewayInfoStream}; -use chrono::{DateTime, Utc}; use file_store::traits::MsgVerify; use futures::stream::{self, StreamExt}; use helium_crypto::{Keypair, PublicKey, PublicKeyBinary, Sign}; @@ -50,7 +49,6 @@ pub trait GatewayInfoResolver: Clone + Send + Sync + 'static { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, - gateway_query_timestamp: &DateTime, ) -> Result, ClientError>; async fn stream_gateways_info( @@ -64,30 +62,27 @@ impl GatewayInfoResolver for GatewayClient { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, - gateway_query_timestamp: &DateTime, ) -> Result, ClientError> { if let Some(cached_response) = self.cache.get(address).await { return Ok(cached_response.value().clone()); } - let mut request = mobile_config::GatewayInfoAtTimestampReqV1 { + let mut request = mobile_config::GatewayInfoReqV1 { address: address.clone().into(), signer: self.signing_key.public_key().into(), signature: vec![], - query_time: gateway_query_timestamp.clone().timestamp() as u64, }; request.signature = self.signing_key.sign(&request.encode_to_vec())?; tracing::debug!(pubkey = address.to_string(), "fetching gateway info"); - let response = - match call_with_retry!(self.client.clone().info_at_timestamp(request.clone())) { - Ok(info_res) => { - let response = info_res.into_inner(); - response.verify(&self.config_pubkey)?; - response.info.map(GatewayInfo::try_from).transpose()? - } - Err(status) if status.code() == tonic::Code::NotFound => None, - Err(status) => Err(status)?, - }; + let response = match call_with_retry!(self.client.clone().info_v2(request.clone())) { + Ok(info_res) => { + let response = info_res.into_inner(); + response.verify(&self.config_pubkey)?; + response.info.map(GatewayInfo::try_from).transpose()? + } + Err(status) if status.code() == tonic::Code::NotFound => None, + Err(status) => Err(status)?, + }; self.cache .insert(address.clone(), response.clone(), self.cache_ttl) diff --git a/mobile_packet_verifier/src/lib.rs b/mobile_packet_verifier/src/lib.rs index d96592300..0a02cd8b9 100644 --- a/mobile_packet_verifier/src/lib.rs +++ b/mobile_packet_verifier/src/lib.rs @@ -1,6 +1,5 @@ extern crate tls_init; -use chrono::Utc; use helium_crypto::PublicKeyBinary; use helium_proto::services::mobile_config::NetworkKeyRole; use mobile_config::{ @@ -51,12 +50,7 @@ pub trait MobileConfigResolverExt { #[async_trait::async_trait] impl MobileConfigResolverExt for MobileConfigClients { async fn is_gateway_known(&self, public_key: &PublicKeyBinary) -> bool { - let gateway_query_timestamp = Utc::now(); - match self - .gateway_client - .resolve_gateway_info(public_key, &gateway_query_timestamp) - .await - { + match self.gateway_client.resolve_gateway_info(public_key).await { Ok(res) => res.is_some(), Err(_err) => false, } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 50c32b2f6..3a3e79c2c 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -296,7 +296,6 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &Range>, geofence: &impl GeofenceValidator, - gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let Some(coverage_object) = heartbeat.coverage_object else { return Ok(Self::new( @@ -378,7 +377,7 @@ impl ValidatedHeartbeat { } match gateway_info_resolver - .resolve_gateway(&heartbeat.hotspot_key, gateway_query_timestamp) + .resolve_gateway(&heartbeat.hotspot_key) .await? { GatewayResolution::DataOnly => Ok(Self::new( @@ -489,7 +488,6 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &'a Range>, geofence: &'a impl GeofenceValidator, - gateway_query_timestamp: &'a DateTime, ) -> impl Stream> + 'a { heartbeats.then(move |heartbeat| async move { Self::validate( @@ -500,7 +498,6 @@ impl ValidatedHeartbeat { max_distance_to_coverage, epoch, geofence, - gateway_query_timestamp, ) .await }) diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index a867834a4..0aa5577be 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -152,7 +152,6 @@ where .into_stream(&mut transaction) .await? .map(Heartbeat::from); - let gateway_query_timestamp = Utc::now(); process_validated_heartbeats( ValidatedHeartbeat::validate_heartbeats( heartbeats, @@ -162,7 +161,6 @@ where self.max_distance_to_coverage, &epoch, &self.geofence, - &gateway_query_timestamp, ), heartbeat_cache, coverage_claim_time_cache, diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index 2cfb0f1e4..bbdc58c30 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -22,7 +22,6 @@ pub mod unique_connections; pub use settings::Settings; use async_trait::async_trait; -use chrono::{DateTime, Utc}; use mobile_config::client::ClientError; use rust_decimal::Decimal; use solana::SolPubkey; @@ -45,7 +44,6 @@ pub trait GatewayResolver: Clone + Send + Sync + 'static { async fn resolve_gateway( &self, address: &helium_crypto::PublicKeyBinary, - gateway_query_timestamp: &DateTime, ) -> Result; } @@ -54,15 +52,11 @@ impl GatewayResolver for mobile_config::GatewayClient { async fn resolve_gateway( &self, address: &helium_crypto::PublicKeyBinary, - gateway_query_timestamp: &DateTime, ) -> Result { use mobile_config::gateway::client::GatewayInfoResolver; use mobile_config::gateway::service::info::{DeviceType, GatewayInfo}; - match self - .resolve_gateway_info(address, gateway_query_timestamp) - .await? - { + match self.resolve_gateway_info(address).await? { None => Ok(GatewayResolution::GatewayNotFound), Some(GatewayInfo { device_type: DeviceType::WifiDataOnly, diff --git a/mobile_verifier/src/speedtests.rs b/mobile_verifier/src/speedtests.rs index 25fb0c57f..f5106595d 100644 --- a/mobile_verifier/src/speedtests.rs +++ b/mobile_verifier/src/speedtests.rs @@ -150,12 +150,8 @@ where tracing::info!("Processing speedtest file {}", file.file_info.key); let mut transaction = self.pool.begin().await?; let mut speedtests = file.into_stream(&mut transaction).await?; - let gateway_query_timestamp = Utc::now(); - while let Some(speedtest_report) = speedtests.next().await { - let result = self - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) - .await?; + let result = self.validate_speedtest(&speedtest_report).await?; if result == SpeedtestResult::SpeedtestValid { save_speedtest(&speedtest_report.report, &mut transaction).await?; let latest_speedtests = get_latest_speedtests_for_pubkey( @@ -180,13 +176,12 @@ where pub async fn validate_speedtest( &self, speedtest: &CellSpeedtestIngestReport, - gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let pubkey = speedtest.report.pubkey.clone(); match self .gateway_info_resolver - .resolve_gateway_info(&pubkey, gateway_query_timestamp) + .resolve_gateway_info(&pubkey) .await? { Some(gw_info) if gw_info.is_data_only() => { diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index ec31a6ca1..b3233bffb 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -478,7 +478,6 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri let location_cache = LocationCache::new(&pool); let epoch = start..end; - let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -487,7 +486,6 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri 2000, &epoch, &MockGeofence, - &gateway_query_timestamp, )); let mut transaction = pool.begin().await?; while let Some(heartbeat) = heartbeats.next().await.transpose()? { diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 4615a6d44..292e522a4 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -216,7 +216,6 @@ impl GatewayResolver for GatewayClientAllOwnersValid { async fn resolve_gateway( &self, _address: &PublicKeyBinary, - _gateway_query_timestamp: &DateTime, ) -> Result { Ok(GatewayResolution::AssertedLocation(0x8c2681a3064d9ff)) } diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index ab77c0560..17912c33c 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -41,7 +41,6 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; - let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -53,7 +52,6 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -72,7 +70,6 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -106,7 +103,6 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; - let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -118,7 +114,6 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -142,7 +137,6 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -180,7 +174,6 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( transaction.commit().await?; let location_validation_timestamp = Utc::now(); - let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -194,7 +187,6 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -215,7 +207,6 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 9482c3cd2..9ce9b35ac 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -329,7 +329,6 @@ async fn process_wifi_input( .await?; let mut transaction = pool.begin().await?; - let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -338,7 +337,6 @@ async fn process_wifi_input( 2000, epoch, &MockGeofence, - &gateway_query_timestamp, )); while let Some(heartbeat) = heartbeats.next().await.transpose()? { let coverage_claim_time = coverage_claim_time_cache @@ -992,7 +990,6 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow let max_covered_distance = 5_000; let coverage_object_cache = CoverageObjectCache::new(&pool); let location_cache = LocationCache::new(&pool); - let gateway_query_timestamp = Utc::now(); let mk_heartbeat = |latlng: LatLng| WifiHeartbeatIngestReport { report: WifiHeartbeat { @@ -1017,7 +1014,6 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow max_covered_distance, &(DateTime::::MIN_UTC..DateTime::::MAX_UTC), &MockGeofence, - &gateway_query_timestamp, ) }; diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index 768f588a2..bb386aec0 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -27,7 +27,6 @@ impl GatewayInfoResolver for MockGatewayInfoResolver { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, - _gateway_query_timestamp: &DateTime, ) -> Result, ClientError> { Ok(Some(GatewayInfo { address: address.clone(), From 4d6cfa5ab1eecfaaefad75adf4bd13001e0368f8 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Tue, 28 Oct 2025 14:09:07 +0000 Subject: [PATCH 25/35] Update mobile verifiers to use new info_at_timestamp API --- mobile_config/src/gateway/client.rs | 25 +++++++++++-------- mobile_packet_verifier/src/lib.rs | 8 +++++- mobile_verifier/src/heartbeats/mod.rs | 5 +++- mobile_verifier/src/heartbeats/wifi.rs | 2 ++ mobile_verifier/src/lib.rs | 8 +++++- mobile_verifier/src/speedtests.rs | 9 +++++-- .../tests/integrations/boosting_oracles.rs | 2 ++ .../tests/integrations/common/mod.rs | 1 + .../tests/integrations/last_location.rs | 9 +++++++ .../tests/integrations/modeled_coverage.rs | 4 +++ .../tests/integrations/speedtests.rs | 1 + 11 files changed, 59 insertions(+), 15 deletions(-) diff --git a/mobile_config/src/gateway/client.rs b/mobile_config/src/gateway/client.rs index 5c92bedd5..54bfe632b 100644 --- a/mobile_config/src/gateway/client.rs +++ b/mobile_config/src/gateway/client.rs @@ -1,5 +1,6 @@ use crate::client::{call_with_retry, ClientError, Settings}; use crate::gateway::service::info::{GatewayInfo, GatewayInfoStream}; +use chrono::{DateTime, Utc}; use file_store::traits::MsgVerify; use futures::stream::{self, StreamExt}; use helium_crypto::{Keypair, PublicKey, PublicKeyBinary, Sign}; @@ -49,6 +50,7 @@ pub trait GatewayInfoResolver: Clone + Send + Sync + 'static { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result, ClientError>; async fn stream_gateways_info( @@ -62,27 +64,30 @@ impl GatewayInfoResolver for GatewayClient { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result, ClientError> { if let Some(cached_response) = self.cache.get(address).await { return Ok(cached_response.value().clone()); } - let mut request = mobile_config::GatewayInfoReqV1 { + let mut request = mobile_config::GatewayInfoAtTimestampReqV1 { address: address.clone().into(), signer: self.signing_key.public_key().into(), signature: vec![], + query_time: gateway_query_timestamp.clone().timestamp() as u64, }; request.signature = self.signing_key.sign(&request.encode_to_vec())?; tracing::debug!(pubkey = address.to_string(), "fetching gateway info"); - let response = match call_with_retry!(self.client.clone().info_v2(request.clone())) { - Ok(info_res) => { - let response = info_res.into_inner(); - response.verify(&self.config_pubkey)?; - response.info.map(GatewayInfo::try_from).transpose()? - } - Err(status) if status.code() == tonic::Code::NotFound => None, - Err(status) => Err(status)?, - }; + let response = + match call_with_retry!(self.client.clone().info_at_timestamp(request.clone())) { + Ok(info_res) => { + let response = info_res.into_inner(); + response.verify(&self.config_pubkey)?; + response.info.map(GatewayInfo::try_from).transpose()? + } + Err(status) if status.code() == tonic::Code::NotFound => None, + Err(status) => Err(status)?, + }; self.cache .insert(address.clone(), response.clone(), self.cache_ttl) diff --git a/mobile_packet_verifier/src/lib.rs b/mobile_packet_verifier/src/lib.rs index 0a02cd8b9..d96592300 100644 --- a/mobile_packet_verifier/src/lib.rs +++ b/mobile_packet_verifier/src/lib.rs @@ -1,5 +1,6 @@ extern crate tls_init; +use chrono::Utc; use helium_crypto::PublicKeyBinary; use helium_proto::services::mobile_config::NetworkKeyRole; use mobile_config::{ @@ -50,7 +51,12 @@ pub trait MobileConfigResolverExt { #[async_trait::async_trait] impl MobileConfigResolverExt for MobileConfigClients { async fn is_gateway_known(&self, public_key: &PublicKeyBinary) -> bool { - match self.gateway_client.resolve_gateway_info(public_key).await { + let gateway_query_timestamp = Utc::now(); + match self + .gateway_client + .resolve_gateway_info(public_key, &gateway_query_timestamp) + .await + { Ok(res) => res.is_some(), Err(_err) => false, } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 3a3e79c2c..50c32b2f6 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -296,6 +296,7 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &Range>, geofence: &impl GeofenceValidator, + gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let Some(coverage_object) = heartbeat.coverage_object else { return Ok(Self::new( @@ -377,7 +378,7 @@ impl ValidatedHeartbeat { } match gateway_info_resolver - .resolve_gateway(&heartbeat.hotspot_key) + .resolve_gateway(&heartbeat.hotspot_key, gateway_query_timestamp) .await? { GatewayResolution::DataOnly => Ok(Self::new( @@ -488,6 +489,7 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &'a Range>, geofence: &'a impl GeofenceValidator, + gateway_query_timestamp: &'a DateTime, ) -> impl Stream> + 'a { heartbeats.then(move |heartbeat| async move { Self::validate( @@ -498,6 +500,7 @@ impl ValidatedHeartbeat { max_distance_to_coverage, epoch, geofence, + gateway_query_timestamp, ) .await }) diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index 0aa5577be..a867834a4 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -152,6 +152,7 @@ where .into_stream(&mut transaction) .await? .map(Heartbeat::from); + let gateway_query_timestamp = Utc::now(); process_validated_heartbeats( ValidatedHeartbeat::validate_heartbeats( heartbeats, @@ -161,6 +162,7 @@ where self.max_distance_to_coverage, &epoch, &self.geofence, + &gateway_query_timestamp, ), heartbeat_cache, coverage_claim_time_cache, diff --git a/mobile_verifier/src/lib.rs b/mobile_verifier/src/lib.rs index bbdc58c30..2cfb0f1e4 100644 --- a/mobile_verifier/src/lib.rs +++ b/mobile_verifier/src/lib.rs @@ -22,6 +22,7 @@ pub mod unique_connections; pub use settings::Settings; use async_trait::async_trait; +use chrono::{DateTime, Utc}; use mobile_config::client::ClientError; use rust_decimal::Decimal; use solana::SolPubkey; @@ -44,6 +45,7 @@ pub trait GatewayResolver: Clone + Send + Sync + 'static { async fn resolve_gateway( &self, address: &helium_crypto::PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result; } @@ -52,11 +54,15 @@ impl GatewayResolver for mobile_config::GatewayClient { async fn resolve_gateway( &self, address: &helium_crypto::PublicKeyBinary, + gateway_query_timestamp: &DateTime, ) -> Result { use mobile_config::gateway::client::GatewayInfoResolver; use mobile_config::gateway::service::info::{DeviceType, GatewayInfo}; - match self.resolve_gateway_info(address).await? { + match self + .resolve_gateway_info(address, gateway_query_timestamp) + .await? + { None => Ok(GatewayResolution::GatewayNotFound), Some(GatewayInfo { device_type: DeviceType::WifiDataOnly, diff --git a/mobile_verifier/src/speedtests.rs b/mobile_verifier/src/speedtests.rs index f5106595d..25fb0c57f 100644 --- a/mobile_verifier/src/speedtests.rs +++ b/mobile_verifier/src/speedtests.rs @@ -150,8 +150,12 @@ where tracing::info!("Processing speedtest file {}", file.file_info.key); let mut transaction = self.pool.begin().await?; let mut speedtests = file.into_stream(&mut transaction).await?; + let gateway_query_timestamp = Utc::now(); + while let Some(speedtest_report) = speedtests.next().await { - let result = self.validate_speedtest(&speedtest_report).await?; + let result = self + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; if result == SpeedtestResult::SpeedtestValid { save_speedtest(&speedtest_report.report, &mut transaction).await?; let latest_speedtests = get_latest_speedtests_for_pubkey( @@ -176,12 +180,13 @@ where pub async fn validate_speedtest( &self, speedtest: &CellSpeedtestIngestReport, + gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let pubkey = speedtest.report.pubkey.clone(); match self .gateway_info_resolver - .resolve_gateway_info(&pubkey) + .resolve_gateway_info(&pubkey, gateway_query_timestamp) .await? { Some(gw_info) if gw_info.is_data_only() => { diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index b3233bffb..ec31a6ca1 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -478,6 +478,7 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri let location_cache = LocationCache::new(&pool); let epoch = start..end; + let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -486,6 +487,7 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri 2000, &epoch, &MockGeofence, + &gateway_query_timestamp, )); let mut transaction = pool.begin().await?; while let Some(heartbeat) = heartbeats.next().await.transpose()? { diff --git a/mobile_verifier/tests/integrations/common/mod.rs b/mobile_verifier/tests/integrations/common/mod.rs index 292e522a4..4615a6d44 100644 --- a/mobile_verifier/tests/integrations/common/mod.rs +++ b/mobile_verifier/tests/integrations/common/mod.rs @@ -216,6 +216,7 @@ impl GatewayResolver for GatewayClientAllOwnersValid { async fn resolve_gateway( &self, _address: &PublicKeyBinary, + _gateway_query_timestamp: &DateTime, ) -> Result { Ok(GatewayResolution::AssertedLocation(0x8c2681a3064d9ff)) } diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index 17912c33c..ab77c0560 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -41,6 +41,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; + let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -52,6 +53,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -70,6 +72,7 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -103,6 +106,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; + let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -114,6 +118,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -137,6 +142,7 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -174,6 +180,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( transaction.commit().await?; let location_validation_timestamp = Utc::now(); + let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -187,6 +194,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; @@ -207,6 +215,7 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, + &gateway_query_timestamp, ) .await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index 9ce9b35ac..9482c3cd2 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -329,6 +329,7 @@ async fn process_wifi_input( .await?; let mut transaction = pool.begin().await?; + let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -337,6 +338,7 @@ async fn process_wifi_input( 2000, epoch, &MockGeofence, + &gateway_query_timestamp, )); while let Some(heartbeat) = heartbeats.next().await.transpose()? { let coverage_claim_time = coverage_claim_time_cache @@ -990,6 +992,7 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow let max_covered_distance = 5_000; let coverage_object_cache = CoverageObjectCache::new(&pool); let location_cache = LocationCache::new(&pool); + let gateway_query_timestamp = Utc::now(); let mk_heartbeat = |latlng: LatLng| WifiHeartbeatIngestReport { report: WifiHeartbeat { @@ -1014,6 +1017,7 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow max_covered_distance, &(DateTime::::MIN_UTC..DateTime::::MAX_UTC), &MockGeofence, + &gateway_query_timestamp, ) }; diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index bb386aec0..768f588a2 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -27,6 +27,7 @@ impl GatewayInfoResolver for MockGatewayInfoResolver { async fn resolve_gateway_info( &self, address: &PublicKeyBinary, + _gateway_query_timestamp: &DateTime, ) -> Result, ClientError> { Ok(Some(GatewayInfo { address: address.clone(), From 7b4d4444f231d9b9e6488bcdaac7bf334b192fc5 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 13 Nov 2025 15:35:45 +0000 Subject: [PATCH 26/35] Update packet_verifier to use BucketClient --- mobile_packet_verifier/src/banning/mod.rs | 4 ++-- mobile_packet_verifier/src/daemon.rs | 9 +++++---- mobile_packet_verifier/src/settings.rs | 13 ++++++++++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/mobile_packet_verifier/src/banning/mod.rs b/mobile_packet_verifier/src/banning/mod.rs index b97aebe03..8a48800d1 100644 --- a/mobile_packet_verifier/src/banning/mod.rs +++ b/mobile_packet_verifier/src/banning/mod.rs @@ -40,12 +40,12 @@ fn default_ingest_start_after() -> DateTime { pub async fn create_managed_task( pool: PgPool, - client: file_store::Client, + client: file_store::BucketClient, settings: &BanSettings, ) -> anyhow::Result { let (ban_report_rx, ban_report_server) = file_source::continuous_source() .state(pool.clone()) - .file_store(client, settings.input_bucket.clone()) + .bucket_client(client) .lookback_start_after(settings.start_after) .prefix(FileType::VerifiedMobileBanReport.to_string()) .create() diff --git a/mobile_packet_verifier/src/daemon.rs b/mobile_packet_verifier/src/daemon.rs index 515a78c95..ff7edd1f5 100644 --- a/mobile_packet_verifier/src/daemon.rs +++ b/mobile_packet_verifier/src/daemon.rs @@ -156,9 +156,8 @@ impl Cmd { None }; - let file_store_client = settings.file_store.connect().await; let (file_upload, file_upload_server) = - file_upload::FileUpload::new(file_store_client.clone(), settings.output_bucket.clone()) + file_upload::FileUpload::from_bucket_client(settings.buckets.output.connect().await) .await; let (valid_sessions, valid_sessions_server) = ValidDataTransferSession::file_sink( @@ -187,9 +186,11 @@ impl Cmd { settings.txn_confirmation_check_interval, ); + let ingest_bucket_client = settings.buckets.ingest.connect().await; + let (reports, reports_server) = file_source::continuous_source() .state(pool.clone()) - .file_store(file_store_client.clone(), settings.ingest_bucket.clone()) + .bucket_client(ingest_bucket_client.clone()) .prefix(FileType::DataTransferSessionIngestReport.to_string()) .lookback_start_after(settings.start_after) .create() @@ -208,7 +209,7 @@ impl Cmd { let event_id_purger = EventIdPurger::from_settings(pool.clone(), settings); let banning = - banning::create_managed_task(pool, file_store_client, &settings.banning).await?; + banning::create_managed_task(pool, ingest_bucket_client, &settings.banning).await?; TaskManager::builder() .add_task(file_upload_server) diff --git a/mobile_packet_verifier/src/settings.rs b/mobile_packet_verifier/src/settings.rs index 3d4336707..c87069d98 100644 --- a/mobile_packet_verifier/src/settings.rs +++ b/mobile_packet_verifier/src/settings.rs @@ -9,14 +9,23 @@ use std::{ use crate::banning; +#[derive(Debug, Deserialize, Serialize)] +pub struct Buckets { + pub ingest: file_store::BucketSettings, + pub output: file_store::BucketSettings, +} + #[derive(Debug, Deserialize, Serialize)] pub struct Settings { - /// RUST_LOG compatible settings string. Defsault to + /// RUST_LOG compatible settings string. Default to /// "mobile_verifier=debug,poc_store=info" #[serde(default = "default_log")] pub log: String, #[serde(default)] pub custom_tracing: custom_tracing::Settings, + #[serde(default)] + pub file_store: file_store::Settings, + pub buckets: Buckets, /// Cache location for generated verified reports #[serde(default = "default_cache")] pub cache: PathBuf, @@ -27,8 +36,6 @@ pub struct Settings { #[serde(with = "humantime_serde", default = "default_min_burn_period")] pub min_burn_period: Duration, pub database: db_store::Settings, - #[serde(default)] - pub file_store: file_store::Settings, pub ingest_bucket: String, pub output_bucket: String, #[serde(default)] From 321ab637af0194e3c2661efe94b9d704f2a07f0f Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 13 Nov 2025 15:46:39 +0000 Subject: [PATCH 27/35] Add banning bucket settings --- mobile_packet_verifier/src/daemon.rs | 12 +++++++----- mobile_packet_verifier/src/settings.rs | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mobile_packet_verifier/src/daemon.rs b/mobile_packet_verifier/src/daemon.rs index ff7edd1f5..70e148045 100644 --- a/mobile_packet_verifier/src/daemon.rs +++ b/mobile_packet_verifier/src/daemon.rs @@ -186,11 +186,9 @@ impl Cmd { settings.txn_confirmation_check_interval, ); - let ingest_bucket_client = settings.buckets.ingest.connect().await; - let (reports, reports_server) = file_source::continuous_source() .state(pool.clone()) - .bucket_client(ingest_bucket_client.clone()) + .bucket_client(settings.buckets.ingest.connect().await) .prefix(FileType::DataTransferSessionIngestReport.to_string()) .lookback_start_after(settings.start_after) .create() @@ -208,8 +206,12 @@ impl Cmd { ); let event_id_purger = EventIdPurger::from_settings(pool.clone(), settings); - let banning = - banning::create_managed_task(pool, ingest_bucket_client, &settings.banning).await?; + let banning = banning::create_managed_task( + pool, + settings.buckets.banning.connect().await, + &settings.banning, + ) + .await?; TaskManager::builder() .add_task(file_upload_server) diff --git a/mobile_packet_verifier/src/settings.rs b/mobile_packet_verifier/src/settings.rs index c87069d98..1470a719c 100644 --- a/mobile_packet_verifier/src/settings.rs +++ b/mobile_packet_verifier/src/settings.rs @@ -13,6 +13,7 @@ use crate::banning; pub struct Buckets { pub ingest: file_store::BucketSettings, pub output: file_store::BucketSettings, + pub banning: file_store::BucketSettings, } #[derive(Debug, Deserialize, Serialize)] From 50124320759c3b0fd691c4e19d1fcc5f044f267e Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 13 Nov 2025 16:50:02 +0000 Subject: [PATCH 28/35] Add query_timestamp to new ITs --- .../tests/integrations/speedtests.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index 34de9564d..6bc880040 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -119,7 +119,9 @@ async fn speedtest_upload_exceeds_300megabits_ps_limit(pool: Pool) -> }, }; - let result = daemon.validate_speedtest(&speedtest_report).await?; + let gateway_query_timestamp = Utc::now(); + + let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -158,7 +160,9 @@ async fn speedtest_download_exceeds_300_megabits_ps_limit( }, }; - let result = daemon.validate_speedtest(&speedtest_report).await?; + let gateway_query_timestamp = Utc::now(); + + let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -197,7 +201,9 @@ async fn speedtest_both_speeds_exceed_300_megabits_ps_limit( }, }; - let result = daemon.validate_speedtest(&speedtest_report).await?; + let gateway_query_timestamp = Utc::now(); + + let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -236,7 +242,9 @@ async fn speedtest_within_300_megabits_ps_limit_should_be_valid( }, }; - let result = daemon.validate_speedtest(&speedtest_report).await?; + let gateway_query_timestamp = Utc::now(); + + let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); Ok(()) @@ -274,8 +282,9 @@ async fn speedtest_exactly_300_megabits_ps_limit_should_be_valid( latency: 10, }, }; + let gateway_query_timestamp = Utc::now(); - let result = daemon.validate_speedtest(&speedtest_report).await?; + let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); Ok(()) From 473f02eae3da19b2e128654bbd85fb5c3e8727f7 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 13 Nov 2025 16:52:02 +0000 Subject: [PATCH 29/35] Fix fmt --- .../tests/integrations/speedtests.rs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index 6bc880040..e05c54ebe 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -121,7 +121,9 @@ async fn speedtest_upload_exceeds_300megabits_ps_limit(pool: Pool) -> let gateway_query_timestamp = Utc::now(); - let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; + let result = daemon + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -162,7 +164,9 @@ async fn speedtest_download_exceeds_300_megabits_ps_limit( let gateway_query_timestamp = Utc::now(); - let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; + let result = daemon + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -203,7 +207,9 @@ async fn speedtest_both_speeds_exceed_300_megabits_ps_limit( let gateway_query_timestamp = Utc::now(); - let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; + let result = daemon + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -244,7 +250,9 @@ async fn speedtest_within_300_megabits_ps_limit_should_be_valid( let gateway_query_timestamp = Utc::now(); - let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; + let result = daemon + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); Ok(()) @@ -284,7 +292,9 @@ async fn speedtest_exactly_300_megabits_ps_limit_should_be_valid( }; let gateway_query_timestamp = Utc::now(); - let result = daemon.validate_speedtest(&speedtest_report, &gateway_query_timestamp).await?; + let result = daemon + .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); Ok(()) From c0f42573cc06f0976eb13eff3104624c28b4dc07 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Thu, 13 Nov 2025 20:03:26 +0000 Subject: [PATCH 30/35] Removed old settings & moved ban settings --- mobile_packet_verifier/src/banning/mod.rs | 5 ++--- mobile_packet_verifier/src/daemon.rs | 1 - mobile_packet_verifier/src/settings.rs | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/mobile_packet_verifier/src/banning/mod.rs b/mobile_packet_verifier/src/banning/mod.rs index 8a48800d1..cae17fb19 100644 --- a/mobile_packet_verifier/src/banning/mod.rs +++ b/mobile_packet_verifier/src/banning/mod.rs @@ -21,7 +21,7 @@ pub const BAN_CLEANUP_DAYS: i64 = 7; #[derive(Debug, Deserialize, Serialize)] pub struct BanSettings { /// Where do we look in s3 for ban files - pub input_bucket: String, + pub input_bucket: file_store::BucketSettings, /// How often to purge expired bans #[serde(with = "humantime_serde", default = "default_purge_interval")] pub purge_interval: Duration, @@ -40,12 +40,11 @@ fn default_ingest_start_after() -> DateTime { pub async fn create_managed_task( pool: PgPool, - client: file_store::BucketClient, settings: &BanSettings, ) -> anyhow::Result { let (ban_report_rx, ban_report_server) = file_source::continuous_source() .state(pool.clone()) - .bucket_client(client) + .bucket_client(settings.input_bucket.connect().await) .lookback_start_after(settings.start_after) .prefix(FileType::VerifiedMobileBanReport.to_string()) .create() diff --git a/mobile_packet_verifier/src/daemon.rs b/mobile_packet_verifier/src/daemon.rs index 70e148045..54325e5ae 100644 --- a/mobile_packet_verifier/src/daemon.rs +++ b/mobile_packet_verifier/src/daemon.rs @@ -208,7 +208,6 @@ impl Cmd { let event_id_purger = EventIdPurger::from_settings(pool.clone(), settings); let banning = banning::create_managed_task( pool, - settings.buckets.banning.connect().await, &settings.banning, ) .await?; diff --git a/mobile_packet_verifier/src/settings.rs b/mobile_packet_verifier/src/settings.rs index 1470a719c..e365314b4 100644 --- a/mobile_packet_verifier/src/settings.rs +++ b/mobile_packet_verifier/src/settings.rs @@ -13,7 +13,6 @@ use crate::banning; pub struct Buckets { pub ingest: file_store::BucketSettings, pub output: file_store::BucketSettings, - pub banning: file_store::BucketSettings, } #[derive(Debug, Deserialize, Serialize)] @@ -37,8 +36,6 @@ pub struct Settings { #[serde(with = "humantime_serde", default = "default_min_burn_period")] pub min_burn_period: Duration, pub database: db_store::Settings, - pub ingest_bucket: String, - pub output_bucket: String, #[serde(default)] pub metrics: poc_metrics::Settings, #[serde(default)] From e22ac9095b244b99b20ddd77e73783eb81147fb5 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Fri, 14 Nov 2025 11:00:51 +0000 Subject: [PATCH 31/35] Fix fmt --- mobile_packet_verifier/src/daemon.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mobile_packet_verifier/src/daemon.rs b/mobile_packet_verifier/src/daemon.rs index 54325e5ae..b268401de 100644 --- a/mobile_packet_verifier/src/daemon.rs +++ b/mobile_packet_verifier/src/daemon.rs @@ -206,11 +206,7 @@ impl Cmd { ); let event_id_purger = EventIdPurger::from_settings(pool.clone(), settings); - let banning = banning::create_managed_task( - pool, - &settings.banning, - ) - .await?; + let banning = banning::create_managed_task(pool, &settings.banning).await?; TaskManager::builder() .add_task(file_upload_server) From a52ad495f5d2d5da1e9b4c8d7adeda82359eed3f Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Mon, 17 Nov 2025 13:14:14 +0000 Subject: [PATCH 32/35] Moved bucket settings to individual fields --- mobile_packet_verifier/src/daemon.rs | 4 ++-- mobile_packet_verifier/src/settings.rs | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/mobile_packet_verifier/src/daemon.rs b/mobile_packet_verifier/src/daemon.rs index b268401de..47e05cb96 100644 --- a/mobile_packet_verifier/src/daemon.rs +++ b/mobile_packet_verifier/src/daemon.rs @@ -157,7 +157,7 @@ impl Cmd { }; let (file_upload, file_upload_server) = - file_upload::FileUpload::from_bucket_client(settings.buckets.output.connect().await) + file_upload::FileUpload::from_bucket_client(settings.output_bucket.connect().await) .await; let (valid_sessions, valid_sessions_server) = ValidDataTransferSession::file_sink( @@ -188,7 +188,7 @@ impl Cmd { let (reports, reports_server) = file_source::continuous_source() .state(pool.clone()) - .bucket_client(settings.buckets.ingest.connect().await) + .bucket_client(settings.ingest_bucket.connect().await) .prefix(FileType::DataTransferSessionIngestReport.to_string()) .lookback_start_after(settings.start_after) .create() diff --git a/mobile_packet_verifier/src/settings.rs b/mobile_packet_verifier/src/settings.rs index e365314b4..1c67c67d0 100644 --- a/mobile_packet_verifier/src/settings.rs +++ b/mobile_packet_verifier/src/settings.rs @@ -9,12 +9,6 @@ use std::{ use crate::banning; -#[derive(Debug, Deserialize, Serialize)] -pub struct Buckets { - pub ingest: file_store::BucketSettings, - pub output: file_store::BucketSettings, -} - #[derive(Debug, Deserialize, Serialize)] pub struct Settings { /// RUST_LOG compatible settings string. Default to @@ -25,7 +19,6 @@ pub struct Settings { pub custom_tracing: custom_tracing::Settings, #[serde(default)] pub file_store: file_store::Settings, - pub buckets: Buckets, /// Cache location for generated verified reports #[serde(default = "default_cache")] pub cache: PathBuf, @@ -38,6 +31,8 @@ pub struct Settings { pub database: db_store::Settings, #[serde(default)] pub metrics: poc_metrics::Settings, + pub ingest_bucket: file_store::BucketSettings, + pub output_bucket: file_store::BucketSettings, #[serde(default)] pub enable_solana_integration: bool, pub solana: Option, From 2b43ff77f51687ce9b12ed9a0674e4a931322778 Mon Sep 17 00:00:00 2001 From: Brian Balser Date: Mon, 17 Nov 2025 09:56:41 -0500 Subject: [PATCH 33/35] Remove unused file_store from settings --- mobile_packet_verifier/src/settings.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mobile_packet_verifier/src/settings.rs b/mobile_packet_verifier/src/settings.rs index 1c67c67d0..72b54f42c 100644 --- a/mobile_packet_verifier/src/settings.rs +++ b/mobile_packet_verifier/src/settings.rs @@ -17,8 +17,6 @@ pub struct Settings { pub log: String, #[serde(default)] pub custom_tracing: custom_tracing::Settings, - #[serde(default)] - pub file_store: file_store::Settings, /// Cache location for generated verified reports #[serde(default = "default_cache")] pub cache: PathBuf, From ac1d846b60d7f143b862610ea712aa3f7f03c7a4 Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Mon, 17 Nov 2025 15:34:48 +0000 Subject: [PATCH 34/35] Update query time to receieved_timestamp --- mobile_packet_verifier/.idea/workspace.xml | 420 ++++++++++++++++++ mobile_packet_verifier/src/accumulate.rs | 5 +- mobile_packet_verifier/src/lib.rs | 17 +- .../tests/integrations/common/mod.rs | 7 +- mobile_verifier/src/heartbeats/mod.rs | 5 +- mobile_verifier/src/heartbeats/wifi.rs | 2 - mobile_verifier/src/speedtests.rs | 6 +- .../tests/integrations/boosting_oracles.rs | 2 - .../tests/integrations/last_location.rs | 9 - .../tests/integrations/modeled_coverage.rs | 4 - .../tests/integrations/speedtests.rs | 19 +- 11 files changed, 450 insertions(+), 46 deletions(-) create mode 100644 mobile_packet_verifier/.idea/workspace.xml diff --git a/mobile_packet_verifier/.idea/workspace.xml b/mobile_packet_verifier/.idea/workspace.xml new file mode 100644 index 000000000..0f20b8889 --- /dev/null +++ b/mobile_packet_verifier/.idea/workspace.xml @@ -0,0 +1,420 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { + "associatedIndex": 2 +} + + + + { + "keyToString": { + "ModuleVcsDetector.initialDetectionPerformed": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true", + "RunOnceActivity.git.unshallow": "true", + "RunOnceActivity.rust.reset.selective.auto.import": "true", + "com.intellij.ml.llm.matterhorn.ej.ui.settings.DefaultModelSelectionForGA.v1": "true", + "git-widget-placeholder": "connor/mobile-verifier-historical", + "junie.onboarding.icon.badge.shown": "true", + "last_opened_file_path": "/Users/connormckenzie/Helium/oracles/mobile_packet_verifier", + "node.js.detected.package.eslint": "true", + "node.js.detected.package.tslint": "true", + "node.js.selected.package.eslint": "(autodetect)", + "node.js.selected.package.tslint": "(autodetect)", + "nodejs_package_manager_path": "npm", + "org.rust.cargo.project.model.PROJECT_DISCOVERY": "true", + "org.rust.cargo.project.model.impl.CargoExternalSystemProjectAware.subscribe.first.balloon": "", + "org.rust.first.attach.projects": "true", + "to.speed.mode.migration.done": "true", + "vue.rearranger.settings.migration": "true" + } +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1761651090148 + + + + + + + + + + \ No newline at end of file diff --git a/mobile_packet_verifier/src/accumulate.rs b/mobile_packet_verifier/src/accumulate.rs index 82adffeb1..12b66d844 100644 --- a/mobile_packet_verifier/src/accumulate.rs +++ b/mobile_packet_verifier/src/accumulate.rs @@ -79,7 +79,10 @@ async fn verify_report( return Ok(ReportStatus::Banned); } - if !mobile_config.is_gateway_known(gw_pub_key).await { + if !mobile_config + .is_gateway_known(gw_pub_key, &report.received_timestamp) + .await + { return Ok(ReportStatus::InvalidGatewayKey); } diff --git a/mobile_packet_verifier/src/lib.rs b/mobile_packet_verifier/src/lib.rs index d96592300..140159a2c 100644 --- a/mobile_packet_verifier/src/lib.rs +++ b/mobile_packet_verifier/src/lib.rs @@ -1,6 +1,6 @@ extern crate tls_init; -use chrono::Utc; +use chrono::{DateTime, Utc}; use helium_crypto::PublicKeyBinary; use helium_proto::services::mobile_config::NetworkKeyRole; use mobile_config::{ @@ -44,17 +44,24 @@ impl MobileConfigClients { #[async_trait::async_trait] pub trait MobileConfigResolverExt { - async fn is_gateway_known(&self, public_key: &PublicKeyBinary) -> bool; + async fn is_gateway_known( + &self, + public_key: &PublicKeyBinary, + gateway_query_time: &DateTime, + ) -> bool; async fn is_routing_key_known(&self, public_key: &PublicKeyBinary) -> bool; } #[async_trait::async_trait] impl MobileConfigResolverExt for MobileConfigClients { - async fn is_gateway_known(&self, public_key: &PublicKeyBinary) -> bool { - let gateway_query_timestamp = Utc::now(); + async fn is_gateway_known( + &self, + public_key: &PublicKeyBinary, + gateway_query_time: &DateTime, + ) -> bool { match self .gateway_client - .resolve_gateway_info(public_key, &gateway_query_timestamp) + .resolve_gateway_info(public_key, &gateway_query_time) .await { Ok(res) => res.is_some(), diff --git a/mobile_packet_verifier/tests/integrations/common/mod.rs b/mobile_packet_verifier/tests/integrations/common/mod.rs index 0f6a48946..6bf7d2288 100644 --- a/mobile_packet_verifier/tests/integrations/common/mod.rs +++ b/mobile_packet_verifier/tests/integrations/common/mod.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use helium_crypto::PublicKeyBinary; use mobile_packet_verifier::MobileConfigResolverExt; @@ -22,7 +23,11 @@ pub struct TestMobileConfig { #[async_trait::async_trait] impl MobileConfigResolverExt for TestMobileConfig { - async fn is_gateway_known(&self, public_key: &PublicKeyBinary) -> bool { + async fn is_gateway_known( + &self, + public_key: &PublicKeyBinary, + _gateway_query_time: &DateTime, + ) -> bool { self.valid_gateways.is_valid(public_key) } diff --git a/mobile_verifier/src/heartbeats/mod.rs b/mobile_verifier/src/heartbeats/mod.rs index 359d6ead5..4e0eccda7 100644 --- a/mobile_verifier/src/heartbeats/mod.rs +++ b/mobile_verifier/src/heartbeats/mod.rs @@ -297,7 +297,6 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &Range>, geofence: &impl GeofenceValidator, - gateway_query_timestamp: &DateTime, ) -> anyhow::Result { let Some(coverage_object) = heartbeat.coverage_object else { return Ok(Self::new( @@ -379,7 +378,7 @@ impl ValidatedHeartbeat { } match gateway_info_resolver - .resolve_gateway(&heartbeat.hotspot_key, gateway_query_timestamp) + .resolve_gateway(&heartbeat.hotspot_key, &heartbeat.timestamp) .await? { GatewayResolution::DataOnly => Ok(Self::new( @@ -490,7 +489,6 @@ impl ValidatedHeartbeat { max_distance_to_coverage: u32, epoch: &'a Range>, geofence: &'a impl GeofenceValidator, - gateway_query_timestamp: &'a DateTime, ) -> impl Stream> + 'a { heartbeats.then(move |heartbeat| async move { Self::validate( @@ -501,7 +499,6 @@ impl ValidatedHeartbeat { max_distance_to_coverage, epoch, geofence, - gateway_query_timestamp, ) .await }) diff --git a/mobile_verifier/src/heartbeats/wifi.rs b/mobile_verifier/src/heartbeats/wifi.rs index d5615eb85..ece7e7b20 100644 --- a/mobile_verifier/src/heartbeats/wifi.rs +++ b/mobile_verifier/src/heartbeats/wifi.rs @@ -148,7 +148,6 @@ where .into_stream(&mut transaction) .await? .map(Heartbeat::from); - let gateway_query_timestamp = Utc::now(); process_validated_heartbeats( ValidatedHeartbeat::validate_heartbeats( heartbeats, @@ -158,7 +157,6 @@ where self.max_distance_to_coverage, &epoch, &self.geofence, - &gateway_query_timestamp, ), heartbeat_cache, coverage_claim_time_cache, diff --git a/mobile_verifier/src/speedtests.rs b/mobile_verifier/src/speedtests.rs index 46b445f20..17d525fdb 100644 --- a/mobile_verifier/src/speedtests.rs +++ b/mobile_verifier/src/speedtests.rs @@ -151,11 +151,10 @@ where tracing::info!("Processing speedtest file {}", file.file_info.key); let mut transaction = self.pool.begin().await?; let mut speedtests = file.into_stream(&mut transaction).await?; - let gateway_query_timestamp = Utc::now(); while let Some(speedtest_report) = speedtests.next().await { let result = self - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .validate_speedtest(&speedtest_report) .await?; if result == SpeedtestResult::SpeedtestValid { save_speedtest(&speedtest_report.report, &mut transaction).await?; @@ -181,7 +180,6 @@ where pub async fn validate_speedtest( &self, speedtest: &CellSpeedtestIngestReport, - gateway_query_timestamp: &DateTime, ) -> anyhow::Result { if speedtest.report.upload_speed > SPEEDTEST_MAX_BYTES_PER_SECOND || speedtest.report.download_speed > SPEEDTEST_MAX_BYTES_PER_SECOND @@ -191,7 +189,7 @@ where match self .gateway_info_resolver - .resolve_gateway_info(&speedtest.report.pubkey, gateway_query_timestamp) + .resolve_gateway_info(&speedtest.report.pubkey, &speedtest.received_timestamp) .await? { Some(gw_info) if gw_info.is_data_only() => { diff --git a/mobile_verifier/tests/integrations/boosting_oracles.rs b/mobile_verifier/tests/integrations/boosting_oracles.rs index a8ed104ca..fa4c2a911 100644 --- a/mobile_verifier/tests/integrations/boosting_oracles.rs +++ b/mobile_verifier/tests/integrations/boosting_oracles.rs @@ -483,7 +483,6 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri let location_cache = LocationCache::new(&pool); let epoch = start..end; - let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -492,7 +491,6 @@ async fn test_footfall_and_urbanization_and_landtype_and_service_provider_overri 2000, &epoch, &MockGeofence, - &gateway_query_timestamp, )); let mut transaction = pool.begin().await?; while let Some(heartbeat) = heartbeats.next().await.transpose()? { diff --git a/mobile_verifier/tests/integrations/last_location.rs b/mobile_verifier/tests/integrations/last_location.rs index 09066110b..89660ab7f 100644 --- a/mobile_verifier/tests/integrations/last_location.rs +++ b/mobile_verifier/tests/integrations/last_location.rs @@ -41,7 +41,6 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; - let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -53,7 +52,6 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -72,7 +70,6 @@ async fn heartbeat_uses_last_good_location_when_invalid_location( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -106,7 +103,6 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: let mut transaction = pool.begin().await?; let coverage_object = coverage_object(&hotspot, &mut transaction).await?; transaction.commit().await?; - let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -118,7 +114,6 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -142,7 +137,6 @@ async fn heartbeat_will_use_last_good_location_from_db(pool: PgPool) -> anyhow:: u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -180,7 +174,6 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( transaction.commit().await?; let location_validation_timestamp = Utc::now(); - let gateway_query_timestamp = Utc::now(); let validated_heartbeat_1 = ValidatedHeartbeat::validate( heartbeat(&hotspot, &coverage_object) @@ -194,7 +187,6 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; @@ -215,7 +207,6 @@ async fn heartbeat_does_not_use_last_good_location_when_more_than_24_hours( u32::MAX, &(epoch_start..epoch_end), &MockGeofence, - &gateway_query_timestamp, ) .await?; diff --git a/mobile_verifier/tests/integrations/modeled_coverage.rs b/mobile_verifier/tests/integrations/modeled_coverage.rs index dd986df88..7ff7e6b9e 100644 --- a/mobile_verifier/tests/integrations/modeled_coverage.rs +++ b/mobile_verifier/tests/integrations/modeled_coverage.rs @@ -329,7 +329,6 @@ async fn process_wifi_input( .await?; let mut transaction = pool.begin().await?; - let gateway_query_timestamp = Utc::now(); let mut heartbeats = pin!(ValidatedHeartbeat::validate_heartbeats( stream::iter(heartbeats.map(Heartbeat::from)), &GatewayClientAllOwnersValid, @@ -338,7 +337,6 @@ async fn process_wifi_input( 2000, epoch, &MockGeofence, - &gateway_query_timestamp, )); while let Some(heartbeat) = heartbeats.next().await.transpose()? { let coverage_claim_time = coverage_claim_time_cache @@ -992,7 +990,6 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow let max_covered_distance = 5_000; let coverage_object_cache = CoverageObjectCache::new(&pool); let location_cache = LocationCache::new(&pool); - let gateway_query_timestamp = Utc::now(); let mk_heartbeat = |latlng: LatLng| WifiHeartbeatIngestReport { report: WifiHeartbeat { @@ -1017,7 +1014,6 @@ async fn ensure_lower_trust_score_for_distant_heartbeats(pool: PgPool) -> anyhow max_covered_distance, &(DateTime::::MIN_UTC..DateTime::::MAX_UTC), &MockGeofence, - &gateway_query_timestamp, ) }; diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index e05c54ebe..a5d951ca0 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -119,10 +119,8 @@ async fn speedtest_upload_exceeds_300megabits_ps_limit(pool: Pool) -> }, }; - let gateway_query_timestamp = Utc::now(); - let result = daemon - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .validate_speedtest(&speedtest_report) .await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); @@ -162,10 +160,8 @@ async fn speedtest_download_exceeds_300_megabits_ps_limit( }, }; - let gateway_query_timestamp = Utc::now(); - let result = daemon - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .validate_speedtest(&speedtest_report) .await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); @@ -205,10 +201,8 @@ async fn speedtest_both_speeds_exceed_300_megabits_ps_limit( }, }; - let gateway_query_timestamp = Utc::now(); - let result = daemon - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .validate_speedtest(&speedtest_report) .await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); @@ -248,10 +242,8 @@ async fn speedtest_within_300_megabits_ps_limit_should_be_valid( }, }; - let gateway_query_timestamp = Utc::now(); - let result = daemon - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .validate_speedtest(&speedtest_report) .await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); @@ -290,10 +282,9 @@ async fn speedtest_exactly_300_megabits_ps_limit_should_be_valid( latency: 10, }, }; - let gateway_query_timestamp = Utc::now(); let result = daemon - .validate_speedtest(&speedtest_report, &gateway_query_timestamp) + .validate_speedtest(&speedtest_report) .await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); From d2bd53595b8a77c03abbab981f8b85656747a1bd Mon Sep 17 00:00:00 2001 From: Connor McKenzie Date: Mon, 17 Nov 2025 15:45:12 +0000 Subject: [PATCH 35/35] Fix clippy & fmt --- mobile_packet_verifier/.idea/workspace.xml | 420 ------------------ mobile_packet_verifier/src/lib.rs | 2 +- mobile_verifier/src/speedtests.rs | 4 +- .../tests/integrations/speedtests.rs | 20 +- 4 files changed, 7 insertions(+), 439 deletions(-) delete mode 100644 mobile_packet_verifier/.idea/workspace.xml diff --git a/mobile_packet_verifier/.idea/workspace.xml b/mobile_packet_verifier/.idea/workspace.xml deleted file mode 100644 index 0f20b8889..000000000 --- a/mobile_packet_verifier/.idea/workspace.xml +++ /dev/null @@ -1,420 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { - "associatedIndex": 2 -} - - - - { - "keyToString": { - "ModuleVcsDetector.initialDetectionPerformed": "true", - "RunOnceActivity.ShowReadmeOnStart": "true", - "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true", - "RunOnceActivity.git.unshallow": "true", - "RunOnceActivity.rust.reset.selective.auto.import": "true", - "com.intellij.ml.llm.matterhorn.ej.ui.settings.DefaultModelSelectionForGA.v1": "true", - "git-widget-placeholder": "connor/mobile-verifier-historical", - "junie.onboarding.icon.badge.shown": "true", - "last_opened_file_path": "/Users/connormckenzie/Helium/oracles/mobile_packet_verifier", - "node.js.detected.package.eslint": "true", - "node.js.detected.package.tslint": "true", - "node.js.selected.package.eslint": "(autodetect)", - "node.js.selected.package.tslint": "(autodetect)", - "nodejs_package_manager_path": "npm", - "org.rust.cargo.project.model.PROJECT_DISCOVERY": "true", - "org.rust.cargo.project.model.impl.CargoExternalSystemProjectAware.subscribe.first.balloon": "", - "org.rust.first.attach.projects": "true", - "to.speed.mode.migration.done": "true", - "vue.rearranger.settings.migration": "true" - } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1761651090148 - - - - - - - - - - \ No newline at end of file diff --git a/mobile_packet_verifier/src/lib.rs b/mobile_packet_verifier/src/lib.rs index 140159a2c..6c0bddb14 100644 --- a/mobile_packet_verifier/src/lib.rs +++ b/mobile_packet_verifier/src/lib.rs @@ -61,7 +61,7 @@ impl MobileConfigResolverExt for MobileConfigClients { ) -> bool { match self .gateway_client - .resolve_gateway_info(public_key, &gateway_query_time) + .resolve_gateway_info(public_key, gateway_query_time) .await { Ok(res) => res.is_some(), diff --git a/mobile_verifier/src/speedtests.rs b/mobile_verifier/src/speedtests.rs index 17d525fdb..44b702237 100644 --- a/mobile_verifier/src/speedtests.rs +++ b/mobile_verifier/src/speedtests.rs @@ -153,9 +153,7 @@ where let mut speedtests = file.into_stream(&mut transaction).await?; while let Some(speedtest_report) = speedtests.next().await { - let result = self - .validate_speedtest(&speedtest_report) - .await?; + let result = self.validate_speedtest(&speedtest_report).await?; if result == SpeedtestResult::SpeedtestValid { save_speedtest(&speedtest_report.report, &mut transaction).await?; let latest_speedtests = get_latest_speedtests_for_pubkey( diff --git a/mobile_verifier/tests/integrations/speedtests.rs b/mobile_verifier/tests/integrations/speedtests.rs index a5d951ca0..34de9564d 100644 --- a/mobile_verifier/tests/integrations/speedtests.rs +++ b/mobile_verifier/tests/integrations/speedtests.rs @@ -119,9 +119,7 @@ async fn speedtest_upload_exceeds_300megabits_ps_limit(pool: Pool) -> }, }; - let result = daemon - .validate_speedtest(&speedtest_report) - .await?; + let result = daemon.validate_speedtest(&speedtest_report).await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -160,9 +158,7 @@ async fn speedtest_download_exceeds_300_megabits_ps_limit( }, }; - let result = daemon - .validate_speedtest(&speedtest_report) - .await?; + let result = daemon.validate_speedtest(&speedtest_report).await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -201,9 +197,7 @@ async fn speedtest_both_speeds_exceed_300_megabits_ps_limit( }, }; - let result = daemon - .validate_speedtest(&speedtest_report) - .await?; + let result = daemon.validate_speedtest(&speedtest_report).await?; assert_eq!(result, SpeedtestResult::SpeedtestValueOutOfBounds); Ok(()) @@ -242,9 +236,7 @@ async fn speedtest_within_300_megabits_ps_limit_should_be_valid( }, }; - let result = daemon - .validate_speedtest(&speedtest_report) - .await?; + let result = daemon.validate_speedtest(&speedtest_report).await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); Ok(()) @@ -283,9 +275,7 @@ async fn speedtest_exactly_300_megabits_ps_limit_should_be_valid( }, }; - let result = daemon - .validate_speedtest(&speedtest_report) - .await?; + let result = daemon.validate_speedtest(&speedtest_report).await?; assert_eq!(result, SpeedtestResult::SpeedtestValid); Ok(())