refactor(sps): move to macro config (#4194)

Author: whutchinson98

.github/workspace-dep-closures.json

@@ -3377,6 +3377,8 @@
       "lexical_client",
       "macro_auth",
       "macro_aws_config",
+      "macro_config",
+      "macro_config_derive",
       "macro_cors",
       "macro_db_client",
       "macro_entrypoint",

rust/cloud-storage/Cargo.lock

@@ -30,6 +30,7 @@ http-body-util = { workspace = true }
 lexical_client = { path = "../lexical_client" }
 macro_aws_config = { path = "../macro_aws_config", features = ["s3"] }
 macro_cors = { path = "../macro_cors" }
+macro_config = { path = "../macro_config" }
 macro_db_client = { path = "../macro_db_client", optional = true } # Make this optional
 macro_entrypoint = { path = "../macro_entrypoint" }
 macro_env = { path = "../macro_env" }

rust/cloud-storage/search_processing_service/Cargo.toml

@@ -2,29 +2,25 @@ use anyhow::Context;
 pub use macro_env::Environment;
 use macro_env_var::{env_vars, maybe_env_vars};
 use macro_service_urls::LexicalServiceUrl;
+use secretsmanager_client::LocalOrRemoteSecret;
 
 env_vars! {
-    struct DatabaseUrl;
-    struct SearchEventQueue;
-    struct OpensearchUrl;
-    struct OpensearchUsername;
-    struct OpensearchPassword;
-    struct DocumentStorageBucket;
-    struct BackfillJobsTable;
+    pub struct DatabaseUrl;
+    pub struct SearchEventQueue;
+    pub struct OpensearchUrl;
+    pub struct OpensearchUsername;
+    pub struct OpensearchPassword;
+    pub struct DocumentStorageBucket;
+    pub struct BackfillJobsTable;
 }
 
 maybe_env_vars! {
-    struct DatabaseUrlReadonly;
-    struct Port;
-    struct QueueMaxMessages;
-    struct QueueWaitTimeSeconds;
-    struct WorkerCount;
-    struct BackfillCallsPageSize;
-    struct BackfillChatsPageSize;
-    struct BackfillChannelsPageSize;
-    struct BackfillDocumentsPageSize;
-    struct BackfillEmailsPageSize;
-    struct BackfillJobTtlSeconds;
+    pub struct BackfillCallsPageSize;
+    pub struct BackfillChatsPageSize;
+    pub struct BackfillChannelsPageSize;
+    pub struct BackfillDocumentsPageSize;
+    pub struct BackfillEmailsPageSize;
+    pub struct BackfillJobTtlSeconds;
 }
 
 /// Per-entity DB page sizes used by the backfill source adapters. Tunable at
@@ -43,11 +39,41 @@ const DEFAULT_CHATS_PAGE: usize = 5000;
 const DEFAULT_CHANNELS_PAGE: usize = 5000;
 const DEFAULT_DOCUMENTS_PAGE: usize = 1000;
 const DEFAULT_EMAILS_PAGE: usize = 1000;
+const DEFAULT_BACKFILL_JOB_TTL_SECONDS: u64 = 24 * 60 * 60;
 
+fn parse_page_size(name: &str, raw_value: Option<&str>, default: usize) -> anyhow::Result<usize> {
+    let page_size = raw_value
+        .map(|raw| {
+            raw.parse::<usize>()
+                .with_context(|| format!("{name} must be a positive integer"))
+        })
+        .transpose()?
+        .unwrap_or(default);
+
+    if page_size == 0 {
+        anyhow::bail!("{name} must be > 0");
+    }
+
+    Ok(page_size)
+}
+
+fn parse_u64(name: &str, raw_value: Option<&str>, default: u64) -> anyhow::Result<u64> {
+    raw_value
+        .map(|raw| {
+            raw.parse::<u64>()
+                .with_context(|| format!("{name} must be a positive integer"))
+        })
+        .transpose()
+        .map(|value| value.unwrap_or(default))
+}
+
+/// The configuration parameters for the application.
+#[derive(macro_config::MacroConfig)]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub struct Config {
     /// The connection URL for the Postgres database this application should use.
     /// For deployed applications, this is a secret stored in AWS Secrets Manager.
-    pub database_url: String,
+    pub database_url: LocalOrRemoteSecret<DatabaseUrl>,
 
     /// Optional connection URL (or SM secret id when `environment != Local`)
     /// for the macrodb read-replica. When present, backfill reads run against
@@ -58,181 +84,101 @@ pub struct Config {
     pub database_url_readonly: Option<String>,
 
     /// The port to listen for HTTP requests on.
+    #[macro_config_default(8080)]
     pub port: usize,
 
     /// The search text extractor queue
-    pub search_event_queue: String,
+    pub search_event_queue: SearchEventQueue,
     /// The queue max messages per poll
+    #[macro_config_default(10)]
     pub queue_max_messages: i32,
     /// The queue wait time seconds
+    #[macro_config_default(20)]
     pub queue_wait_time_seconds: i32,
 
     /// The environment we are in
+    #[macro_config_default(Environment::new_or_prod())]
     pub environment: Environment,
 
     /// The URL for the Opensearch instance
-    pub opensearch_url: String,
+    pub opensearch_url: OpensearchUrl,
     /// The username for the Opensearch instance
-    pub opensearch_username: String,
+    pub opensearch_username: OpensearchUsername,
     /// The password for the Opensearch instance
-    pub opensearch_password: String,
+    pub opensearch_password: LocalOrRemoteSecret<OpensearchPassword>,
 
     /// The bucket where documents are stored
-    pub document_storage_bucket: String,
+    pub document_storage_bucket: DocumentStorageBucket,
 
     /// The number of workers to spawn
+    #[macro_config_default(10)]
     pub worker_count: u8,
 
     /// The URL for the Lexical service
+    #[macro_config_default(LexicalServiceUrl::unwrap_new().to_string())]
     pub lexical_service_url: String,
 
-    /// Per-entity DB page sizes for backfill adapters.
-    pub backfill_page_sizes: BackfillPageSizes,
+    /// DB page size used when backfilling call records.
+    pub backfill_calls_page_size: BackfillCallsPageSize,
+    /// DB page size used when backfilling chats.
+    pub backfill_chats_page_size: BackfillChatsPageSize,
+    /// DB page size used when backfilling channels.
+    pub backfill_channels_page_size: BackfillChannelsPageSize,
+    /// DB page size used when backfilling documents.
+    pub backfill_documents_page_size: BackfillDocumentsPageSize,
+    /// DB page size used when backfilling emails.
+    pub backfill_emails_page_size: BackfillEmailsPageSize,
 
     /// DynamoDB table name backing the backfill job registry. Items carry an
     /// `expires_at` epoch attribute that DynamoDB's TTL sweeps in the
     /// background, so completed jobs vanish on their own.
-    pub backfill_jobs_table: String,
+    pub backfill_jobs_table: BackfillJobsTable,
 
     /// TTL applied to the `expires_at` attribute on each job record. Acts as
     /// the GC mechanism — DynamoDB removes items shortly after this elapses.
-    pub backfill_job_ttl_seconds: u64,
-}
-
-fn parse_page_size(name: &str, raw_value: Option<String>, default: usize) -> anyhow::Result<usize> {
-    match raw_value {
-        Some(raw) => raw
-            .parse::<usize>()
-            .with_context(|| format!("{name} must be a positive integer"))
-            .and_then(|n| {
-                if n == 0 {
-                    anyhow::bail!("{name} must be > 0");
-                }
-                Ok(n)
-            }),
-        None => Ok(default),
-    }
+    pub backfill_job_ttl_seconds: BackfillJobTtlSeconds,
 }
 
 impl Config {
     pub fn from_env() -> anyhow::Result<Self> {
-        let database_url = DatabaseUrl::new()
-            .context("DATABASE_URL must be provided")?
-            .to_string();
-
-        let database_url_readonly = DatabaseUrlReadonly::new().map(|url| url.to_string());
-
-        let port = Port::new()
-            .map(|port| port.parse::<usize>().context("should be valid port number"))
-            .transpose()?
-            .unwrap_or(8080);
-
-        let environment = Environment::new_or_prod();
-
-        let search_event_queue = SearchEventQueue::new()
-            .context("SEARCH_EVENT_QUEUE must be provided")?
-            .to_string();
-
-        let queue_max_messages = QueueMaxMessages::new()
-            .map(|queue_max_messages| {
-                queue_max_messages
-                    .parse::<i32>()
-                    .context("QUEUE_MAX_MESSAGES must be a valid i32")
-            })
-            .transpose()?
-            .unwrap_or(10);
-
-        let queue_wait_time_seconds = QueueWaitTimeSeconds::new()
-            .map(|queue_wait_time_seconds| {
-                queue_wait_time_seconds
-                    .parse::<i32>()
-                    .context("QUEUE_WAIT_TIME_SECONDS must be a valid i32")
-            })
-            .transpose()?
-            .unwrap_or(20);
-
-        let opensearch_url = OpensearchUrl::new()
-            .context("OPENSEARCH_URL must be provided")?
-            .to_string();
-        let opensearch_username = OpensearchUsername::new()
-            .context("OPENSEARCH_USERNAME must be provided")?
-            .to_string();
-        let opensearch_password = OpensearchPassword::new()
-            .context("OPENSEARCH_PASSWORD must be provided")?
-            .to_string();
-
-        let document_storage_bucket = DocumentStorageBucket::new()
-            .context("DOCUMENT_STORAGE_BUCKET must be provided")?
-            .to_string();
-
-        let worker_count = WorkerCount::new()
-            .map(|worker_count| {
-                worker_count
-                    .parse::<u8>()
-                    .context("WORKER_COUNT must be a valid u8")
-            })
-            .transpose()?
-            .unwrap_or(10);
-
-        let lexical_service_url = LexicalServiceUrl::new()?.to_string();
-
-        let backfill_page_sizes = BackfillPageSizes {
+        macro_config::ConfigLoader::load::<Config>().context("failed to load config")
+    }
+
+    pub fn backfill_page_sizes(&self) -> anyhow::Result<BackfillPageSizes> {
+        Ok(BackfillPageSizes {
             calls: parse_page_size(
                 "BACKFILL_CALLS_PAGE_SIZE",
-                BackfillCallsPageSize::new().map(|value| value.to_string()),
+                self.backfill_calls_page_size.value(),
                 DEFAULT_CALLS_PAGE,
             )?,
             chats: parse_page_size(
                 "BACKFILL_CHATS_PAGE_SIZE",
-                BackfillChatsPageSize::new().map(|value| value.to_string()),
+                self.backfill_chats_page_size.value(),
                 DEFAULT_CHATS_PAGE,
             )?,
             channels: parse_page_size(
                 "BACKFILL_CHANNELS_PAGE_SIZE",
-                BackfillChannelsPageSize::new().map(|value| value.to_string()),
+                self.backfill_channels_page_size.value(),
                 DEFAULT_CHANNELS_PAGE,
             )?,
             documents: parse_page_size(
                 "BACKFILL_DOCUMENTS_PAGE_SIZE",
-                BackfillDocumentsPageSize::new().map(|value| value.to_string()),
+                self.backfill_documents_page_size.value(),
                 DEFAULT_DOCUMENTS_PAGE,
             )?,
             emails: parse_page_size(
                 "BACKFILL_EMAILS_PAGE_SIZE",
-                BackfillEmailsPageSize::new().map(|value| value.to_string()),
+                self.backfill_emails_page_size.value(),
                 DEFAULT_EMAILS_PAGE,
             )?,
-        };
-
-        let backfill_jobs_table = BackfillJobsTable::new()
-            .context("BACKFILL_JOBS_TABLE must be provided")?
-            .to_string();
-        let backfill_job_ttl_seconds = BackfillJobTtlSeconds::new()
-            .map(|backfill_job_ttl_seconds| {
-                backfill_job_ttl_seconds
-                    .parse::<u64>()
-                    .context("BACKFILL_JOB_TTL_SECONDS must be a positive integer")
-            })
-            .transpose()?
-            .unwrap_or(24 * 60 * 60);
-
-        Ok(Config {
-            database_url,
-            database_url_readonly,
-            port,
-            search_event_queue,
-            queue_max_messages,
-            queue_wait_time_seconds,
-            environment,
-            opensearch_url,
-            opensearch_username,
-            opensearch_password,
-            document_storage_bucket,
-            worker_count,
-            lexical_service_url,
-            backfill_page_sizes,
-            backfill_jobs_table,
-            backfill_job_ttl_seconds,
         })
     }
+
+    pub fn backfill_job_ttl_seconds(&self) -> anyhow::Result<u64> {
+        parse_u64(
+            "BACKFILL_JOB_TTL_SECONDS",
+            self.backfill_job_ttl_seconds.value(),
+            DEFAULT_BACKFILL_JOB_TTL_SECONDS,
+        )
+    }
 }