feat: optimize metastore migration for reliability and scale

- Implement streaming for split processing to reduce memory usage.
- Add resume capability by skipping existing indexes.
- Add fault tolerance to continue migration on individual index failures.
- Pin Rust version to 1.93 via rust-toolchain.toml.

Author: hatemosphere

.gitignore

@@ -6,6 +6,7 @@ test/results/
 
 # Extra test doc generated at runtime
 test/extra-docs.ndjson
+test/node-local.yaml
 
 # OS junk
 .DS_Store

Cargo.lock

@@ -23,3 +23,4 @@ tokio = { version = "1.48", features = ["full"] }
 tokio-metrics = { version = "0.4", features = ["rt"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+futures = "0.3.31"

Cargo.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "1.93"

rust-toolchain.toml

@@ -84,11 +84,21 @@ async fn main() -> anyhow::Result<()> {
     }
 
     tracing::info!(
-        indexes_migrated = report.indexes_migrated,
+        indexes_succeeded = report.indexes_succeeded,
+        indexes_failed = report.indexes_failed,
         total_splits = report.total_splits,
         total_delete_tasks = report.total_delete_tasks,
         "migration complete"
     );
 
+    if !report.index_failures.is_empty() {
+        tracing::error!("failures detected:");
+        for (idx, err) in &report.index_failures {
+             tracing::error!(index_id = %idx, error = %err, "failed");
+        }
+        // Exit non-zero if there were failures
+        std::process::exit(1);
+    }
+
     Ok(())
 }