Introduce ephemeral index cache

.cspell/project_words.txt

@@ -38,6 +38,7 @@ lütke
 marietti
 mbps
 mehrzad
+memmap
 Mjpeg
 Mmap
 mpscrr
@@ -77,6 +78,7 @@ tobiaslutke
 tokio
 tombstoned
 typecheck
+Uninit
 unwatch
 uuid
 vdfs

apps/cli/src/domains/index/args.rs

@@ -4,6 +4,7 @@ use uuid::Uuid;
 
 use sd_core::{
 	domain::addressing::SdPath,
+	ops::core::ephemeral_status::EphemeralCacheStatusInput,
 	ops::indexing::{
 		input::IndexInput,
 		job::{IndexMode, IndexPersistence, IndexScope},
@@ -169,3 +170,19 @@ impl IndexVerifyArgs {
 		}
 	}
 }
+
+/// Arguments for ephemeral cache status
+#[derive(Args, Debug, Clone)]
+pub struct EphemeralCacheArgs {
+	/// Filter by path substring
+	#[arg(long)]
+	pub filter: Option<String>,
+}
+
+impl EphemeralCacheArgs {
+	pub fn to_input(&self) -> EphemeralCacheStatusInput {
+		EphemeralCacheStatusInput {
+			path_filter: self.filter.clone(),
+		}
+	}
+}

apps/cli/src/domains/index/mod.rs

@@ -2,6 +2,7 @@ pub mod args;
 
 use anyhow::Result;
 use clap::Subcommand;
+use comfy_table::{presets::UTF8_BORDERS_ONLY, Attribute, Cell, Table};
 
 use crate::util::prelude::*;
 
@@ -20,6 +21,8 @@ pub enum IndexCmd {
 	Browse(BrowseArgs),
 	/// Verify index integrity for a path
 	Verify(IndexVerifyArgs),
+	/// Show ephemeral index cache status
+	EphemeralCache(EphemeralCacheArgs),
 }
 
 pub async fn run(ctx: &Context, cmd: IndexCmd) -> Result<()> {
@@ -232,6 +235,118 @@ pub async fn run(ctx: &Context, cmd: IndexCmd) -> Result<()> {
 				}
 			);
 		}
+		IndexCmd::EphemeralCache(args) => {
+			let input = args.to_input();
+			let out: sd_core::ops::core::ephemeral_status::EphemeralCacheStatus =
+				execute_core_query!(ctx, input);
+
+			print_output!(
+				ctx,
+				&out,
+				|status: &sd_core::ops::core::ephemeral_status::EphemeralCacheStatus| {
+					println!();
+					println!("╔══════════════════════════════════════════════════════════════╗");
+					println!("║           UNIFIED EPHEMERAL INDEX CACHE                      ║");
+					println!("╠══════════════════════════════════════════════════════════════╣");
+					println!(
+						"║ Indexed Paths: {:3}    In Progress: {:3}                       ║",
+						status.indexed_paths_count, status.indexing_in_progress_count
+					);
+					println!("╚══════════════════════════════════════════════════════════════╝");
+
+					// Show unified index stats
+					let stats = &status.index_stats;
+					println!();
+					let mut stats_table = Table::new();
+					stats_table.load_preset(UTF8_BORDERS_ONLY);
+					stats_table.set_header(vec![
+						Cell::new("SHARED INDEX STATS").add_attribute(Attribute::Bold),
+						Cell::new(""),
+					]);
+
+					stats_table.add_row(vec![
+						"Total entries (shared arena)",
+						&stats.total_entries.to_string(),
+					]);
+					stats_table.add_row(vec![
+						"Path index count",
+						&stats.path_index_count.to_string(),
+					]);
+					stats_table.add_row(vec![
+						"Unique names (shared)",
+						&stats.unique_names.to_string(),
+					]);
+					stats_table.add_row(vec![
+						"Interned strings (shared)",
+						&stats.interned_strings.to_string(),
+					]);
+					stats_table.add_row(vec!["Content kinds", &stats.content_kinds.to_string()]);
+					stats_table.add_row(vec![
+						"Memory usage",
+						&format_bytes(stats.memory_bytes as u64),
+					]);
+					stats_table.add_row(vec!["Cache age", &format!("{:.1}s", stats.age_seconds)]);
+					stats_table.add_row(vec!["Idle time", &format!("{:.1}s", stats.idle_seconds)]);
+
+					println!("{}", stats_table);
+
+					// Show indexed paths
+					if status.indexed_paths.is_empty() && status.paths_in_progress.is_empty() {
+						println!("\n  No paths indexed yet.");
+					} else {
+						// Paths in progress
+						if !status.paths_in_progress.is_empty() {
+							println!();
+							let mut progress_table = Table::new();
+							progress_table.load_preset(UTF8_BORDERS_ONLY);
+							progress_table
+								.set_header(vec![Cell::new("INDEXING IN PROGRESS")
+									.add_attribute(Attribute::Bold)]);
+							for path in &status.paths_in_progress {
+								progress_table.add_row(vec![format!("● {}", path.display())]);
+							}
+							println!("{}", progress_table);
+						}
+
+						// Indexed paths
+						if !status.indexed_paths.is_empty() {
+							println!();
+							let mut paths_table = Table::new();
+							paths_table.load_preset(UTF8_BORDERS_ONLY);
+							paths_table.set_header(vec![
+								Cell::new("INDEXED PATHS").add_attribute(Attribute::Bold),
+								Cell::new("Children"),
+							]);
+							for info in &status.indexed_paths {
+								paths_table.add_row(vec![
+									format!("○ {}", info.path.display()),
+									info.child_count.to_string(),
+								]);
+							}
+							println!("{}", paths_table);
+						}
+					}
+					println!();
+				}
+			);
+		}
 	}
 	Ok(())
 }
+
+fn format_bytes(bytes: u64) -> String {
+	const UNITS: &[&str] = &["B", "KB", "MB", "GB", "TB"];
+	let mut size = bytes as f64;
+	let mut unit_index = 0;
+
+	while size >= 1024.0 && unit_index < UNITS.len() - 1 {
+		size /= 1024.0;
+		unit_index += 1;
+	}
+
+	if unit_index == 0 {
+		format!("{} {}", bytes, UNITS[unit_index])
+	} else {
+		format!("{:.1} {}", size, UNITS[unit_index])
+	}
+}

apps/cli/src/domains/update/mod.rs

@@ -268,4 +268,4 @@ async fn start_daemon(data_dir: &PathBuf) -> Result<()> {
 	tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
 
 	Ok(())
-}
+}

apps/mobile/modules/sd-mobile-core/package.json

@@ -2,5 +2,8 @@
   "name": "sd-mobile-core",
   "version": "1.0.0",
   "main": "./src/index.ts",
-  "types": "./src/index.ts"
+  "types": "./src/index.ts",
+  "peerDependencies": {
+    "expo-modules-core": "*"
+  }
 }

apps/mobile/modules/sd-mobile-core/src/index.ts

@@ -1,13 +1,7 @@
-// @ts-ignore - Expo modules types may not be available in all environments
-const { EventEmitter, NativeModulesProxy } = require("expo-modules-core");
+// TODO: Test if we can rely on Expo's autolinking instead of manually requiring the module
+import { requireNativeModule, EventEmitter } from "expo-modules-core";
 
-const SDMobileCoreModule = NativeModulesProxy?.SDMobileCore;
-
-if (!SDMobileCoreModule) {
-	throw new Error(
-		"SDMobileCore native module not found. Did you run 'cargo xtask build-mobile' and rebuild the app?",
-	);
-}
+const SDMobileCoreModule = requireNativeModule("SDMobileCore");
 
 const emitter = new EventEmitter(SDMobileCoreModule);
 

apps/tauri/src-tauri/src/main.rs

@@ -800,17 +800,25 @@ async fn stop_daemon_process(
 async fn check_daemon_installed() -> Result<bool, String> {
 	#[cfg(target_os = "macos")]
 	{
-		let home = std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
-		let plist_path = std::path::PathBuf::from(home).join("Library/LaunchAgents/com.spacedrive.daemon.plist");
+		let home =
+			std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
+		let plist_path =
+			std::path::PathBuf::from(home).join("Library/LaunchAgents/com.spacedrive.daemon.plist");
 		let exists = plist_path.exists();
-		tracing::info!("Checking daemon installation at {}: {}", plist_path.display(), exists);
+		tracing::info!(
+			"Checking daemon installation at {}: {}",
+			plist_path.display(),
+			exists
+		);
 		Ok(exists)
 	}
 
 	#[cfg(target_os = "linux")]
 	{
-		let home = std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
-		let service_path = std::path::PathBuf::from(home).join(".config/systemd/user/spacedrive-daemon.service");
+		let home =
+			std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
+		let service_path =
+			std::path::PathBuf::from(home).join(".config/systemd/user/spacedrive-daemon.service");
 		Ok(service_path.exists())
 	}
 
@@ -865,7 +873,8 @@ async fn install_daemon_service(
 	{
 		use std::io::Write;
 
-		let home = std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
+		let home =
+			std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
 		let launch_agents_dir = std::path::PathBuf::from(&home).join("Library/LaunchAgents");
 
 		std::fs::create_dir_all(&launch_agents_dir)
@@ -881,7 +890,10 @@ async fn install_daemon_service(
 			.join("sd-daemon");
 
 		if !daemon_path.exists() {
-			return Err(format!("Daemon binary not found at {}", daemon_path.display()));
+			return Err(format!(
+				"Daemon binary not found at {}",
+				daemon_path.display()
+			));
 		}
 
 		let log_dir = data_dir.join("logs");
@@ -938,7 +950,10 @@ async fn install_daemon_service(
 			.output()
 			.map_err(|e| format!("Failed to load service: {}", e))?;
 
-		tracing::info!("launchctl load output: {:?}", String::from_utf8_lossy(&output.stdout));
+		tracing::info!(
+			"launchctl load output: {:?}",
+			String::from_utf8_lossy(&output.stdout)
+		);
 		if !output.status.success() {
 			let stderr = String::from_utf8_lossy(&output.stderr);
 			tracing::error!("launchctl load failed: {:?}", stderr);
@@ -980,7 +995,8 @@ async fn install_daemon_service(
 	{
 		use std::io::Write;
 
-		let home = std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
+		let home =
+			std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
 		let systemd_dir = std::path::PathBuf::from(&home).join(".config/systemd/user");
 
 		std::fs::create_dir_all(&systemd_dir)
@@ -995,7 +1011,10 @@ async fn install_daemon_service(
 			.join("sd-daemon");
 
 		if !daemon_path.exists() {
-			return Err(format!("Daemon binary not found at {}", daemon_path.display()));
+			return Err(format!(
+				"Daemon binary not found at {}",
+				daemon_path.display()
+			));
 		}
 
 		let service_content = format!(
@@ -1112,7 +1131,10 @@ WantedBy=default.target
 			.join("sd-daemon.exe");
 
 		if !daemon_path.exists() {
-			return Err(format!("Daemon binary not found at {}", daemon_path.display()));
+			return Err(format!(
+				"Daemon binary not found at {}",
+				daemon_path.display()
+			));
 		}
 
 		// Delete existing task if it exists
@@ -1248,8 +1270,10 @@ WantedBy=default.target
 async fn uninstall_daemon_service() -> Result<(), String> {
 	#[cfg(target_os = "macos")]
 	{
-		let home = std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
-		let plist_path = std::path::PathBuf::from(&home).join("Library/LaunchAgents/com.spacedrive.daemon.plist");
+		let home =
+			std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
+		let plist_path = std::path::PathBuf::from(&home)
+			.join("Library/LaunchAgents/com.spacedrive.daemon.plist");
 
 		if plist_path.exists() {
 			// Unload the service
@@ -1266,8 +1290,10 @@ async fn uninstall_daemon_service() -> Result<(), String> {
 
 	#[cfg(target_os = "linux")]
 	{
-		let home = std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
-		let service_path = std::path::PathBuf::from(&home).join(".config/systemd/user/spacedrive-daemon.service");
+		let home =
+			std::env::var("HOME").map_err(|_| "Could not determine home directory".to_string())?;
+		let service_path =
+			std::path::PathBuf::from(&home).join(".config/systemd/user/spacedrive-daemon.service");
 
 		if service_path.exists() {
 			// Stop and disable the service

core/Cargo.toml

@@ -19,9 +19,10 @@ cli = []
 
 [dependencies]
 # Async runtime
-async-trait = "0.1"
-futures     = "0.3"
-tokio       = { version = "1.40", features = ["full"] }
+async-channel = { workspace = true }
+async-trait   = "0.1"
+futures       = "0.3"
+tokio         = { version = "1.40", features = ["full"] }
 
 # Database
 sea-orm = { version = "1.1", features = [
@@ -166,6 +167,12 @@ once_cell = "1.20"
 rand      = "0.8"                                                        # Random number generation for secure delete
 tempfile  = "3.14"                                                       # Temporary directories for testing
 uuid      = { version = "1.11", features = ["serde", "v4", "v5", "v7"] }
+
+# High-performance ephemeral index
+memmap2      = "0.9"  # Memory-mapped file support for arena storage
+smallvec     = "1.13" # Small vector optimization for children arrays
+parking_lot  = "0.12" # Fast mutex for name cache
+num_cpus     = "1.16" # CPU count for parallel walker
 whoami    = "1.5"
 
 # Secure storage