add migration and checks for distributed

Author: Eshanatnight

server/src/handlers/http/modal/ingest_server.rs

@@ -33,6 +33,7 @@ use crate::storage::object_storage::ingestor_metadata_path;
 use crate::storage::object_storage::parseable_json_path;
 use crate::storage::staging;
 use crate::storage::ObjectStorageError;
+use crate::storage::StorageMetadata;
 use crate::sync;
 
 use super::server::Server;
@@ -276,7 +277,24 @@ impl IngestServer {
         let path = parseable_json_path();
 
         match store.get_object(&path).await {
-            Ok(_) => Ok(()),
+            Ok(bytes) => {
+                let size = serde_json::from_slice::<StorageMetadata>(&bytes)?.hot_tier_capacity;
+                let hot_tier_enabled = CONFIG.is_hot_tier_active();
+                match size {
+                    Some(size) => {
+                        if hot_tier_enabled && CONFIG.parseable.local_cache_size != size {
+                            return Err(ObjectStorageError::Custom("Hot Tier Capacity does not match with Other Nodes. Please check the hot tier capacity and try again."));
+                        }
+                    }
+                    None => {
+                        if hot_tier_enabled {
+                            return Err(ObjectStorageError::Custom("Hot Tier is active on Current Node but disabled on Other Nodes. Please set hot tier and try again."));
+                        }
+                    }
+                }
+                // fall through
+                Ok(())
+            }
             Err(_) => Err(ObjectStorageError::Custom(
                 "Query Server has not been started yet. Please start the querier server first."
                     ,

server/src/migration/metadata_migration.rs

@@ -121,6 +121,14 @@ pub fn update_v3(mut storage_metadata: JsonValue) -> JsonValue {
         );
     }
 
+    let hot_tier_capacity = metadata.get("hot_tier_capacity");
+    if hot_tier_capacity.is_none() {
+        metadata.insert(
+            "hot_tier_capacity".to_string(),
+            JsonValue::Bool(CONFIG.is_hot_tier_active()),
+        );
+    }
+
     storage_metadata
 }
 

server/src/storage/store_metadata.rs

@@ -61,10 +61,17 @@ pub struct StorageMetadata {
     pub roles: HashMap<String, Vec<DefaultPrivilege>>,
     #[serde(default)]
     pub default_role: Option<String>,
+    pub hot_tier_capacity: Option<u64>,
 }
 
 impl StorageMetadata {
     pub fn new() -> Self {
+        let hot_tier_capacity = if CONFIG.is_hot_tier_active() {
+            Some(CONFIG.parseable.local_cache_size)
+        } else {
+            None
+        };
+
         Self {
             version: "v3".to_string(),
             mode: CONFIG.storage_name.to_owned(),
@@ -76,6 +83,7 @@ impl StorageMetadata {
             streams: Vec::new(),
             roles: HashMap::default(),
             default_role: None,
+            hot_tier_capacity,
         }
     }