add migration from v3 to v4 (#806)

Several older releases had changes in the parseable.json files
which was done using the `update_v3` function. This PR fixes 
that issue and instead does a clean migration to `v4`. 

Also rename the `ingester` to `ingestor`.

Author: nikhilsinhaparseable

server/src/migration.rs

@@ -54,16 +54,18 @@ pub async fn run_metadata_migration(config: &Config) -> anyhow::Result<()> {
     if let Some(storage_metadata) = storage_metadata {
         match get_version(&storage_metadata) {
             Some("v1") => {
-                let metadata = metadata_migration::v1_v3(storage_metadata);
+                let mut metadata = metadata_migration::v1_v3(storage_metadata);
+                metadata = metadata_migration::v3_v4(metadata);
                 put_remote_metadata(&*object_store, &metadata).await?;
             }
             Some("v2") => {
-                let metadata = metadata_migration::v2_v3(storage_metadata);
+                let mut metadata = metadata_migration::v2_v3(storage_metadata);
+                metadata = metadata_migration::v3_v4(metadata);
                 put_remote_metadata(&*object_store, &metadata).await?;
             }
             Some("v3") => {
-                let mdata = metadata_migration::update_v3(storage_metadata);
-                put_remote_metadata(&*object_store, &mdata).await?;
+                let metadata = metadata_migration::v3_v4(storage_metadata);
+                put_remote_metadata(&*object_store, &metadata).await?;
             }
             _ => (),
         }
@@ -73,16 +75,18 @@ pub async fn run_metadata_migration(config: &Config) -> anyhow::Result<()> {
     if let Some(staging_metadata) = staging_metadata {
         match get_version(&staging_metadata) {
             Some("v1") => {
-                let metadata = metadata_migration::v1_v3(staging_metadata);
+                let mut metadata = metadata_migration::v1_v3(staging_metadata);
+                metadata = metadata_migration::v3_v4(metadata);
                 put_staging_metadata(config, &metadata)?;
             }
             Some("v2") => {
-                let metadata = metadata_migration::v2_v3(staging_metadata);
+                let mut metadata = metadata_migration::v2_v3(staging_metadata);
+                metadata = metadata_migration::v3_v4(metadata);
                 put_staging_metadata(config, &metadata)?;
             }
             Some("v3") => {
-                let mdata = metadata_migration::update_v3(staging_metadata);
-                put_staging_metadata(config, &mdata)?;
+                let metadata = metadata_migration::v3_v4(staging_metadata);
+                put_staging_metadata(config, &metadata)?;
             }
             _ => (),
         }

server/src/migration/metadata_migration.rs

@@ -83,13 +83,22 @@ pub fn v2_v3(mut storage_metadata: JsonValue) -> JsonValue {
         // user is an object
         let user = user.as_object_mut().unwrap();
         // take out privileges
-        let JsonValue::Array(privileges) = user.remove("role").expect("role exists for v2") else {
+        let JsonValue::Array(mut privileges) = user.remove("role").expect("role exists for v2")
+        else {
             panic!("privileges is an arrray")
         };
 
         let mut roles = Vec::new();
 
         if !privileges.is_empty() {
+            for privilege in privileges.iter_mut() {
+                let privilege_value = privilege.get_mut("privilege");
+                if let Some(value) = privilege_value {
+                    if value.as_str().unwrap() == "ingester" {
+                        *value = JsonValue::String("ingestor".to_string());
+                    }
+                }
+            }
             let role_name =
                 rand::distributions::Alphanumeric.sample_string(&mut rand::thread_rng(), 8);
             privileges_map.push((role_name.clone(), JsonValue::Array(privileges)));
@@ -110,8 +119,9 @@ pub fn v2_v3(mut storage_metadata: JsonValue) -> JsonValue {
 }
 
 // maybe rename
-pub fn update_v3(mut storage_metadata: JsonValue) -> JsonValue {
+pub fn v3_v4(mut storage_metadata: JsonValue) -> JsonValue {
     let metadata = storage_metadata.as_object_mut().unwrap();
+    *metadata.get_mut("version").unwrap() = JsonValue::String("v4".to_string());
     let sm = metadata.get("server_mode");
 
     if sm.is_none() || sm.unwrap().as_str().unwrap() == "All" {
@@ -121,6 +131,20 @@ pub fn update_v3(mut storage_metadata: JsonValue) -> JsonValue {
         );
     }
 
+    let roles = metadata.get_mut("roles").unwrap().as_object_mut().unwrap();
+    for (_, privileges) in roles.iter_mut() {
+        let JsonValue::Array(privileges) = privileges else {
+            panic!("privileges is an array")
+        };
+        for privilege in privileges.iter_mut() {
+            let privilege_value = privilege.get_mut("privilege");
+            if let Some(value) = privilege_value {
+                if value.as_str().unwrap() == "ingester" {
+                    *value = JsonValue::String("ingestor".to_string());
+                }
+            }
+        }
+    }
     storage_metadata
 }