update experiment scripts

Author: happyandslow

benchmarks/autoscaling/README.md

@@ -43,10 +43,10 @@ For example,
 There are two plots that you can plot.
 
 ### Generating report
-`python <aibrix_root_repo>/benchmarks/plot/plot-everything.py <experiment_home_dir>`
+`python plot-everything.py <experiment_home_dir>`
 
 For example, 
-`python <aibrix_root_repo>/benchmarks/plot/plot-everything.py experiment_results/25min_test`
+`python plot-everything.py experiment_results/25min_test`
 
 The directories should look like
 ```bash

benchmarks/autoscaling/plot-everything.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import os
 import json
+import re
 import pandas as pd
 import matplotlib.pyplot as plt
 import numpy as np
@@ -24,10 +25,15 @@ def parse_experiment_output(lines):
             continue
         try:
             data = json.loads(line.strip())
-            required_fields = ['status_code', 'start_time', 'end_time', 'latency', 'throughput', 
+            # required_fields = ['status_code', 'start_time', 'end_time', 'latency', 'throughput', 
+            #                  'prompt_tokens', 'output_tokens', 'total_tokens', 'input', 'output']
+            required_fields = ['start_time', 'end_time', 'latency', 'throughput', 
                              'prompt_tokens', 'output_tokens', 'total_tokens', 'input', 'output']
             if any(field not in data for field in required_fields):
+                missingfields = [field not in data for field in required_fields]
+                print(missingfields)
                 continue
+            data['status_code'] = 200
             results.append(data)
         except json.JSONDecodeError:
             continue
@@ -52,15 +58,27 @@ def parse_experiment_output(lines):
 
 def get_autoscaler_name(output_dir):
     autoscaling = None
-    with open(f"{output_dir}/output.txt", 'r', encoding='utf-8') as f_:
-        lines = f_.readlines()
-        for line in lines:
-            if "autoscaler" in line:
-                autoscaling = line.split(":")[-1].strip()
-                break
+    print(f"output_dir: {output_dir}")
+    # Extract the last part of the path after the last slash
+    filename = output_dir.split("/")[-1]
+
+    # Regular expression to match the autoscaler name
+    match = re.search(r"^[^-]+-[^-]+-([^-]+(?:-[^-]+)*)-\d{8}-\d{6}$", filename)
+    
+    if match:
+        print(match)
+        autoscaling = match.group(1)
+    
+    # with open(f"{output_dir}/output.txt", 'r', encoding='utf-8') as f_:
+    #     lines = f_.readlines()
+    #     for line in lines:
+    #         if "autoscaler" in line:
+    #             autoscaling = line.split(":")[-1].strip()
+    #             break
     if autoscaling == None:
         print(f"Invalid parsed autoscaling name: {autoscaling}")
         assert False
+    print(autoscaling)
     return autoscaling.upper()
 
 def parse_performance_stats(file_content):

benchmarks/autoscaling/run-test.sh

@@ -5,6 +5,8 @@ autoscaler=$2
 aibrix_repo=$3 # root dir of aibrix repo
 api_key=$4 # set your api key
 kube_context=$5
+workload_type=$6
+
 k8s_yaml_dir="deepseek-llm-7b-chat"
 target_deployment="deepseek-llm-7b-chat" # "aibrix-model-deepseek-llm-7b-chat"
 target_ai_model=deepseek-llm-7b-chat
@@ -38,7 +40,7 @@ fi
 
 # Setup experiment directory
 workload_name=$(echo $input_workload_path | tr '/' '\n' | grep .jsonl | cut -d '.' -f 1)
-experiment_result_dir="experiment_results/${workload_name}-${autoscaler}-$(date +%Y%m%d-%H%M%S)"
+experiment_result_dir="experiment_results/${workload_type}/${workload_name}-${workload_type}-${autoscaler}-$(date +%Y%m%d-%H%M%S)"
 if [ ! -d ${experiment_result_dir} ]; then
     echo "output directory does not exist. Create the output directory (${experiment_result_dir})"
     mkdir -p ${experiment_result_dir}
@@ -83,9 +85,9 @@ kubectl rollout restart deploy ${target_deployment} -n default
 sleep_before_pod_check=20
 echo "Sleep for ${sleep_before_pod_check} seconds after restarting deployment"
 sleep ${sleep_before_pod_check}
-python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py ${target_deployment}
-python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-controller-manager
-python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-gateway-plugins
+python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py ${target_deployment} ${kube_context}
+python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-controller-manager ${kube_context}
+python3 ${aibrix_repo}/benchmarks/utils/check_k8s_is_ready.py aibrix-gateway-plugins ${kube_context}
 
 # Start pod log monitoring
 pod_log_dir="${experiment_result_dir}/pod_logs"
@@ -95,7 +97,7 @@ mkdir -p ${pod_log_dir}
 cp ${input_workload_path} ${experiment_result_dir}
 
 # Start pod counter. It will run on background until the end of the experiment.
-python3 ${aibrix_repo}/benchmarks/utils/count_num_pods.py ${target_deployment} ${experiment_result_dir} &
+python3 ${aibrix_repo}/benchmarks/utils/count_num_pods.py ${target_deployment} ${experiment_result_dir} ${kube_context} &
 COUNT_NUM_POD_PID=$!
 echo "started count_num_pods.py with PID: $COUNT_NUM_POD_PID"
 
@@ -106,13 +108,13 @@ python3 ${aibrix_repo}/benchmarks/utils/streaming_pod_log_to_file.py aibrix-gate
 
 # Run experiment!!!
 output_jsonl_path=${experiment_result_dir}/output.jsonl
-python3 ${aibrix_repo}/benchmarks/generator/client.py \
+python3 ${aibrix_repo}/benchmarks/client/client.py \
     --workload-path ${input_workload_path} \
-    --endpoint "localhost:8888" \
+    --endpoint "http://localhost:8888" \
     --model ${target_ai_model} \
     --api-key ${api_key} \
-    --output-dir ${experiment_result_dir} \
-    --output-file-path ${output_jsonl_path}
+    --output-file-path ${output_jsonl_path} \
+    #--output-dir ${experiment_result_dir} \
 
 echo "Experiment is done. date: $(date)"
 
@@ -124,7 +126,7 @@ sleep 1
 
 # Cleanup
 kubectl delete podautoscaler --all --all-namespaces
-python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1
+python3 ${aibrix_repo}/benchmarks/utils/set_num_replicas.py --deployment ${target_deployment} --replicas 1 --context ${kube_context}
 kubectl delete -f ${k8s_yaml_dir}/deploy.yaml
 
 # Stop monitoring processes

benchmarks/autoscaling/run.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
-
+set -x
 #./run.sh workload/workload/25min_up_and_down/25min_up_and_down.jsonl
 
 export KUBECONFIG=~/.kube/config-vke
-export aibrix_repo="/root/aibrix"
+export aibrix_repo="/root/aibrix-local"
 export api_key="sk-kFJ12nKsFVfVmGpj3QzX65s4RbN2xJqWzPYCjYu7wT3BlbLi"
 export kube_context="ccr3aths9g2gqedu8asdg@35122069-kcu0n2lfb7pjdd83330h0"
 
@@ -23,11 +23,16 @@ do
         echo "started experiment at $(date)"
         echo autoscaler: ${autoscaler}
         echo workload: ${workload_path} 
-        echo "The stdout/stderr is being logged in ./output.txt"
-        ./run-test.sh ${workload_path} ${autoscaler} ${aibrix_repo} ${api_key} ${kube_context} &> output-${WORKLOAD_TYPE}.txt 
+        echo "The stdout/stderr is being logged in output-${WORKLOAD_TYPE}.txt"
+        ./run-test.sh ${workload_path} ${autoscaler} ${aibrix_repo} ${api_key} ${kube_context} ${WORKLOAD_TYPE} > output-${WORKLOAD_TYPE}.txt  2>&1
         end_time=$(date +%s)
         echo "Done: Time taken: $((end_time-start_time)) seconds"
         echo "--------------------------------"
         sleep 10
     done
-done
+done
+
+# for WORKLOAD_TYPE in "T_HighSlow_I_HighSlow_O_HighFast"  "T_HighSlow_I_HighSlow_O_HighSlow" "T_HighSlow_I_LowFast_O_HighSlow" "T_HighSlow_I_LowSlow_O_HighSlow"
+# do
+#     python plot-everything.py experiment_results/${WORKLOAD_TYPE}
+# done

benchmarks/utils/check_k8s_is_ready.py

@@ -58,6 +58,7 @@ def wait_for_all_podautoscaler_ready(namespace="default"):
 
 if __name__ == "__main__":
     target_deployment = sys.argv[1]
-    config.load_kube_config(context="ccr3aths9g2gqedu8asdg@41073177-kcu0mslcp5mhjsva38rpg")
+    kube_context = sys.argv[2]
+    config.load_kube_config(context=kube_context)
     wait_for_pods_ready(target_deployment)
     print("All pods are ready")

benchmarks/utils/count_num_pods.py

@@ -6,8 +6,8 @@
 import os
 import asyncio
 
-def get_pod_status_counts(deployment_name, namespace="default"):
-   config.load_kube_config(context="ccr3aths9g2gqedu8asdg@41073177-kcu0mslcp5mhjsva38rpg")
+def get_pod_status_counts(deployment_name, kube_context, namespace="default"):
+   config.load_kube_config(context=kube_context)
    v1 = client.CoreV1Api()
    pods = v1.list_namespaced_pod(namespace)
    filtered_pods = [pod for pod in pods.items if deployment_name in pod.metadata.name]
@@ -39,12 +39,13 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("deployment", help="Deployment name")
     parser.add_argument("output_dir", help="Output directory")
+    parser.add_argument("kube_context", help="Kube context")
     args = parser.parse_args()
 
     filename = f"{args.output_dir}/pod_count.csv"
     idx = 0
     while True:
-        status_counts = get_pod_status_counts(args.deployment)
+        status_counts = get_pod_status_counts(args.deployment, args.kube_context)
         write_to_csv(args.deployment, status_counts, filename, idx)
         time.sleep(1)
         idx += 1

benchmarks/utils/streaming_pod_log_to_file.py

@@ -57,12 +57,12 @@ def signal_handler(sig, frame):
     target_deployment = sys.argv[1]
     namespace = sys.argv[2]
     pod_log_dir = sys.argv[3]
-    include = sys.argv[4]
-    exclude = sys.argv[5]
-    if include == "none":
-        include = None
-    if exclude == "none":
-        exclude = None
+    include = None
+    exclude = None
+    if len(sys.argv) > 4:
+        include = sys.argv[4]
+    if len(sys.argv) > 5:
+        exclude = sys.argv[5]
 
     running_processes = []
     signal.signal(signal.SIGINT, signal_handler)