Adding retrieval time ram

Author: rcap107

prepare_results.py

@@ -23,8 +23,19 @@ def fix_duplicate_runs(df):
 df = pl.read_parquet("results/master_list.parquet")
 df = fix_duplicate_runs(df)
 df = df.filter(~pl.col("estimator").is_in(["nojoin", "top_k_full_join"]))
-#%%
-df = df.with_columns(base_table=pl.col("base_table").str.split("-").list.first())
+
+df_ram = pl.read_csv("stats/dummy_peak_ram.csv")
+df_query_time_retrieval = pl.read_csv(
+    "stats/avg_query_time_for_pareto_plot_retrieval.csv"
+)
+df_query_time_all_datalakes = pl.read_csv(
+    "stats/avg_query_time_for_pareto_plot_all_datalakes.csv"
+)
+
+# %%
+df = df.with_columns(base_table=pl.col("base_table").str.split("-").list.first()).join(
+    df_ram, on="jd_method"
+).with_columns(peak_ram=pl.max_horizontal("peak_ram", "peak_fit", "peak_predict", "peak_test"))
 # %%
 # General configuration (all data lakes, no Starmie)
 config_general = json.load(
@@ -33,6 +44,9 @@ def fix_duplicate_runs(df):
 df_config = prepare_config(config_general)
 group_keys = df_config.columns
 df_test = df_config.join(df, on=group_keys, how="inner")
+df_test.join(df_query_time_all_datalakes, on="jd_method").with_columns(
+    total_runtime=pl.col("time_run") + pl.col("time_query")
+)
 df_test.write_parquet("results/results_general.parquet")
 # %%
 # Retrieval method configuration (Starmie, no 50k/open data)
@@ -42,6 +56,10 @@ def fix_duplicate_runs(df):
 df_config = prepare_config(config_general)
 group_keys = df_config.columns
 df_test = df_config.join(df, on=group_keys, how="inner")
+df_test = df_test.join(df_query_time_retrieval, on="jd_method").with_columns(
+    total_runtime=pl.col("time_run") + pl.col("time_query")
+)
+
 df_test.write_parquet("results/results_retrieval.parquet")
 
 # %%
@@ -52,6 +70,10 @@ def fix_duplicate_runs(df):
 df_config = prepare_config(config_general)
 group_keys = df_config.columns
 df_test = df_config.join(df, on=group_keys, how="inner")
+df_test = df_test.join(df_query_time_all_datalakes, on="jd_method").with_columns(
+    total_runtime=pl.col("time_run") + pl.col("time_query")
+)
+
 df_test.write_parquet("results/results_aggregation.parquet")
 
 # %%