Adding scripts and updating git ignore to exclude results

lorainemg · lorainemg · commit 9c493c53f44f · 2021-11-03T16:27:24.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -127,3 +127,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# results
+plots/
+results/
diff --git a/dataset_plots.py b/dataset_plots.py
@@ -0,0 +1,107 @@
+import json
+from pathlib import Path
+from utils import get_plot_folder
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns
+
+
+def build_info(file_name, meta_features):
+    """Builds the info of a dataset in a presentable manner"""
+    info = {'dataset': int(file_name[:-5])}
+    for feat_name, value in meta_features.items():
+        new_name = feat_name.replace("_", " ")
+        info[new_name] = value
+    return info
+
+
+def extract_feature(task):
+    """Extracts a feature from an specific task info"""
+    info_folder = Path('results/95_datasets_info') / task
+    data_dict = []
+    for fn in info_folder.glob('*.json'):
+        info = json.load(open(fn, 'r+'))
+        data_dict.append(build_info(fn.name, info['meta_features']))
+    return pd.DataFrame(data_dict)
+
+
+def plot_numerical_values(data, prop_name):
+    """Plots numerical values in a kde format"""
+    #     sns.displot(data=data[prop_name], kind='kde')
+    plt.figure(prop_name)
+    sns.kdeplot(x=prop_name, data=data, shade=True)
+
+
+def plot_categorical_values(data, prop_name):
+    """Plots categorical values counting values"""
+    plt.figure(prop_name)
+    sns.countplot(x=prop_name, data=data, palette='rainbow')
+
+
+def plot_values(data, values, plot_folder, type_):
+    """Plots categorical or numeric values"""
+    for value in values:
+        if type_ == 'numerical':
+            plot_numerical_values(data, value)
+        elif type_ == 'categorical':
+            try:
+                plot_categorical_values(data, value)
+            except:
+                print(value)
+        plt.savefig(plot_folder / f'{value}.pdf')
+        plt.close()
+
+
+def plot_scatterplot(data, plot_folder):
+    """Scatter plot of #instances-#features, #instances-#classes (auto-sklearn style)"""
+    fig, axs = plt.subplots(nrows=2)
+    sns.scatterplot(data=data, x='input dimensionality', y='number of samples', ax=axs[0])
+    sns.scatterplot(data=data, x='input dimensionality', y='number of classes', ax=axs[1])
+    plt.savefig(plot_folder / f'scatterplot.pdf')
+    plt.close()
+
+
+def plot_lineplot(data, plot_folder):
+    """Line plot of #instances, #features, #classes - dataset id (atm style)"""
+    fig, axs = plt.subplots(nrows=3)
+    datasets_number = range(len(data))
+    sns.lineplot(data=data, x=datasets_number, y='number of samples', ax=axs[0])
+    sns.lineplot(data=data, x=datasets_number, y='input dimensionality', ax=axs[1])
+    sns.lineplot(data=data, x=datasets_number, y='number of classes', ax=axs[2])
+    plt.savefig(plot_folder / f'lineplot.pdf')
+    plt.close()
+
+
+def main():
+    """Does all main stuff"""
+    numerical_features = ['number of samples', 'input dimensionality',
+                          'output dimensionality', 'dataset dimensionality', 'standard deviation',
+                          'coefficient of variation', 'covariance avg', 'linear corr coef',
+                          'skewness', 'skewness 1', 'skewness 2', 'skewness 3', 'kurtosis',
+                          'kurtosis 1', 'kurtosis 2', 'kurtosis 3', 'normalized class entropy',
+                          'normalized attr entropy', 'normalized attr entropy 1',
+                          'normalized attr entropy 2', 'normalized attr entropy 3',
+                          'joint entropy', 'joint entropy 1', 'joint entropy 2',
+                          'joint entropy 3', 'mutual information', 'equivalent number of attr',
+                          'noise signal ratio']
+    categorical_features = ['is supervised', 'has numeric features',
+                            'average number of words', 'has text features', 'semantic input types',
+                            'semantic output types']
+
+    df = extract_feature('Classification')  # build dataframe with all datasets characteristics
+
+    plt_folder = get_plot_folder('plots/meta_features')
+
+    plot_scatterplot(df, plt_folder)
+    plot_lineplot(df, plt_folder)
+
+    # Plots numerical values
+    plot_values(df, numerical_features, plt_folder, 'numerical')
+
+    # Plots categorical values
+    plot_values(df, categorical_features, plt_folder, 'categorical')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/metalearner_plots.py b/metalearner_plots.py
@@ -0,0 +1,63 @@
+import json
+from pathlib import Path
+from typing import List
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns
+
+from utils import get_plot_folder
+
+
+def build_info(scores):
+    return [{'dataset': dataset, **dscores} for dataset, dscores in scores.items()]
+
+
+def extract_scores(strategy):
+    """Extracts a feature from an specific task info"""
+    info_folder = Path('results') / strategy / 'results'
+    data_dict = []
+    for fn in info_folder.glob('*.json'):
+        info = json.load(open(fn, 'r+'))
+        data_dict.append(build_info(info))
+    return [pd.DataFrame(data) for data in data_dict]
+
+
+def get_globals(data):
+    """Get the globals of different iterations"""
+    return pd.DataFrame([df.iloc[-1] for df in data])
+
+
+def plot_boxplot(data: pd.DataFrame, metric: str, fig_path: Path):
+    """Plots the data in a given metric and stores it in the figure path"""
+    plt.figure(metric)
+    sns.boxplot(data=data, y=metric)
+    plt.savefig(fig_path)
+    plt.close()
+
+
+def plot_results(strategies: List[str], metrics: List[str], plot_folder: Path):
+    """
+    Plots the results of a list of strategies by a given metrics
+    and results are stored in plot folder.
+    """
+    for strategy in strategies:
+        data = extract_scores(strategy)
+        plot_folder = get_plot_folder(plot_folder / strategy)
+        globl = get_globals(data)
+        for metric in metrics:
+            for i, df in enumerate(data, 1):
+                plot_boxplot(df, metric, plot_folder / f'{metric}_{i}.pdf')
+            plot_boxplot(globl, metric, plot_folder / f'global_{metric}.pdf')
+
+
+def main():
+    """Configures everything to save all plots"""
+    strategies = ['xgb_metalearner']    #, 'nn_metalearner']
+    metrics = ['srcc_score', 'wrc_score', 'dcg_score', 'ndcg_score']
+    plot_folder = get_plot_folder('plots/meta_learners')
+    plot_results(strategies, metrics, plot_folder)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/results_plots.py b/results_plots.py
@@ -0,0 +1,198 @@
+import seaborn as sns
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+from utils import get_plot_folder
+from functools import reduce
+from pathlib import Path
+import json
+import re
+
+
+def build_info(file_name, info, i):
+    data = {key: value for key, value in info.items() if not isinstance(value, list)}
+    # data['failed_pipelines'] = data['failed_pipelines'] / len(info['scores'])
+    data['dataset'] = file_name
+    data['i'] = i
+    if 'max_idx' not in data:
+        try:
+            max_idx = info['scores'].index(data['best_fn'])
+        except:
+            max_idx = None
+        data['max_idx'] = max_idx
+    return data
+
+def extract_scores(metalearner_path: Path):
+    "Extracts a feature from an especific task info"
+    data_dict = {}
+    file_re = re.compile('\w+_(\d+)_(\d+)\.json')
+    for fn in metalearner_path.glob('*.json'):
+        info = json.load(open(fn, 'r+'))
+        m = file_re.match(fn.name)
+        dataset_name = m.group(1)
+        iteration = int(m.group(2))
+
+        try:
+            data_dict[iteration].append(build_info(dataset_name, info, iteration))
+        except KeyError:
+            data_dict[iteration] = [build_info(dataset_name, info, iteration)]
+
+    return [pd.DataFrame(data) for data in data_dict.values()]
+
+
+def build_average_dataframe(dfs, metalearner):
+    datasets = {}
+    for df in dfs:
+        for _, row in df.iterrows():
+            dataset = row['dataset']
+            for column in df.columns:
+                if column == 'dataset' or column == 'i':
+                    continue
+                try:
+                    datasets[dataset][column].append(row[column])
+                except KeyError:
+                    try:
+                        datasets[dataset][column] = [row[column]]
+                    except KeyError:
+                        datasets[dataset] = {column: [row[column]]}
+    aggregate_datasets = []
+    for ds, values in datasets.items():
+        ds_dict = {prop: np.mean(list_v) for prop, list_v in values.items()}
+        aggregate_datasets.append({'dataset': ds, 'i': metalearner, **ds_dict})
+    return pd.DataFrame(aggregate_datasets)
+
+
+def plot_boxplot(data, prop_name, folder):
+    plt.figure(prop_name).suptitle(prop_name)
+    sns.boxplot(data=data, y=prop_name)
+    plt.savefig(folder)
+    plt.close()
+
+
+def plot_multiple_boxplot(data, prop_name, folder):
+    plt.figure(prop_name).suptitle(prop_name)
+    g = sns.boxplot(data=data, x='i', y=prop_name, hue='i', dodge=False)
+    plt.legend(title='Estrategias', loc='best')
+    g.set(xticklabels=[])
+    g.set(xlabel=None)
+    plt.savefig(folder)
+    plt.close()
+
+
+def plot_histogram(data, metalearners, folder):
+    autogoal = metalearners[0]
+    fig, axs = plt.subplots(nrows=len(metalearners)-1)
+    for i, metalearner in enumerate(metalearners[1:]):
+        df = data[(data['i'] == autogoal) | (data['i'] == metalearner)]
+        axs[i].set_xlabel('Accuracy Obtenido')
+        axs[i].set_ylabel('Datasets')
+        sns.histplot(data=df, x='best_fn', hue='i', bins=20, ax=axs[i])
+        axs[i].legend(title='Estrategias', loc='best', labels=[autogoal, metalearner])
+    plt.savefig(folder / 'histogram')
+    plt.close()
+
+
+def plot_results(metalearners, metalearners_path: Path, plot_folder: Path):
+    avg_results = []
+    for j, metalearner in enumerate(metalearners):
+        metalearner_folder = get_plot_folder(plot_folder / metalearner)
+        data = extract_scores(metalearners_path / metalearner)
+        avg = build_average_dataframe(data, metalearner)
+        data.append(avg)
+        avg_results.append(avg)
+        for i, df in enumerate(data):
+            for column in df.columns:
+                if column in ['i', 'dataset']:
+                    continue
+                plot_boxplot(df, column, metalearner_folder / f'{column}_{i}')
+    df = pd.concat(avg_results)
+    df.loc[df['best_fn'] < 0, 'best_fn'] = 0
+    for column in df.columns:
+        if column in ['i', 'dataset']:
+            continue
+        plot_multiple_boxplot(df, column, plot_folder / f'{column}')
+
+    plot_histogram(df, metalearners, plot_folder)
+
+    dfs = build_performance_info(metalearners, metalearners_path)
+    plotting_performance(dfs, plot_folder / 'performance')
+
+
+def fix_performance_info(performance: list):
+    new_performance = [performance[0]]
+    for p in performance:
+        if p > new_performance[-1]:
+            new_performance.append(p)
+        else:
+            new_performance.append(new_performance[-1])
+    return new_performance
+
+
+def build_performance_info(metalearners, metalearners_path: Path):
+    dataframes = {}
+    for metalearner in metalearners:
+        data_dict = {'i': [], metalearner: []}
+        metalearner_path = metalearners_path / metalearner
+        for fn in metalearner_path.glob('*.json'):
+            info = json.load(open(fn, 'r+'))
+
+            # performance = info['scores']
+            #
+            performance = [p if p > 0 else 0 for p in info['scores']]
+            # performance = [p if p > 0 else 0 for p in info['scores']]
+            if len(performance) == 0:
+                continue
+            performance = fix_performance_info(performance)
+            data_dict[metalearner].extend(performance)
+            data_dict['i'].extend(range(len(performance)))
+        dataframes[metalearner] = pd.DataFrame(data_dict)
+    return dataframes
+
+
+def plotting_performance(data, folder):
+    fig = plt.figure()
+    for metalearner, df in data.items():
+        sns.lineplot(data=df, x='i', y=metalearner)
+    # fig.legend(title='Estrategias', labels=data.keys())
+    plt.legend(loc='best', title='Estrategias', labels=data.keys())
+    plt.ylabel(None)
+    plt.xlabel('Iteraciones')
+    plt.xlim([0, 200])
+    plt.savefig(folder)
+    plt.close()
+
+
+def main():
+    # plot_folder = get_plot_folder('plots/results/l1 distance')
+    # metalearners = ['autogoal', 'nn_learner_aggregated', 'nn_learner_simple', 'xgb_metalearner']
+    # plot_results(metalearners, Path('results/l1 distance/results'), plot_folder)
+
+    # plot_folder = get_plot_folder('plots/results/l2 distance')
+    # metalearners = ['autogoal', 'nn_learner_aggregated', 'nn_metalearner_simple', 'xgb_metalearner']
+    # plot_results(metalearners, Path('results/l2 distance/results'), plot_folder)
+
+    # plot_folder = get_plot_folder('plots/results/xgb_metalearner_v2')
+    # metalearners = ['autogoal', 'xgb_metalearner_v2',
+    #                 'nn_learner_aggregated_l1', 'nn_learner_simple_l1', 'xgb_metalearner_l1',
+    #                 'nn_learner_aggregated_l2', 'nn_metalearner_simple_l2', 'xgb_metalearner_l2']
+    # plot_results(metalearners, Path('results/xgb_metalearner_v2/results'), plot_folder)
+
+    plot_folder = get_plot_folder('plots/results/paper')
+    metalearners = ['Autogoal', 'Vecinos Cercanos Simple',
+                    'Vecinos Cercanos Ponderado', 'XGBRanker']
+    plot_results(metalearners, Path('results/paper/results'), plot_folder)
+
+    # plot_folder = get_plot_folder('plots/results/new_paper')
+    # metalearners = ['Autogoal', 'Vecinos Cercanos Simple',
+    #                 'Vecinos Cercanos Ponderado', 'XGBRanker']
+    # plot_results(metalearners, Path('results/new paperr/results'), plot_folder)
+
+    # plot_folder = get_plot_folder('plots/results/new_paper_performance')
+    # metalearners = ['Autogoal', 'Vecinos Cercanos Simple',
+    #                 'Vecinos Cercanos Ponderado', 'XGBRanker']
+    # plot_performance(metalearners, Path('results/new paperr/results'), plot_folder)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/utils.py b/utils.py
@@ -0,0 +1,9 @@
+from pathlib import Path
+
+
+def get_plot_folder(folder_path: str):
+    """Creates a folder for plots, creating also parents directories if necessary"""
+    folder = Path(folder_path)
+    if not folder.exists():
+        folder.mkdir(parents=True)
+    return folder
