From 0c003ee3a7081ed6aa8c4e242d42cbb762fc4483 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Sat, 19 Oct 2019 22:21:44 +0200 Subject: [PATCH 1/4] Making example faster and adding unit test for it --- .../40_paper/2018_neurips_perrone_example.py | 47 ++++------- .../test_2018_neurips_perrone.py | 82 +++++++++++++++++++ 2 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 tests/test_examples/test_2018_neurips_perrone.py diff --git a/examples/40_paper/2018_neurips_perrone_example.py b/examples/40_paper/2018_neurips_perrone_example.py index 6c8207153..5513fab30 100644 --- a/examples/40_paper/2018_neurips_perrone_example.py +++ b/examples/40_paper/2018_neurips_perrone_example.py @@ -39,7 +39,7 @@ ############################################################################ # The subsequent functions are defined to fetch tasks, flows, evaluations and preprocess them into # a tabular format that can be used to build models. -# + def fetch_evaluations(run_full=False, flow_type='svm', @@ -79,25 +79,25 @@ def fetch_evaluations(run_full=False, 3492, 3493, 37, 3896, 3903, 3913, 3917, 3918, 3, 49, 9914, 9946, 9952, 9967, ] - else: #flow_type == 'xgboost' and not run_full: + else: # flow_type == 'xgboost' and not run_full: task_ids = [3903, 37, 3485, 49, 3913] # Fetching the relevant flow flow_id = 5891 if flow_type == 'svm' else 6767 # Fetching evaluations - eval_df = openml.evaluations.list_evaluations(function=metric, - task=task_ids, - flow=[flow_id], - uploader=[2702], - output_format='dataframe') + eval_df = openml.evaluations.list_evaluations_setups(function=metric, + task=task_ids, + flow=[flow_id], + uploader=[2702], + output_format='dataframe', + parameters_in_separate_columns=True) return eval_df, task_ids, flow_id def create_table_from_evaluations(eval_df, flow_type='svm', run_count=np.iinfo(np.int64).max, - metric = 'area_under_roc_curve', task_ids=None): ''' Create a tabular data with its ground truth from a dataframe of evaluations. @@ -111,8 +111,6 @@ def create_table_from_evaluations(eval_df, To select whether svm or xgboost experiments are to be run run_count : int Maximum size of the table created, or number of runs included in the table - metric : str - The evaluation measure that is passed to openml.evaluations.list_evaluations task_ids : list, (optional) List of integers specifying the tasks to be retained from the evaluations dataframe @@ -132,18 +130,11 @@ def create_table_from_evaluations(eval_df, 'subsample', ] eval_df = eval_df.sample(frac=1) # shuffling rows - run_ids = eval_df["run_id"][:run_count] - eval_table = pd.DataFrame(np.nan, index=run_ids, columns=colnames) - values = [] - runs = openml.runs.get_runs(run_ids) - for r in runs: - params = r.parameter_settings - for p in params: - name, value = p['oml:name'], p['oml:value'] - if name in colnames: - eval_table.loc[r.run_id, name] = value - values.append(r.evaluations[metric]) - return eval_table, values + eval_df = eval_df.iloc[:run_count, :] + eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] + eval_table = eval_df.loc[:, colnames] + value = eval_df.loc[:, 'value'] + return eval_table, value def list_categorical_attributes(flow_type='svm'): @@ -160,9 +151,7 @@ def list_categorical_attributes(flow_type='svm'): # pre-processing all retrieved evaluations. eval_df, task_ids, flow_id = fetch_evaluations(run_full=False, flow_type=flow_type) -# run_count can not be passed if all the results are required -# it is set to 500 here arbitrarily to get results quickly -X, y = create_table_from_evaluations(eval_df, run_count=500, flow_type=flow_type) +X, y = create_table_from_evaluations(eval_df, flow_type=flow_type) print(X.head()) print("Y : ", y[:5]) @@ -176,8 +165,6 @@ def list_categorical_attributes(flow_type='svm'): # Separating data into categorical and non-categorical (numeric for this example) columns cat_cols = list_categorical_attributes(flow_type=flow_type) num_cols = list(set(X.columns) - set(cat_cols)) -X_cat = X.loc[:, cat_cols] -X_num = X.loc[:, num_cols] # Missing value imputers cat_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='None') @@ -187,7 +174,7 @@ def list_categorical_attributes(flow_type='svm'): enc = OneHotEncoder(handle_unknown='ignore') # Pipeline to handle categorical column transformations -cat_transforms = Pipeline([('impute', cat_imputer), ('encode', enc)]) +cat_transforms = Pipeline(steps=[('impute', cat_imputer), ('encode', enc)]) # Combining column transformers ct = ColumnTransformer([('cat', cat_transforms, cat_cols), ('num', num_imputer, num_cols)]) @@ -207,7 +194,7 @@ def list_categorical_attributes(flow_type='svm'): # Selecting a task for the surrogate task_id = task_ids[-1] print("Task ID : ", task_id) -X, y = create_table_from_evaluations(eval_df, run_count=1000, task_ids=[task_id], flow_type='svm') +X, y = create_table_from_evaluations(eval_df, task_ids=[task_id], flow_type='svm') model.fit(X, y) y_pred = model.predict(X) @@ -224,6 +211,7 @@ def list_categorical_attributes(flow_type='svm'): # # NOTE: This section is written exclusively for the SVM flow + # Sampling random configurations def random_sample_configurations(num_samples=100): colnames = ['cost', 'degree', 'gamma', 'kernel'] @@ -240,6 +228,7 @@ def random_sample_configurations(num_samples=100): X.iloc[:, i] = col_val return X + configs = random_sample_configurations(num_samples=1000) print(configs) diff --git a/tests/test_examples/test_2018_neurips_perrone.py b/tests/test_examples/test_2018_neurips_perrone.py new file mode 100644 index 000000000..6a0999a24 --- /dev/null +++ b/tests/test_examples/test_2018_neurips_perrone.py @@ -0,0 +1,82 @@ +import openml +from openml.testing import TestBase + +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt +from sklearn.pipeline import Pipeline +from sklearn.impute import SimpleImputer +from sklearn.compose import ColumnTransformer +from sklearn.metrics import mean_squared_error +from sklearn.preprocessing import OneHotEncoder +from sklearn.ensemble import RandomForestRegressor + + +class TestPerroneExample(TestBase): + + def test_perrone_example(self): + metric = 'area_under_roc_curve' + task_ids = [9983, 3485, 3902, 3903, 145878] + flow_id = 5891 + colnames = ['cost', 'degree', 'gamma', 'kernel'] + eval_df = openml.evaluations.list_evaluations_setups(function=metric, + task=task_ids, + flow=[flow_id], + uploader=[2702], + output_format='dataframe', + parameters_in_separate_columns=True) + self.assertEqual(eval_df.shape[1], 21) + self.assertGreaterEqual(eval_df.shape[0], 5000) + + eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] + self.assertTrue(all(np.isin(colnames, eval_df.columns))) + self.assertTrue(all(np.isin(task_ids, eval_df['task_id']))) + + eval_df = eval_df.sample(frac=1) # shuffling rows + eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] + X = eval_df.loc[:, colnames] + y = eval_df.loc[:, 'value'] + cat_cols = ['kernel'] + num_cols = ['cost', 'degree', 'gamma'] + # Missing value imputers + cat_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='None') + num_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=-1) + # Creating the one-hot encoder + enc = OneHotEncoder(handle_unknown='ignore') + # Pipeline to handle categorical column transformations + cat_transforms = Pipeline(steps=[('impute', cat_imputer), ('encode', enc)]) + # Combining column transformers + ct = ColumnTransformer([('cat', cat_transforms, cat_cols), ('num', num_imputer, num_cols)]) + # Creating the full pipeline with the surrogate model + clf = RandomForestRegressor(n_estimators=50) + model = Pipeline(steps=[('preprocess', ct), ('surrogate', clf)]) + model.fit(X, y) + y_pred = model.predict(X) + mean_squared_error(y, y_pred) + + def random_sample_configurations(num_samples=100): + colnames = ['cost', 'degree', 'gamma', 'kernel'] + ranges = [(0.000986, 998.492437), + (2.0, 5.0), + (0.000988, 913.373845), + (['linear', 'polynomial', 'radial', 'sigmoid'])] + X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames) + for i in range(len(colnames)): + if len(ranges[i]) == 2: + col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], + size=num_samples) + else: + col_val = np.random.choice(ranges[i], size=num_samples) + X.iloc[:, i] = col_val + return X + configs = random_sample_configurations(num_samples=1000) + preds = model.predict(configs) + # tracking the maximum AUC obtained over the functions evaluations + preds = np.maximum.accumulate(preds) + # computing regret (1 - predicted_auc) + regret = 1 - preds + # plotting the regret curve + plt.plot(regret) + plt.title('AUC regret for Random Search on surrogate') + plt.xlabel('Numbe of function evaluations') + plt.ylabel('Regret') From 83c428ad0815435bc7d87d75d4491d0c4cccce73 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Sat, 19 Oct 2019 23:00:55 +0200 Subject: [PATCH 2/4] Fixing server for the unit test --- tests/test_examples/test_2018_neurips_perrone.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_examples/test_2018_neurips_perrone.py b/tests/test_examples/test_2018_neurips_perrone.py index 6a0999a24..2d1679367 100644 --- a/tests/test_examples/test_2018_neurips_perrone.py +++ b/tests/test_examples/test_2018_neurips_perrone.py @@ -15,6 +15,8 @@ class TestPerroneExample(TestBase): def test_perrone_example(self): + openml.config.server = self.production_server + metric = 'area_under_roc_curve' task_ids = [9983, 3485, 3902, 3903, 145878] flow_id = 5891 @@ -25,12 +27,12 @@ def test_perrone_example(self): uploader=[2702], output_format='dataframe', parameters_in_separate_columns=True) + self.assertTrue(all(np.isin(task_ids, eval_df['task_id']))) self.assertEqual(eval_df.shape[1], 21) self.assertGreaterEqual(eval_df.shape[0], 5000) eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] self.assertTrue(all(np.isin(colnames, eval_df.columns))) - self.assertTrue(all(np.isin(task_ids, eval_df['task_id']))) eval_df = eval_df.sample(frac=1) # shuffling rows eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] @@ -80,3 +82,5 @@ def random_sample_configurations(num_samples=100): plt.title('AUC regret for Random Search on surrogate') plt.xlabel('Numbe of function evaluations') plt.ylabel('Regret') + + openml.config.server = self.test_server From 86e6c233a141e1b4acda76bf2ef0438be2124a1e Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Sun, 20 Oct 2019 00:14:36 +0200 Subject: [PATCH 3/4] Fixing sklearn version issues in unit test --- tests/test_examples/test_2018_neurips_perrone.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_examples/test_2018_neurips_perrone.py b/tests/test_examples/test_2018_neurips_perrone.py index 2d1679367..c0d5597aa 100644 --- a/tests/test_examples/test_2018_neurips_perrone.py +++ b/tests/test_examples/test_2018_neurips_perrone.py @@ -1,12 +1,15 @@ import openml from openml.testing import TestBase +import unittest +from distutils.version import LooseVersion + import numpy as np import pandas as pd from matplotlib import pyplot as plt + +import sklearn from sklearn.pipeline import Pipeline -from sklearn.impute import SimpleImputer -from sklearn.compose import ColumnTransformer from sklearn.metrics import mean_squared_error from sklearn.preprocessing import OneHotEncoder from sklearn.ensemble import RandomForestRegressor @@ -14,6 +17,9 @@ class TestPerroneExample(TestBase): + @unittest.skipIf(LooseVersion(sklearn.__version__) < "0.20", reason="SimpleImputer and " + "ColumnTransformer doesn't" + "exist in older versions.") def test_perrone_example(self): openml.config.server = self.production_server @@ -41,6 +47,7 @@ def test_perrone_example(self): cat_cols = ['kernel'] num_cols = ['cost', 'degree', 'gamma'] # Missing value imputers + from sklearn.impute import SimpleImputer cat_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='None') num_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=-1) # Creating the one-hot encoder @@ -48,6 +55,7 @@ def test_perrone_example(self): # Pipeline to handle categorical column transformations cat_transforms = Pipeline(steps=[('impute', cat_imputer), ('encode', enc)]) # Combining column transformers + from sklearn.compose import ColumnTransformer ct = ColumnTransformer([('cat', cat_transforms, cat_cols), ('num', num_imputer, num_cols)]) # Creating the full pipeline with the surrogate model clf = RandomForestRegressor(n_estimators=50) From dc1ce3370f59977f027707473dc5c7a32f2ff1a6 Mon Sep 17 00:00:00 2001 From: neeratyoy Date: Wed, 23 Oct 2019 14:05:09 +0200 Subject: [PATCH 4/4] Removing redundant unit test --- .../test_2018_neurips_perrone.py | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 tests/test_examples/test_2018_neurips_perrone.py diff --git a/tests/test_examples/test_2018_neurips_perrone.py b/tests/test_examples/test_2018_neurips_perrone.py deleted file mode 100644 index c0d5597aa..000000000 --- a/tests/test_examples/test_2018_neurips_perrone.py +++ /dev/null @@ -1,94 +0,0 @@ -import openml -from openml.testing import TestBase - -import unittest -from distutils.version import LooseVersion - -import numpy as np -import pandas as pd -from matplotlib import pyplot as plt - -import sklearn -from sklearn.pipeline import Pipeline -from sklearn.metrics import mean_squared_error -from sklearn.preprocessing import OneHotEncoder -from sklearn.ensemble import RandomForestRegressor - - -class TestPerroneExample(TestBase): - - @unittest.skipIf(LooseVersion(sklearn.__version__) < "0.20", reason="SimpleImputer and " - "ColumnTransformer doesn't" - "exist in older versions.") - def test_perrone_example(self): - openml.config.server = self.production_server - - metric = 'area_under_roc_curve' - task_ids = [9983, 3485, 3902, 3903, 145878] - flow_id = 5891 - colnames = ['cost', 'degree', 'gamma', 'kernel'] - eval_df = openml.evaluations.list_evaluations_setups(function=metric, - task=task_ids, - flow=[flow_id], - uploader=[2702], - output_format='dataframe', - parameters_in_separate_columns=True) - self.assertTrue(all(np.isin(task_ids, eval_df['task_id']))) - self.assertEqual(eval_df.shape[1], 21) - self.assertGreaterEqual(eval_df.shape[0], 5000) - - eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] - self.assertTrue(all(np.isin(colnames, eval_df.columns))) - - eval_df = eval_df.sample(frac=1) # shuffling rows - eval_df.columns = [column.split('_')[-1] for column in eval_df.columns] - X = eval_df.loc[:, colnames] - y = eval_df.loc[:, 'value'] - cat_cols = ['kernel'] - num_cols = ['cost', 'degree', 'gamma'] - # Missing value imputers - from sklearn.impute import SimpleImputer - cat_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value='None') - num_imputer = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=-1) - # Creating the one-hot encoder - enc = OneHotEncoder(handle_unknown='ignore') - # Pipeline to handle categorical column transformations - cat_transforms = Pipeline(steps=[('impute', cat_imputer), ('encode', enc)]) - # Combining column transformers - from sklearn.compose import ColumnTransformer - ct = ColumnTransformer([('cat', cat_transforms, cat_cols), ('num', num_imputer, num_cols)]) - # Creating the full pipeline with the surrogate model - clf = RandomForestRegressor(n_estimators=50) - model = Pipeline(steps=[('preprocess', ct), ('surrogate', clf)]) - model.fit(X, y) - y_pred = model.predict(X) - mean_squared_error(y, y_pred) - - def random_sample_configurations(num_samples=100): - colnames = ['cost', 'degree', 'gamma', 'kernel'] - ranges = [(0.000986, 998.492437), - (2.0, 5.0), - (0.000988, 913.373845), - (['linear', 'polynomial', 'radial', 'sigmoid'])] - X = pd.DataFrame(np.nan, index=range(num_samples), columns=colnames) - for i in range(len(colnames)): - if len(ranges[i]) == 2: - col_val = np.random.uniform(low=ranges[i][0], high=ranges[i][1], - size=num_samples) - else: - col_val = np.random.choice(ranges[i], size=num_samples) - X.iloc[:, i] = col_val - return X - configs = random_sample_configurations(num_samples=1000) - preds = model.predict(configs) - # tracking the maximum AUC obtained over the functions evaluations - preds = np.maximum.accumulate(preds) - # computing regret (1 - predicted_auc) - regret = 1 - preds - # plotting the regret curve - plt.plot(regret) - plt.title('AUC regret for Random Search on surrogate') - plt.xlabel('Numbe of function evaluations') - plt.ylabel('Regret') - - openml.config.server = self.test_server