From 2bb557bd209a0fa25df055b1f056081e96d4318b Mon Sep 17 00:00:00 2001 From: Anatoly Volkov Date: Fri, 21 Aug 2026 06:10:04 -0700 Subject: [PATCH 1/4] Initial commit --- configs/regular/xgboost_binary.json | 72 +++++++++-- configs/regular/xgboost_multi.json | 40 +------ configs/regular/xgboost_regression.json | 38 ++++++ sklbench/datasets/common.py | 7 +- sklbench/datasets/loaders.py | 152 ++++++++++++++++++++++++ sklbench/datasets/transformer.py | 34 +++++- 6 files changed, 291 insertions(+), 52 deletions(-) diff --git a/configs/regular/xgboost_binary.json b/configs/regular/xgboost_binary.json index fa86cd00..b9ea4faa 100644 --- a/configs/regular/xgboost_binary.json +++ b/configs/regular/xgboost_binary.json @@ -6,11 +6,11 @@ "data": { "dataset": "airline_depdelay", "preprocessing_kwargs": { - "category_encoding": ["onehot", "ordinal"], - "subsample": 600000 + "category_encoding": ["ordinal"], + "subsample": 700000 }, "split_kwargs": { - "train_size": 100000, + "train_size": 200000, "test_size": 500000 } }, @@ -28,7 +28,7 @@ "data": { "dataset": "hepmass", "split_kwargs": { - "train_size": 200000, + "train_size": 400000, "test_size": 1000000 } }, @@ -38,25 +38,71 @@ "reg_alpha": 1.0, "reg_lambda": 1.0, "max_leaves": 256, - "n_estimators": 500 + "n_estimators": 200 } } }, { "data": { - "dataset": "gisette", + "dataset": "airline_satisfaction", + "preprocessing_kwargs": { + "category_encoding": ["ignore", "ordinal"] + }, "split_kwargs": { - "train_size": 2000, - "test_size": 5000 + "test_size": 0.2, + "random_state": 42 } }, "algorithm": { "estimator_params": { - "learning_rate": 0.15, - "max_leaves": 256, - "colsample_bytree": 0.1, - "colsample_bynode": 0.1, - "n_estimators": 100 + "learning_rate": 0.1, + "max_depth": 6, + "reg_lambda": 1.0, + "n_estimators": 1000, + "enable_categorical": true + } + } + }, + { + "data": { + "dataset": "apsfailure", + "preprocessing_kwargs": { + "replace_nan": ["ignore", "mean"] + }, + "split_kwargs": { + "test_size": 0.2, + "random_state": 42 + } + }, + "algorithm": { + "estimator_params": { + "learning_rate": 0.05, + "max_depth": 6, + "subsample": 0.8, + "colsample_bytree": 0.8, + "reg_lambda": 1.0, + "n_estimators": 200 + } + } + }, + { + "data": { + "dataset": "amazon_employee_access", + "preprocessing_kwargs": { + "category_encoding": ["ignore", "ordinal"] + }, + "split_kwargs": { + "test_size": 0.2, + "random_state": 42 + } + }, + "algorithm": { + "estimator_params": { + "learning_rate": 0.05, + "max_depth": 6, + "reg_lambda": 1.0, + "n_estimators": 500, + "enable_categorical": true } } } diff --git a/configs/regular/xgboost_multi.json b/configs/regular/xgboost_multi.json index d51c1be7..4a0611f5 100644 --- a/configs/regular/xgboost_multi.json +++ b/configs/regular/xgboost_multi.json @@ -2,50 +2,16 @@ "INCLUDE": ["../common/xgboost.json"], "PARAMETERS_SETS": { "gbt multi classification data": [ - { - "data": { - "dataset": "letters", - "split_kwargs": { - "train_size": 0.5, - "test_size": 0.5 - } - }, - "algorithm": { - "estimator_params": { - "learning_rate": 0.05, - "reg_lambda": 1, - "max_depth": 6, - "n_estimators": 200 - } - } - }, - { - "data": { - "dataset": "mnist", - "split_kwargs": { - "train_size": 20000, - "test_size": 50000 - } - }, - "algorithm": { - "estimator_params": { - "max_leaves": 256, - "colsample_bytree": 0.2, - "colsample_bynode": 0.5, - "n_estimators": 100 - } - } - }, { "data": { "dataset": "covtype" }, "algorithm": { "estimator_params": { - "learning_rate": 0.1, + "learning_rate": 0.3, "reg_lambda": 1, - "max_depth": 8, - "n_estimators": 200 + "max_depth": 14, + "n_estimators": 100 } } }, diff --git a/configs/regular/xgboost_regression.json b/configs/regular/xgboost_regression.json index 01046df0..4d0d4125 100644 --- a/configs/regular/xgboost_regression.json +++ b/configs/regular/xgboost_regression.json @@ -89,6 +89,44 @@ "n_estimators": 100 } } + }, + { + "data": { + "dataset": "superconductivity", + "split_kwargs": { + "test_size": 0.2, + "random_state": 42 + } + }, + "algorithm": { + "estimator_params": { + "learning_rate": 0.02, + "max_depth": 7, + "subsample": 0.8, + "colsample_bytree": 0.8, + "reg_lambda": 1.0, + "n_estimators": 1500 + } + } + }, + { + "data": { + "dataset": "qsar_tid_11", + "split_kwargs": { + "test_size": 0.2, + "random_state": 42 + } + }, + "algorithm": { + "estimator_params": { + "learning_rate": 0.05, + "max_depth": 6, + "subsample": 0.8, + "colsample_bytree": 0.8, + "reg_lambda": 1.0, + "n_estimators": 1000 + } + } } ] }, diff --git a/sklbench/datasets/common.py b/sklbench/datasets/common.py index 73b2fa6d..26631bc8 100644 --- a/sklbench/datasets/common.py +++ b/sklbench/datasets/common.py @@ -59,7 +59,12 @@ def get_filenames_by_prefix(directory: str, prefix: str) -> List[str]: def load_data_file(filepath, extension): if extension == "parq": - data = pd.read_parquet(filepath) + # Read with the same engine used for writing (fastparquet). Reading a + # fastparquet-written file with the default "auto"/pyarrow engine drops + # pandas "category" dtype (string categories come back as object), + # which would prevent XGBoost's native categorical handling from ever + # seeing category columns loaded from cache. + data = pd.read_parquet(filepath, engine="fastparquet") elif extension.endswith("npz"): npz_content = np.load(filepath) if extension == "npz": diff --git a/sklbench/datasets/loaders.py b/sklbench/datasets/loaders.py index eb3e3b6b..769f1efe 100644 --- a/sklbench/datasets/loaders.py +++ b/sklbench/datasets/loaders.py @@ -476,6 +476,110 @@ def load_fraud( return {"x": x, "y": y}, data_desc +@cache +def load_kddcup09_appetency( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + KDD Cup 2009: Customer relationship prediction (appetency task). + https://kdd.org/kdd-cup/view/kdd-cup-2009/Results + + The data comes from the French telecom company Orange and is used to + predict the propensity of customers to buy new products or services + ("appetency"). It is a hard, real-world marketing dataset whose key + characteristics are a large number of missing values (every instance + has at least one) and heavily unbalanced classes (the positive class + is roughly 1.8% of samples). Feature names and categorical values are + anonymized and carry no semantic meaning. + + Source: OpenML dataset id 46939 (kddcup09_appetency, small training set). + + Classification task. n_classes = 2. + """ + x, y = load_openml(46939, raw_data_cache) + data_desc = {"n_classes": 2, "default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + +@cache +def load_amazon_employee_access( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + Amazon employee access dataset. + https://www.openml.org/d/46905 + + Binary classification (predict whether a resource request is approved) from + 9 purely categorical, high-cardinality features (up to ~7500 distinct values + each) with no numeric features and no missing values. Heavily imbalanced + (~94% positive), so ROC AUC is the meaningful metric. + + Loaded with as_frame=True so the categorical columns are preserved as pandas + "category" dtype -- this lets the dataset exercise XGBoost's native + categorical handling (enable_categorical) when preprocessing does not + ordinal-encode the features. + + Source: OpenML dataset id 46905 (Amazon_employee_access, curated by TabArena). + + Classification task. n_classes = 2. + """ + x, y = load_openml(46905, raw_data_cache, as_frame=True) + data_desc = {"n_classes": 2, "default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + +@cache +def load_apsfailure( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + APSFailure dataset (Scania trucks Air Pressure System failure). + https://www.openml.org/d/46908 + + Binary classification from the UCI IDA 2016 Industrial Challenge: predict + whether a truck component failure is related to the APS, from 170 numeric + sensor / histogram features. The dataset has many missing values -- ~99% + of rows have at least one -- which gradient boosting handles natively, and + heavily unbalanced classes (~1.8% positive). + + Source: OpenML dataset id 46908 (APSFailure, curated by TabArena). + + Classification task. n_classes = 2. + """ + x, y = load_openml(46908, raw_data_cache) + data_desc = {"n_classes": 2, "default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + +@cache +def load_airline_satisfaction( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + Airline passenger satisfaction dataset. + https://www.openml.org/d/46920 + + Binary classification: predict whether an airline passenger is satisfied + from flight and service-rating features. The dataset has a genuine mix of + 16 categorical columns (customer/travel type, cabin class, service ratings) + and 5 numerical columns (age, flight distance, boarding, delays), and with + ~130k samples the optimal gradient-boosting model uses close to a thousand + trees. + + Source: OpenML dataset id 46920 (customer_satisfaction_in_airline, + curated by TabArena). + + Loaded with as_frame=True so the categorical columns keep their pandas + "category" dtype, allowing the native categorical path (enable_categorical) + when preprocessing does not ordinal-encode them. + + Classification task. n_classes = 2. + """ + x, y = load_openml(46920, raw_data_cache, as_frame=True) + data_desc = {"n_classes": 2, "default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + @cache def load_ijcnn( data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict @@ -809,6 +913,48 @@ def load_yolanda( return {"x": x, "y": y}, data_desc +@cache +def load_superconductivity( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + Superconductivity dataset from the UCI machine learning repository. + https://archive.ics.uci.edu/dataset/464/superconductivty+data + + Regression task: predict the critical temperature of a superconductor from + 81 features extracted from its chemical composition. Unlike many tabular + datasets, the target has enough signal that gradient boosting keeps + improving well past a thousand trees, making it a good stress test for + XGBoost regression with a large number of estimators. + + Source: OpenML dataset id 46961 (superconductivity, curated by TabArena). + """ + x, y = load_openml(46961, raw_data_cache) + data_desc = {"default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + +@cache +def load_qsar_tid_11( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + QSAR-TID-11 dataset from ChEMBL. + https://www.openml.org/d/46953 + + Regression task: predict a compound's median pXC50 bioactivity + (MEDIAN_PXC50) against a drug target from 1024 numeric molecular + fingerprint / descriptor features. This is a high-dimensional, + low-sample dataset (5742 samples x 1024 features) that rewards a + large number of gradient-boosting estimators. + + Source: OpenML dataset id 46953 (QSAR-TID-11, curated by TabArena). + """ + x, y = load_openml(46953, raw_data_cache) + data_desc = {"default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + @cache def load_road_network( data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict @@ -884,6 +1030,10 @@ def load_gist( "higgs": load_higgs, "susy": load_susy, "ijcnn": load_ijcnn, + "amazon_employee_access": load_amazon_employee_access, + "apsfailure": load_apsfailure, + "airline_satisfaction": load_airline_satisfaction, + "kddcup09_appetency": load_kddcup09_appetency, "klaverjas": load_klaverjas, "cifar": load_cifar, "connect": load_connect, @@ -904,6 +1054,8 @@ def load_gist( "twodplanes": load_twodplanes, "year_prediction_msd": load_year_prediction_msd, "yolanda": load_yolanda, + "superconductivity": load_superconductivity, + "qsar_tid_11": load_qsar_tid_11, "road_network": load_road_network, # index search "sift": load_sift, diff --git a/sklbench/datasets/transformer.py b/sklbench/datasets/transformer.py index 1efc31e6..eb5000ce 100644 --- a/sklbench/datasets/transformer.py +++ b/sklbench/datasets/transformer.py @@ -29,6 +29,28 @@ def convert_data(data, dformat: str, order: str, dtype: str, device: str = None): if isinstance(data, csr_matrix) and dformat != "csr_matrix": data = data.toarray() + # Remember which columns are categorical (and their exact CategoricalDtype) + # before the numpy round-trip below flattens the DataFrame and drops this + # metadata. The categoricals are restored after converting back to pandas, + # so estimators with native categorical support (e.g. XGBoost with + # enable_categorical=True) still see "category" columns. + categorical_dtypes = None + if ( + dformat == "pandas" + and isinstance(data, pd.DataFrame) + and any(str(t) == "category" for t in data.dtypes) + ): + categorical_dtypes = { + column: data[column].dtype + for column in data.columns + if str(data[column].dtype) == "category" + } + column_names = list(data.columns) + # dtype requested for the non-categorical (numeric) columns; re-applied + # after the round-trip. Categoricals can't be cast to a numeric dtype, + # so the numpy round-trip itself must run untyped (object). + numeric_dtype = None if dtype in (None, "preserve") else dtype + dtype = None if dtype == "preserve": dtype = None if order == "F": @@ -42,7 +64,17 @@ def convert_data(data, dformat: str, order: str, dtype: str, device: str = None) elif dformat == "pandas": if data.ndim == 1: return pd.Series(data) - return pd.DataFrame(data) + data = pd.DataFrame(data) + if categorical_dtypes is not None: + # restore original column names, re-apply the saved CategoricalDtype + # to categorical columns, and cast the rest to the requested dtype. + data.columns = column_names + for column in data.columns: + if column in categorical_dtypes: + data[column] = data[column].astype(categorical_dtypes[column]) + elif numeric_dtype is not None: + data[column] = data[column].astype(numeric_dtype) + return data elif dformat == "dpnp": import dpnp From 11849bd040793416bf69d87155629bf847619d82 Mon Sep 17 00:00:00 2001 From: Anatoly Volkov Date: Mon, 24 Aug 2026 10:21:30 -0700 Subject: [PATCH 2/4] Add bioresponse and nyc_taxi_green datasets, update xgboost configs, save preprocessing args to results --- configs/common/xgboost.json | 6 +- configs/regular/xgboost_binary.json | 19 +++++++ configs/regular/xgboost_regression.json | 55 +++++++----------- sklbench/benchmarks/common.py | 18 ++++++ sklbench/datasets/loaders.py | 76 +++++++++++++++++++++++++ sklbench/report/implementation.py | 3 + 6 files changed, 140 insertions(+), 37 deletions(-) diff --git a/configs/common/xgboost.json b/configs/common/xgboost.json index fdb637a1..f210722f 100644 --- a/configs/common/xgboost.json +++ b/configs/common/xgboost.json @@ -6,7 +6,8 @@ "device": "cpu", "estimator_params": { "tree_method": "hist", - "n_jobs": "[SPECIAL_VALUE]physical_cpus" + "n_jobs": "[SPECIAL_VALUE]physical_cpus", + "enable_categorical": true }, "enable_modelbuilders": false } @@ -16,7 +17,8 @@ "device": "gpu", "estimator_params": { "tree_method": "hist", - "n_jobs": "[SPECIAL_VALUE]physical_cpus" + "n_jobs": "[SPECIAL_VALUE]physical_cpus", + "enable_categorical": true } }, "data": { "format": "cudf" } diff --git a/configs/regular/xgboost_binary.json b/configs/regular/xgboost_binary.json index b9ea4faa..1c2f8d38 100644 --- a/configs/regular/xgboost_binary.json +++ b/configs/regular/xgboost_binary.json @@ -105,6 +105,25 @@ "enable_categorical": true } } + }, + { + "data": { + "dataset": "bioresponse", + "split_kwargs": { + "test_size": 0.2, + "random_state": 42 + } + }, + "algorithm": { + "estimator_params": { + "learning_rate": 0.05, + "max_depth": 6, + "subsample": 0.8, + "colsample_bytree": 0.8, + "reg_lambda": 1.0, + "n_estimators": 200 + } + } } ] }, diff --git a/configs/regular/xgboost_regression.json b/configs/regular/xgboost_regression.json index 4d0d4125..482d520a 100644 --- a/configs/regular/xgboost_regression.json +++ b/configs/regular/xgboost_regression.json @@ -2,23 +2,6 @@ "INCLUDE": ["../common/xgboost.json"], "PARAMETERS_SETS": { "gbt regression data": [ - { - "data": { - "dataset": "twodplanes", - "split_kwargs": { - "train_size": 0.33, - "test_size": null - } - }, - "algorithm": { - "estimator_params": { - "learning_rate": 0.01, - "reg_alpha": 1.0, - "reg_lambda": 0.1, - "n_estimators": 500 - } - } - }, { "data": { "dataset": "medical_charges_nominal", @@ -72,24 +55,6 @@ } } }, - { - "data": { - "dataset": "gisette", - "split_kwargs": { - "train_size": 2000, - "test_size": 5000 - } - }, - "algorithm": { - "estimator_params": { - "learning_rate": 0.15, - "max_leaves": 256, - "colsample_bytree": 0.1, - "colsample_bynode": 0.1, - "n_estimators": 100 - } - } - }, { "data": { "dataset": "superconductivity", @@ -127,6 +92,26 @@ "n_estimators": 1000 } } + }, + { + "data": { + "dataset": "nyc_taxi_green", + "preprocessing_kwargs": { + "category_encoding": ["ordinal", "ignore"] + }, + "split_kwargs": { + "test_size": 0.2, + "random_state": 42 + } + }, + "algorithm": { + "estimator_params": { + "learning_rate": 0.1, + "max_depth": 8, + "reg_lambda": 1.0, + "n_estimators": 500 + } + } } ] }, diff --git a/sklbench/benchmarks/common.py b/sklbench/benchmarks/common.py index 1df1e1a5..f3b40897 100644 --- a/sklbench/benchmarks/common.py +++ b/sklbench/benchmarks/common.py @@ -15,9 +15,11 @@ # =============================================================================== import argparse +import inspect import json from typing import Dict +from ..datasets.common import preprocess_x from ..utils.bench_case import get_bench_case_value, get_data_name from ..utils.custom_types import BenchCase from ..utils.logger import logger @@ -50,6 +52,22 @@ def enrich_result(result: Dict, bench_case: BenchCase) -> Dict: f"`{result['library']}` to `sklearnex` in benchmark output." ) result["library"] = "sklearnex" + preproc_kwargs = get_bench_case_value(bench_case, "data:preprocessing_kwargs", dict()) + preproc_defaults = { + name: param.default + for name, param in inspect.signature(preprocess_x).parameters.items() + } + result.update( + { + "category_encoding": preproc_kwargs.get( + "category_encoding", preproc_defaults["category_encoding"] + ), + "replace_nan": preproc_kwargs.get( + "replace_nan", preproc_defaults["replace_nan"] + ), + "normalize": preproc_kwargs.get("normalize", preproc_defaults["normalize"]), + } + ) taskset = get_bench_case_value(bench_case, "bench:taskset", None) if taskset is not None: result.update({"taskset": taskset}) diff --git a/sklbench/datasets/loaders.py b/sklbench/datasets/loaders.py index 769f1efe..148f8a85 100644 --- a/sklbench/datasets/loaders.py +++ b/sklbench/datasets/loaders.py @@ -580,6 +580,28 @@ def load_airline_satisfaction( return {"x": x, "y": y}, data_desc +@cache +def load_bioresponse( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + Bioresponse dataset. + https://www.openml.org/d/4134 + + Binary classification from drug discovery: predict whether a molecule + elicits a biological response (1) or not (0) from 1776 numeric molecular + descriptors. A real, wide, roughly balanced dataset (3751 samples x 1776 + features, ~54% majority class, no missing values). + + Source: OpenML dataset id 4134 (Bioresponse, curated by TabArena). + + Classification task. n_classes = 2. + """ + x, y = load_openml(4134, raw_data_cache) + data_desc = {"n_classes": 2, "default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + @cache def load_ijcnn( data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict @@ -955,6 +977,58 @@ def load_qsar_tid_11( return {"x": x, "y": y}, data_desc +@cache +def load_nyc_taxi_green( + data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict +) -> Tuple[Dict, Dict]: + """ + NYC green taxi trip records, December 2016 (credit-card trips only). + https://www.openml.org/d/42729 + + Regression task: predict the tip amount (`tip_amount`) from trip features. + ~582k samples x 18 features. + + OpenML marks 9 columns as nominal, but several of them are really numeric + fee amounts whose values carry order/magnitude (extra, mta_tax, + improvement_surcharge). Treating those as unordered categories throws away + the numeric meaning, so this loader converts them (and the ordinal + RatecodeID) back to numeric, and keeps only the genuinely categorical + columns -- vendor, forwarding flag, high-cardinality pickup/dropoff + location IDs, and trip type -- as pandas "category" dtype. + """ + # nominal columns that are actually numeric-valued -> cast to numeric + numeric_from_nominal = [ + "RatecodeID", + "extra", + "mta_tax", + "improvement_surcharge", + ] + # columns kept as genuine (unordered) categoricals + categorical_columns = [ + "VendorID", + "store_and_fwd_flag", + "PULocationID", + "DOLocationID", + "trip_type", + ] + + def transform_x_y(x, y): + for col in numeric_from_nominal: + if col in x.columns: + # category -> its (numeric) code values, as float + x[col] = pd.to_numeric(x[col].astype("str"), errors="coerce").astype( + "float32" + ) + for col in categorical_columns: + if col in x.columns: + x[col] = x[col].astype("category") + return x, y + + x, y = load_openml(42729, raw_data_cache, transform_x_y, as_frame=True) + data_desc = {"default_split": {"test_size": 0.2, "random_state": 42}} + return {"x": x, "y": y}, data_desc + + @cache def load_road_network( data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict @@ -1031,6 +1105,7 @@ def load_gist( "susy": load_susy, "ijcnn": load_ijcnn, "amazon_employee_access": load_amazon_employee_access, + "bioresponse": load_bioresponse, "apsfailure": load_apsfailure, "airline_satisfaction": load_airline_satisfaction, "kddcup09_appetency": load_kddcup09_appetency, @@ -1056,6 +1131,7 @@ def load_gist( "yolanda": load_yolanda, "superconductivity": load_superconductivity, "qsar_tid_11": load_qsar_tid_11, + "nyc_taxi_green": load_nyc_taxi_green, "road_network": load_road_network, # index search "sift": load_sift, diff --git a/sklbench/report/implementation.py b/sklbench/report/implementation.py index 7861e3b5..14b94481 100644 --- a/sklbench/report/implementation.py +++ b/sklbench/report/implementation.py @@ -105,6 +105,9 @@ "n_classes", "n_clusters", "batch_size", + "category_encoding", + "replace_nan", + "normalize", ] RED_COLOR, YELLOW_COLOR, GREEN_COLOR, WHITE_COLOR = "F85D5E", "FAF52E", "58C144", "FFFFFF" From 319ebb215363ca8c9229f30f65ccb15f8ea25c7c Mon Sep 17 00:00:00 2001 From: Anatoly Volkov Date: Tue, 1 Sep 2026 07:00:50 -0700 Subject: [PATCH 3/4] Add task_dispatch option to avoid data duplication for airline_depdelay dataset --- configs/regular/xgboost_binary.json | 44 +++++++-------- configs/regular/xgboost_multi.json | 14 ++--- configs/regular/xgboost_regression.json | 73 ++++++++++++++----------- sklbench/datasets/common.py | 27 ++++++++- sklbench/datasets/loaders.py | 25 +++++---- 5 files changed, 107 insertions(+), 76 deletions(-) diff --git a/configs/regular/xgboost_binary.json b/configs/regular/xgboost_binary.json index 1c2f8d38..37eebd1b 100644 --- a/configs/regular/xgboost_binary.json +++ b/configs/regular/xgboost_binary.json @@ -26,25 +26,28 @@ }, { "data": { - "dataset": "hepmass", + "dataset": "airline_satisfaction", + "preprocessing_kwargs": { + "category_encoding": ["ignore", "ordinal"] + }, "split_kwargs": { - "train_size": 400000, - "test_size": 1000000 + "test_size": 0.2, + "random_state": 42 } }, "algorithm": { "estimator_params": { "learning_rate": 0.1, - "reg_alpha": 1.0, + "max_depth": 6, "reg_lambda": 1.0, - "max_leaves": 256, - "n_estimators": 200 + "n_estimators": 1000, + "enable_categorical": true } } }, { "data": { - "dataset": "airline_satisfaction", + "dataset": "amazon_employee_access", "preprocessing_kwargs": { "category_encoding": ["ignore", "ordinal"] }, @@ -55,10 +58,10 @@ }, "algorithm": { "estimator_params": { - "learning_rate": 0.1, + "learning_rate": 0.05, "max_depth": 6, "reg_lambda": 1.0, - "n_estimators": 1000, + "n_estimators": 500, "enable_categorical": true } } @@ -87,10 +90,7 @@ }, { "data": { - "dataset": "amazon_employee_access", - "preprocessing_kwargs": { - "category_encoding": ["ignore", "ordinal"] - }, + "dataset": "bioresponse", "split_kwargs": { "test_size": 0.2, "random_state": 42 @@ -100,27 +100,27 @@ "estimator_params": { "learning_rate": 0.05, "max_depth": 6, + "subsample": 0.8, + "colsample_bytree": 0.8, "reg_lambda": 1.0, - "n_estimators": 500, - "enable_categorical": true + "n_estimators": 200 } } }, { "data": { - "dataset": "bioresponse", + "dataset": "hepmass", "split_kwargs": { - "test_size": 0.2, - "random_state": 42 + "train_size": 400000, + "test_size": 1000000 } }, "algorithm": { "estimator_params": { - "learning_rate": 0.05, - "max_depth": 6, - "subsample": 0.8, - "colsample_bytree": 0.8, + "learning_rate": 0.1, + "reg_alpha": 1.0, "reg_lambda": 1.0, + "max_leaves": 256, "n_estimators": 200 } } diff --git a/configs/regular/xgboost_multi.json b/configs/regular/xgboost_multi.json index 4a0611f5..4cfcba1d 100644 --- a/configs/regular/xgboost_multi.json +++ b/configs/regular/xgboost_multi.json @@ -4,24 +4,24 @@ "gbt multi classification data": [ { "data": { - "dataset": "covtype" + "dataset": "connect" }, "algorithm": { "estimator_params": { - "learning_rate": 0.3, - "reg_lambda": 1, - "max_depth": 14, - "n_estimators": 100 + "n_estimators": 500 } } }, { "data": { - "dataset": "connect" + "dataset": "covtype" }, "algorithm": { "estimator_params": { - "n_estimators": 500 + "learning_rate": 0.3, + "reg_lambda": 1, + "max_depth": 14, + "n_estimators": 100 } } } diff --git a/configs/regular/xgboost_regression.json b/configs/regular/xgboost_regression.json index 482d520a..4a8f02ac 100644 --- a/configs/regular/xgboost_regression.json +++ b/configs/regular/xgboost_regression.json @@ -4,60 +4,70 @@ "gbt regression data": [ { "data": { - "dataset": "medical_charges_nominal", + "dataset": "airline_depdelay", + "dataset_kwargs": { + "task": "regression" + }, + "preprocessing_kwargs": { + "category_encoding": "ordinal", + "subsample": 700000 + }, "split_kwargs": { - "train_size": 0.1, - "test_size": 0.9 + "train_size": 200000, + "test_size": 500000 } }, "algorithm": { "estimator_params": { "learning_rate": 0.1, - "max_depth": 5, + "max_depth": 8, "reg_alpha": 1.0, "reg_lambda": 1.0, - "n_estimators": 1000 + "n_estimators": 500 } } }, { "data": { - "dataset": "year_prediction_msd", + "dataset": "medical_charges_nominal", "split_kwargs": { - "train_size": 0.25, - "test_size": 0.75 + "train_size": 0.5, + "test_size": 0.5 } }, "algorithm": { "estimator_params": { "learning_rate": 0.1, - "reg_alpha": 0.5, - "reg_lambda": 0.5, - "n_estimators": 200 + "max_depth": 5, + "reg_alpha": 1.0, + "reg_lambda": 1.0, + "n_estimators": 1000 } } }, { "data": { - "dataset": "hepmass", + "dataset": "nyc_taxi_green", + "preprocessing_kwargs": { + "category_encoding": ["ordinal", "ignore"] + }, "split_kwargs": { - "train_size": 200000, - "test_size": 1000000 + "test_size": 0.2, + "random_state": 42 } }, "algorithm": { "estimator_params": { "learning_rate": 0.1, - "reg_alpha": 1.0, + "max_depth": 8, "reg_lambda": 1.0, - "max_leaves": 256, "n_estimators": 500 } } }, { "data": { - "dataset": "superconductivity", + "dataset": "qsar_tid_11", "split_kwargs": { "test_size": 0.2, "random_state": 42 @@ -65,18 +75,18 @@ }, "algorithm": { "estimator_params": { - "learning_rate": 0.02, - "max_depth": 7, + "learning_rate": 0.05, + "max_depth": 6, "subsample": 0.8, "colsample_bytree": 0.8, "reg_lambda": 1.0, - "n_estimators": 1500 + "n_estimators": 1000 } } }, { "data": { - "dataset": "qsar_tid_11", + "dataset": "superconductivity", "split_kwargs": { "test_size": 0.2, "random_state": 42 @@ -84,32 +94,29 @@ }, "algorithm": { "estimator_params": { - "learning_rate": 0.05, - "max_depth": 6, + "learning_rate": 0.02, + "max_depth": 7, "subsample": 0.8, "colsample_bytree": 0.8, "reg_lambda": 1.0, - "n_estimators": 1000 + "n_estimators": 1500 } } }, { "data": { - "dataset": "nyc_taxi_green", - "preprocessing_kwargs": { - "category_encoding": ["ordinal", "ignore"] - }, + "dataset": "year_prediction_msd", "split_kwargs": { - "test_size": 0.2, - "random_state": 42 + "train_size": 0.5, + "test_size": 0.5 } }, "algorithm": { "estimator_params": { "learning_rate": 0.1, - "max_depth": 8, - "reg_lambda": 1.0, - "n_estimators": 500 + "reg_alpha": 0.5, + "reg_lambda": 0.5, + "n_estimators": 200 } } } diff --git a/sklbench/datasets/common.py b/sklbench/datasets/common.py index 26631bc8..8aa229ef 100644 --- a/sklbench/datasets/common.py +++ b/sklbench/datasets/common.py @@ -34,19 +34,19 @@ from ..utils.logger import logger # NB: non-registered data components and extensions will not be found by loader -KNOWN_DATA_COMPONENTS = ["x", "y"] +KNOWN_DATA_COMPONENTS = ["x", "y", "y_cls", "y_reg"] KNOWN_DATA_EXTENSIONS = ["parq", "npz", "csr.npz"] def get_expr_by_prefix(prefix: str) -> str: def get_or_expr_from_list(a: List[str]) -> str: # transforms list to OR expression: "['x', 'y']" -> "x|y" - return str(a)[1:-1].replace("'", "").replace(", ", "|") + return "|".join(re.escape(item) for item in a) data_comp_expr = get_or_expr_from_list(KNOWN_DATA_COMPONENTS) data_ext_expr = get_or_expr_from_list(KNOWN_DATA_EXTENSIONS) - return f"{prefix}_({data_comp_expr}).({data_ext_expr})" + return f"^{re.escape(prefix)}_({data_comp_expr})\\.({data_ext_expr})$" def get_filenames_by_prefix(directory: str, prefix: str) -> List[str]: @@ -141,6 +141,27 @@ def save_data_description(data_desc: Dict, data_cache: str, data_name: str): json.dump(data_desc, desc_file) +""" +This function is needed to avoid storing the dataset two times if +it's used for both classification and regression tasks (e.g. airline_deepdelay) +""" + + +def task_dispatch(function): + def task_dispatch_wrapper(**kwargs): + data, data_desc = function(**kwargs) + dataset_params = kwargs.get("dataset_params", dict()) + task = dataset_params.get("task", "classification") + if task == "classification": + return {"x": data["x"], "y": data["y_cls"]}, data_desc + elif task == "regression": + return {"x": data["x"], "y": data["y_reg"]}, data_desc + else: + raise ValueError(f'Unknown "{task}" task type for airline dataset.') + + return task_dispatch_wrapper + + def cache(function): def cache_wrapper(**kwargs): data_name = kwargs["data_name"] diff --git a/sklbench/datasets/loaders.py b/sklbench/datasets/loaders.py index 148f8a85..f47cb344 100644 --- a/sklbench/datasets/loaders.py +++ b/sklbench/datasets/loaders.py @@ -31,7 +31,13 @@ make_regression, ) -from .common import cache, load_data_description, load_data_from_cache, preprocess +from .common import ( + cache, + load_data_description, + load_data_from_cache, + preprocess, + task_dispatch, +) from .downloaders import download_and_read_csv, load_openml, retrieve @@ -103,6 +109,7 @@ def load_custom_data( """ +@task_dispatch @cache def load_airline_depdelay( data_name: str, data_cache: str, raw_data_cache: str, dataset_params: Dict @@ -112,6 +119,9 @@ def load_airline_depdelay( http://kt.ijs.si/elena_ikonomovska/data.html Classification task. n_classes = 2. + + This dataset can also be used for regression problems, to do that + dataset_params:task should be set to "regression". """ url = "http://kt.ijs.si/elena_ikonomovska/datasets/airline/airline_14col.data.bz2" @@ -152,22 +162,15 @@ def load_airline_depdelay( for col in df.select_dtypes(["object"]).columns: df[col] = df[col].astype("category") - task = dataset_params.get("task", "classification") - if task == "classification": - df["ArrDelay"] = (df["ArrDelay"] > 0).astype(int) - elif task == "regression": - pass - else: - raise ValueError(f'Unknown "{task}" task type for airline dataset.') - - y = df["ArrDelay"].to_numpy(dtype=np.float32) + y_cls = (df["ArrDelay"] > 0).astype(int).to_numpy(dtype=np.float32) + y_reg = df["ArrDelay"].to_numpy(dtype=np.float32) x = df.drop(columns=["ArrDelay"]) data_description = { "n_classes": 2, "default_split": {"test_size": 0.2, "random_state": 42}, } - return {"x": x, "y": y}, data_description + return {"x": x, "y_cls": y_cls, "y_reg": y_reg}, data_description @cache From fc72160145dc76ee8c47b6076f1e196f101c12e2 Mon Sep 17 00:00:00 2001 From: Anatoly Volkov Date: Tue, 1 Sep 2026 07:33:38 -0700 Subject: [PATCH 4/4] Minor optimization --- sklbench/datasets/transformer.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sklbench/datasets/transformer.py b/sklbench/datasets/transformer.py index eb5000ce..fac5989b 100644 --- a/sklbench/datasets/transformer.py +++ b/sklbench/datasets/transformer.py @@ -69,11 +69,12 @@ def convert_data(data, dformat: str, order: str, dtype: str, device: str = None) # restore original column names, re-apply the saved CategoricalDtype # to categorical columns, and cast the rest to the requested dtype. data.columns = column_names - for column in data.columns: - if column in categorical_dtypes: - data[column] = data[column].astype(categorical_dtypes[column]) - elif numeric_dtype is not None: - data[column] = data[column].astype(numeric_dtype) + dtype_map = { + column: categorical_dtypes.get(column, numeric_dtype) + for column in column_names + if column in categorical_dtypes or numeric_dtype is not None + } + data = data.astype(dtype_map) return data elif dformat == "dpnp": import dpnp