I am having trouble getting an XGBRegressor run on my GPU when it works fine with XGBClassifier. Here is the code and the error:
Classification:
import xgboost as xgb
from sklearn.model_selection import GridSearchCV, RepeatedKFold
import cupy as cp
# Convert data to CuPy arrays for GPU processing
X_train_gpu = cp.asarray(X_train_preprocessed.values)
y_train_gpu = cp.asarray(y_train.values)
# Define the parameter grid to search
param_grid = {
'n_estimators': [500, 750, 1000],
'learning_rate': [0.005, 0.01, 0.025],
'max_depth': [8, 10, 12],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'min_child_weight': [1, 3, 5],
'gamma': [0, 0.1, 0.2],
'reg_alpha': [0, 0.1, 0.5, 1.0],
'reg_lambda': [0.5, 1.0, 1.5],
'scale_pos_weight': [0.5, 1, 2],
'max_delta_step': [0, 0.1, 0.5, 1],
}
# CV Settings
rkf = RepeatedKFold(n_splits=5, n_repeats=1)
# Initialize the XGB regressor
xgb_reg = xgb.XGBRegressor(tree_method='gpu_hist', device='cuda', eval_metric='rmse')
# Initialize the GridSearchCV object
xgb_gscv = GridSearchCV(estimator=xgb_reg, param_grid=param_grid, scoring='neg_mean_squared_error', cv=rkf, verbose=3, n_jobs=-1)
# Fit the grid search model
xgb_gscv.fit(X_train_gpu.get(), y_train_gpu.get())
Runes fine.
Regression:
import xgboost as xgb
from sklearn.model_selection import GridSearchCV, RepeatedKFold
import cupy as cp
# Convert data to CuPy arrays for GPU processing
X_train_gpu = cp.asarray(X_train_preprocessed.values)
y_train_gpu = cp.asarray(y_train.values)
# Define the parameter grid to search
param_grid = {
'n_estimators': [500, 750, 1000],
'learning_rate': [0.005, 0.01, 0.025],
'max_depth': [8, 10, 12],
'subsample': [0.6, 0.8, 1.0],
'colsample_bytree': [0.6, 0.8, 1.0],
'min_child_weight': [1, 3, 5],
'gamma': [0, 0.1, 0.2],
'reg_alpha': [0, 0.1, 0.5, 1.0],
'reg_lambda': [0.5, 1.0, 1.5],
'scale_pos_weight': [0.5, 1, 2],
'max_delta_step': [0, 0.1, 0.5, 1],
}
# CV Settings
rkf = RepeatedKFold(n_splits=5, n_repeats=1)
# Initialize the XGB regressor
xgb_reg = xgb.XGBRegressor(tree_method='gpu_hist', device='cuda', eval_metric='rmse')
# Initialize the GridSearchCV object
xgb_gscv = GridSearchCV(estimator=xgb_reg, param_grid=param_grid, scoring='neg_mean_squared_error', cv=rkf, verbose=3, n_jobs=-1)
# Fit the grid search model
xgb_gscv.fit(X_train_gpu.get(), y_train_gpu.get())
Error:
UserWarning: [23:08:16] WARNING: C:\buildkite-agent\builds\buildkite-windows-cpu-autoscaling-group-i-0b3782d1791676daf-1\xgboost\xgboost-ci-windows\src\common\error_msg.cc:58: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.
Potential solutions:
- Use a data structure that matches the device ordinal in the booster.
- Set the device for booster before call to inplace_predict.
This warning will only be shown once.
Why is this happening? There is no difference in pre-processing.
This does work with no error:
import numpy as np
import xgboost as xgb
xgb_model = xgb.XGBRegressor( # tree_method="gpu_hist" # deprecated
tree_method="hist",
device="cuda"
)
X = np.random.rand(50, 2)
y = np.random.randint(2, size=50)
xgb_model.fit(X, y)
xgb_model