Modeling 111001001 all features¶
In this notebook, we're specifically working on the dataset formed by dropping (31, 496, 524, 917, 1299) and keeping all.
In [1]:
import itertools
import numpy as np
import pandas as pd
import scipy
from scipy import optimize
pd.set_option('display.precision',20)
pd.set_option('display.max_colwidth',100)
from sklearn import linear_model, svm, tree
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, cross_val_predict, KFold, cross_val_score, \
GridSearchCV, RandomizedSearchCV, ShuffleSplit
from sklearn.neural_network import MLPRegressor
import xgboost as xgb
from time import time
from scipy.stats import randint as sp_randint
import matplotlib.pylab as plt
from matplotlib.pylab import rcParams
from matplotlib import pyplot
rcParams['figure.figsize'] = 12, 4
%matplotlib inline
In [2]:
# def to compare goodness of fit on training set
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
In [3]:
# Utility function to report best scores
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("")
In [4]:
# run randomized search
def random_search(regr, param_dist, n_iter_search):
rs = RandomizedSearchCV(regr, param_distributions=param_dist, scoring = 'neg_mean_squared_error',
n_jobs=-1, n_iter=n_iter_search, cv=kfold) #, verbose = 4)
start = time()
rs.fit(x_train, y_train)
print("RandomizedSearchCV took %.2f seconds for %d candidates"
" parameter settings." % ((time() - start), n_iter_search))
report(rs.cv_results_)
In [5]:
# run single parameter search (for ridge or lasso)
def single_search(regr, params):
regr_results_df = pd.DataFrame(dtype = 'float64')
count = 0
for k, v in params.items():
for val in v:
regr.set_params(**{k: val})
regr_results_df.loc[count, k] = val
results = cross_val_score(regr, x_train, y_train, cv=kfold, scoring = 'neg_mean_squared_error')
(regr_results_df.loc[count, 'RMSE'], regr_results_df.loc[count, 'std dev']) = \
(np.sqrt(-results.mean()), np.sqrt(results.std()))
count += 1
return regr_results_df
In [6]:
# test against validation set
def validate(regr):
regr.fit(x_train, y_train)
y_pred = regr.predict(x_validation)
return rmse(y_validation, y_pred)
In [7]:
# Cross-validation sets
kfold = KFold(n_splits=10, random_state=7)
In [8]:
df = pd.read_csv("./input/train_tidy_111001001.csv")
In [9]:
ss = ShuffleSplit(n_splits=1, test_size=0.20, random_state=71)
X = df.values
for train_idx, validation_idx in ss.split(X):
train_df = df.iloc[train_idx]
validation_df = df.iloc[validation_idx]
y_validation = validation_df['SalePrice'].values
x_validation = validation_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
y_train = train_df['SalePrice'].values
x_train = train_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
Ridge regression¶
In [10]:
ridge_regr = linear_model.Ridge()
In [11]:
ridge_param = {'alpha': scipy.stats.expon(scale=10)}
random_search(ridge_regr, ridge_param, 20)
In [12]:
ridge_params = {'alpha': np.arange(1, 15, 1).tolist()}
ridge_df = single_search(ridge_regr, ridge_params)
ridge_df.plot(x = ['alpha'], y = ['RMSE'])
ridge_df.sort_values(['RMSE'], ascending = False)
Out[12]:
In [13]:
ridge_regr = linear_model.Ridge(alpha = 8)
In [14]:
validate(ridge_regr)
Out[14]:
Lasso¶
In [15]:
lasso_regr = linear_model.Lasso(max_iter=50000)
In [16]:
lasso_param = {'alpha': scipy.stats.expon(scale=0.001)}
random_search(lasso_regr, lasso_param, 20)
In [17]:
lasso_params = {'alpha': np.arange(0.0001, 0.0007, 0.0001).tolist()}
lasso_df = single_search(lasso_regr, lasso_params)
lasso_df.plot(x = ['alpha'], y = ['RMSE'])
lasso_df.sort_values(['RMSE'], ascending = False)
Out[17]:
In [18]:
lasso_regr = linear_model.Lasso(alpha=0.0003, max_iter=50000)
In [19]:
validate(lasso_regr)
Out[19]:
Lasso LARS¶
In [20]:
lassolars_regr = linear_model.LassoLars(max_iter=50000)
In [21]:
lassolars_param = {'alpha': scipy.stats.expon(scale=0.0001)}
random_search(lassolars_regr, lassolars_param, 20)
In [22]:
lassolars_params = {'alpha': np.arange(0.00008, 0.00014, 0.00001).tolist()}
lassolars_df = single_search(lassolars_regr, lassolars_params)
lassolars_df.plot(x = ['alpha'], y = ['RMSE'])
lassolars_df.sort_values(['RMSE'], ascending = False)
Out[22]:
In [23]:
lassolars_regr = linear_model.LassoLars(alpha=0.0001, max_iter=50000)
In [24]:
validate(lassolars_regr)
Out[24]:
Elastic Net¶
In [25]:
elnet_regr = linear_model.ElasticNet(max_iter=15000, random_state=7)
In [26]:
elnet_param = {'alpha': scipy.stats.expon(scale=0.001), 'l1_ratio': scipy.stats.truncnorm(0, 1, loc=0.5, scale=0.25)}
random_search(elnet_regr, elnet_param, 20)
In [28]:
elnet_regr = linear_model.ElasticNet(alpha=0.00038, l1_ratio=0.5, max_iter=15000, random_state=7)
elnet_params = {'alpha': np.arange(0.0001, 0.0008, 0.0001).tolist()}
elnet_df = single_search(elnet_regr, elnet_params)
elnet_df.plot(x = ['alpha'], y = ['RMSE'])
elnet_df.sort_values(['RMSE'], ascending = False)
Out[28]:
In [29]:
elnet_regr = linear_model.ElasticNet(alpha=0.0005, max_iter=15000, random_state=7)
elnet_params = {'l1_ratio': np.arange(0.1, 1, 0.1).tolist()}
elnet_df = single_search(elnet_regr, elnet_params)
elnet_df.plot(x = ['l1_ratio'], y = ['RMSE'])
elnet_df.sort_values(['RMSE'], ascending = False)
Out[29]:
In [30]:
elnet_regr = linear_model.ElasticNet(alpha=0.0005, l1_ratio=0.6, max_iter=15000, random_state=7)
In [31]:
validate(elnet_regr)
Out[31]:
Random Forest¶
In [32]:
rf_regr = RandomForestRegressor(n_jobs=-1, random_state = 7)
In [33]:
rf_param = {'n_estimators': scipy.stats.randint(50,400), 'max_depth': scipy.stats.randint(2,20),
'max_features': scipy.stats.randint(5,30)}
random_search(rf_regr, rf_param, 20)
In [62]:
rf_regr = RandomForestRegressor(n_estimators = 361, max_depth = 15, max_features = 55, random_state = 7)
rf_params = {'n_estimators': np.arange(100, 200, 10).tolist()}
rf_df = single_search(rf_regr, rf_params)
rf_df.plot(x = ['n_estimators'], y = ['RMSE'])
rf_df.sort_values(['RMSE'], ascending = False)
Out[62]:
In [65]:
rf_regr = RandomForestRegressor(n_estimators = 150, max_depth = 15, max_features = 55, random_state = 7)
In [66]:
validate(rf_regr)
Out[66]:
Support Vector Regressor¶
In [34]:
svr_regr = svm.SVR(cache_size=200, degree=3, max_iter=-1, shrinking=True, tol=0.001, verbose=False)
In [35]:
svr_param = {'C': scipy.stats.expon(scale=1), 'gamma': scipy.stats.expon(scale=.1),
'kernel': ['rbf']}
random_search(svr_regr, svr_param, 20)
In [46]:
svr_regr = svm.SVR(C=0.8, cache_size=200, coef0=0.0, degree=3, gamma=0.007,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)
svr_params = {'C': np.arange(0.5, 4.5, 0.5).tolist()}
svr_df = single_search(svr_regr, svr_params)
svr_df.plot(x = ['C'], y = ['RMSE'])
svr_df.sort_values(['RMSE'], ascending = False)
Out[46]:
In [52]:
svr_regr = svm.SVR(C=1.7, cache_size=200, coef0=0.0, degree=3, gamma=0.007,
kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)
In [53]:
validate(svr_regr)
Out[53]:
Orthogonal Matching Pursuit¶
In [36]:
omp_regr = linear_model.OrthogonalMatchingPursuit()
In [37]:
omp_param = {'n_nonzero_coefs': scipy.stats.randint(20,100)}
random_search(omp_regr, omp_param, 20)
In [43]:
omp_regr = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=27)
omp_params = {'n_nonzero_coefs': np.arange(75, 125, 1).tolist()}
omp_df = single_search(omp_regr, omp_params)
omp_df.plot(x = ['n_nonzero_coefs'], y = ['RMSE'])
omp_df.sort_values(['RMSE'], ascending = False)
Out[43]:
In [47]:
omp_regr = linear_model.OrthogonalMatchingPursuit(n_nonzero_coefs=105)
In [48]:
validate(omp_regr)
Out[48]:
XGBoost¶
In [38]:
xgb_regr = xgb.XGBRegressor(seed = 42, nthread = -1, silent = 1)
xgb_param = {'max_depth': scipy.stats.randint(1,10), # Max. Depth [1,∞]
'min_child_weight': scipy.stats.expon(scale=1), # Min. Child Weight [0,∞]
'gamma': scipy.stats.expon(scale=1), # Minimum Split Loss, gamma [0,∞]
'subsample': scipy.stats.uniform(0, 1), # Subsample Ratio of the training instance [0,1]
'colsample_bytree': scipy.stats.uniform(0, 1), # Subsample Ratio of columns [0,1]
'reg_alpha': scipy.stats.expon(scale=1), # L1 regularization, alpha [0,∞]
'reg_lambda': scipy.stats.expon(scale=1), # L2 regularization, lambda [0,∞]
'learning_rate': scipy.stats.uniform(0, 1), # Step size shrinkage (learning rate) [0,1]
'n_estimators': scipy.stats.randint(50,500) # Number of boosted trees, n_estimators
}
random_search(xgb_regr, xgb_param, 20)
In [81]:
xgb_regr = xgb.XGBRegressor(
max_depth = 5,
min_child_weight = 5.1,
gamma = 0.01,
subsample = 0.7,
colsample_bytree = 0.5,
reg_alpha = 0.25,
reg_lambda = 0.024,
learning_rate = 0.09,
n_estimators = 86,
objective='reg:linear',
seed = 42,
nthread = -1,
silent = 1)
xgb_params = {'n_estimators': np.arange(100, 1200, 100).tolist()}
xgb_df = single_search(xgb_regr, xgb_params)
xgb_df.plot(x = ['n_estimators'], y = ['RMSE'])
xgb_df.sort_values(['RMSE'], ascending = False)
Out[81]:
In [82]:
xgb_regr = xgb.XGBRegressor(
max_depth = 5,
min_child_weight = 5.1,
gamma = 0.01,
subsample = 0.7,
colsample_bytree = 0.5,
reg_alpha = 0.25,
reg_lambda = 0.024,
learning_rate = 0.09,
n_estimators = 300,
objective='reg:linear',
seed = 42,
nthread = -1,
silent = 1)
In [83]:
validate(xgb_regr)
Out[83]:
MLP Regressor¶
In [39]:
mlp_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(100, ),
alpha=0.0001,
tol=0.0001,
max_iter=200)
In [40]:
mlp_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641)
mlp_param = {'hidden_layer_sizes': scipy.stats.randint(50,500), # size of a single hidden layer
'alpha': scipy.stats.expon(scale=0.0001), # L2 regularization, [0,∞]
'tol': scipy.stats.expon(scale=0.0001), # Minimum improvement in loss fun. during training
'max_iter': scipy.stats.uniform(200, 400), # Maximum number of iterations of solver
}
random_search(mlp_regr, mlp_param, 20)
In [77]:
mlp_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(3, ),
alpha=0.00006,
tol=0.00038,
max_iter=293)
mlp_params = {'max_iter': np.arange(300, 450, 25).tolist()}
# mlp_params = {'tol': np.logspace(-5, -2, 12).tolist()}
mlp_df = single_search(mlp_regr, mlp_params)
mlp_df.plot(x = ['max_iter'], y = ['RMSE'])
mlp_df.sort_values(['RMSE'], ascending = False)
Out[77]:
In [79]:
mlp_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(3, ),
alpha=0.00006,
tol=0.00038,
max_iter=300)
In [80]:
validate(mlp_regr)
Out[80]:
In [ ]: