Subsetting 111001001 all features¶
In this notebook, we're specifically working on the dataset formed by dropping (31, 496, 524, 917, 1299) with all features.
We note that houses with a basement or garage have additional features, which might improve the regression. We will explore the best way to exploit this by considering subsets of the data.
In [1]:
import itertools
import numpy as np
import pandas as pd
import scipy
from scipy import optimize
pd.set_option('display.precision',20)
pd.set_option('display.max_colwidth',100)
from sklearn import linear_model, svm, tree
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, cross_val_predict, KFold, cross_val_score, \
GridSearchCV, RandomizedSearchCV, ShuffleSplit, StratifiedShuffleSplit
from sklearn.neural_network import MLPRegressor
import xgboost as xgb
from time import time
from scipy.stats import randint as sp_randint
import matplotlib.pylab as plt
from matplotlib.pylab import rcParams
from matplotlib import pyplot
rcParams['figure.figsize'] = 12, 4
%matplotlib inline
In [2]:
# def to compare goodness of fit on training set
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
In [3]:
# Utility function to report best scores
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("")
In [4]:
# run randomized search
def random_search(regr, param_dist, n_iter_search):
rs = RandomizedSearchCV(regr, param_distributions=param_dist, scoring = 'neg_mean_squared_error',
n_jobs=-1, n_iter=n_iter_search, cv=kfold) #, verbose = 4)
start = time()
rs.fit(x_train, y_train)
print("RandomizedSearchCV took %.2f seconds for %d candidates"
" parameter settings." % ((time() - start), n_iter_search))
report(rs.cv_results_)
In [5]:
# run single parameter search (for ridge or lasso)
def single_search(regr, params):
regr_results_df = pd.DataFrame(dtype = 'float64')
count = 0
for k, v in params.items():
for val in v:
regr.set_params(**{k: val})
regr_results_df.loc[count, k] = val
results = cross_val_score(regr, x_train, y_train, cv=kfold, scoring = 'neg_mean_squared_error')
(regr_results_df.loc[count, 'RMSE'], regr_results_df.loc[count, 'std dev']) = \
(np.sqrt(-results.mean()), np.sqrt(results.std()))
count += 1
return regr_results_df
In [6]:
# test against validation set
def validate(regr):
regr.fit(x_train, y_train)
y_pred = regr.predict(x_validation)
return rmse(y_validation, y_pred)
In [7]:
# Cross-validation sets
kfold = KFold(n_splits=10, random_state=7)
In [8]:
df = pd.read_csv("./input/train_tidy_111001001.csv")
test_df = pd.read_csv("./input/test_tidy_111001001.csv")
Garage¶
In [13]:
df[df['NoGarage']==1].shape
Out[13]:
Only 80 of the houses in the training data have no garage. Let us set a baseline validation error for when we don't subset.
In [14]:
ss = ShuffleSplit(n_splits=1, test_size=0.20, random_state=71)
X = df.values
for train_idx, validation_idx in ss.split(X):
train_df = df.iloc[train_idx]
validation_df = df.iloc[validation_idx]
y_validation = validation_df['SalePrice'].values
x_validation = validation_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
y_train = train_df['SalePrice'].values
x_train = train_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
In [16]:
lassolarscv_regr = linear_model.LassoLarsCV()
baseline = validate(lassolarscv_regr)
baseline
Out[16]:
Now we restrict to houses with a garage.
In [9]:
df_hasg = df[df['NoGarage']==0]
In [18]:
X = df_hasg.values
for train_idx, validation_idx in ss.split(X):
train_df = df_hasg.iloc[train_idx]
validation_df = df_hasg.iloc[validation_idx]
y_validation = validation_df['SalePrice'].values
x_validation = validation_df.drop(['HouseId', 'SalePrice'],axis=1).values
y_train = train_df['SalePrice'].values
x_train = train_df.drop(['HouseId', 'SalePrice'],axis=1).values
In [19]:
hasg = validate(lassolarscv_regr)
(hasg, hasg - baseline)
Out[19]:
In this case, our validation error has increased. This could be due to sample size.
Basements¶
Let's examine the houses with basements. Only 37 houses have no basement.
In [20]:
df[df['NoBasement']==1].shape
Out[20]:
In [10]:
df_hasb = df[df['NoBasement']==0]
In [22]:
X = df_hasb.values
for train_idx, validation_idx in ss.split(X):
train_df = df_hasb.iloc[train_idx]
validation_df = df_hasb.iloc[validation_idx]
y_validation = validation_df['SalePrice'].values
x_validation = validation_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
y_train = train_df['SalePrice'].values
x_train = train_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
In [23]:
hasb = validate(lassolarscv_regr)
(hasb, hasb - baseline)
Out[23]:
Here, our validation error is even worse than in the garage case.
Garage and Basement¶
In [11]:
df_hasgb = df_hasg[df_hasg['NoBasement']==0]
df_hasgb.shape
Out[11]:
1345 houses have both a garage and basement.
In [27]:
X = df_hasgb.values
for train_idx, validation_idx in ss.split(X):
train_df = df_hasgb.iloc[train_idx]
validation_df = df_hasgb.iloc[validation_idx]
y_validation = validation_df['SalePrice'].values
x_validation = validation_df.drop(['HouseId', 'SalePrice'],axis=1).values
y_train = train_df['SalePrice'].values
x_train = train_df.drop(['HouseId', 'SalePrice'],axis=1).values
In [28]:
hasgb = validate(lassolarscv_regr)
(hasgb, hasgb - baseline)
Out[28]:
This is a small improvement over the baseline. We will now work out a scheme to make a separate fit on this subset and then combine the predictions.
Modeling on Subsets¶
Let's store a dataframe that is the complement of df_hasgb.
In [12]:
df_nogorb = df[(df['NoGarage']==1) | (df['NoBasement']==1)]
df_nogorb.shape
Out[12]:
We'll build the validation set from the full dataset
In [30]:
X = df.values
for train_idx, validation_idx in ss.split(X):
train_df = df.iloc[train_idx]
validation_df = df.iloc[validation_idx]
We will split the training set
In [31]:
train_df_hasgb = train_df[(train_df['NoGarage']==0) & (train_df['NoBasement']==0)]
train_df_nogorb = train_df[(train_df['NoGarage']==1) | (train_df['NoBasement']==1)]
We need to fit models on the appropriate training data.
In [32]:
llcv_hasgb_regr = linear_model.LassoLarsCV()
llcv_nogorb_regr = linear_model.LassoLarsCV()
In [33]:
y_train_nogorb = train_df['SalePrice'].values
x_train_nogorb = train_df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
y_train_hasgb = train_df_hasgb['SalePrice'].values
x_train_hasgb = train_df_hasgb.drop(['HouseId', 'SalePrice'],axis=1).values
In [34]:
llcv_hasgb_regr.fit(x_train_hasgb, y_train_hasgb)
llcv_nogorb_regr.fit(x_train_nogorb, y_train_nogorb)
Out[34]:
In [36]:
validation_df_hasgb = validation_df[(validation_df['NoGarage']==0) & (validation_df['NoBasement']==0)]
validation_df_nogorb = validation_df[(validation_df['NoGarage']==1) | (validation_df['NoBasement']==1)]
In [37]:
y_validation_nogorb = validation_df_nogorb['SalePrice'].values
x_validation_nogorb = validation_df_nogorb.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
y_validation_hasgb = validation_df_hasgb['SalePrice'].values
x_validation_hasgb = validation_df_hasgb.drop(['HouseId', 'SalePrice'],axis=1).values
In [38]:
y_pred_hasgb = llcv_hasgb_regr.predict(x_validation_hasgb)
y_pred_nogorb = llcv_nogorb_regr.predict(x_validation_nogorb)
In [47]:
y_validation = np.append(y_validation_nogorb,y_validation_hasgb)
In [49]:
y_pred = np.append(y_pred_nogorb,y_pred_hasgb)
In [51]:
error = rmse(y_validation, y_pred)
(error, error-baseline)
Out[51]:
The improvement over the baseline is in line with the previous result.
Refitting on the Full Training Set¶
Before passing to the Kaggle test set, let's fit LassoLars models on the whole training data.
In [13]:
y_train_nogorb = df['SalePrice'].values
x_train_nogorb = df.drop(['HouseId', 'SalePrice', 'GarageAge', 'GarageAgeLin'],axis=1).values
y_train_hasgb = df_hasgb['SalePrice'].values
x_train_hasgb = df_hasgb.drop(['HouseId', 'SalePrice'],axis=1).values
LassoLars had the best results from our modeling experiments, so we'll continue using it, but we'll hand tune the L1 regularization parameter.
In [14]:
x_train = x_train_nogorb
y_train = y_train_nogorb
In [55]:
lassolars_regr = linear_model.LassoLars(max_iter=50000)
lassolars_params = {'alpha': np.arange(0.00008, 0.00014, 0.00001).tolist()}
lassolars_df = single_search(lassolars_regr, lassolars_params)
lassolars_df.plot(x = ['alpha'], y = ['RMSE'])
lassolars_df.sort_values(['RMSE'], ascending = False)
Out[55]:
In [15]:
ll_nogorb_regr = linear_model.LassoLars(alpha=0.00009, max_iter=50000)
In [57]:
x_train = x_train_hasgb
y_train = y_train_hasgb
In [60]:
lassolars_regr = linear_model.LassoLars(max_iter=50000)
lassolars_params = {'alpha': np.arange(0.00004, 0.00011, 0.00001).tolist()}
lassolars_df = single_search(lassolars_regr, lassolars_params)
lassolars_df.plot(x = ['alpha'], y = ['RMSE'])
lassolars_df.sort_values(['RMSE'], ascending = False)
Out[60]:
In [16]:
ll_hasgb_regr = linear_model.LassoLars(alpha=0.00008, max_iter=50000)
In [17]:
ll_hasgb_regr.fit(x_train_hasgb, y_train_hasgb)
Out[17]:
In [22]:
lassolars_features = zip(df_hasgb.drop(['HouseId','SalePrice'],axis=1).columns, ll_hasgb_regr.coef_)
lassolars_features_df = pd.DataFrame.from_dict(lassolars_features)
lassolars_features_df.columns = ["Feature", "Coeff"]
lassolars_features_df = lassolars_features_df[lassolars_features_df["Coeff"]!=0]
lassolars_features_df["sort_ind"] = abs(lassolars_features_df["Coeff"])
lassolars_features_df = lassolars_features_df.sort_values(by="sort_ind", ascending = False)
lassolars_features_df = lassolars_features_df.drop('sort_ind', 1)
In [23]:
lassolars_features_df
Out[23]:
In [24]:
ll_nogorb_regr.fit(x_train_nogorb, y_train_nogorb)
Out[24]:
In [25]:
lassolars_features = zip(df_nogorb.drop(['HouseId','SalePrice'],axis=1).columns, ll_nogorb_regr.coef_)
lassolars_features_df = pd.DataFrame.from_dict(lassolars_features)
lassolars_features_df.columns = ["Feature", "Coeff"]
lassolars_features_df = lassolars_features_df[lassolars_features_df["Coeff"]!=0]
lassolars_features_df["sort_ind"] = abs(lassolars_features_df["Coeff"])
lassolars_features_df = lassolars_features_df.sort_values(by="sort_ind", ascending = False)
lassolars_features_df = lassolars_features_df.drop('sort_ind', 1)
In [26]:
lassolars_features_df
Out[26]:
Making Predictions on the Test Set¶
We want to be able to reassemble subset predictions on the test set back into a dataframe matched on the correct HouseId.
In [110]:
test_df_hasgb = test_df[(test_df['NoGarage']==0) & (test_df['NoBasement']==0)].reset_index(drop=True)
test_df_nogorb = test_df[(test_df['NoGarage']==1) | (test_df['NoBasement']==1)].reset_index(drop=True)
In [89]:
x_test_nogorb = test_df_nogorb.drop(['HouseId', 'GarageAge', 'GarageAgeLin'],axis=1).values
x_test_hasgb = test_df_hasgb.drop(['HouseId'],axis=1).values
In [139]:
y_pred_hasgb = ll_hasgb_regr.predict(x_test_hasgb)
y_pred_nogorb = ll_nogorb_regr.predict(x_test_nogorb)
y_pred_hasgb = np.exp(y_pred_hasgb)
y_pred_nogorb = np.exp(y_pred_nogorb)
In [140]:
ids_hasgb = test_df_hasgb.loc[:,'HouseId'].tolist()
ids_nogorb = test_df_nogorb.loc[:,'HouseId'].tolist()
In [141]:
nogorb_preds = zip(ids_nogorb, y_pred_nogorb)
nogorb_preds_df = pd.DataFrame.from_dict(nogorb_preds)
nogorb_preds_df.columns = ["HouseId", "SalePrice"]
hasgb_preds = zip(ids_hasgb, y_pred_hasgb)
hasgb_preds_df = pd.DataFrame.from_dict(hasgb_preds)
hasgb_preds_df.columns = ["HouseId", "SalePrice"]
In [142]:
preds_df = pd.concat([nogorb_preds_df,hasgb_preds_df]).sort_values('HouseId').set_index('HouseId')
In [144]:
preds_df.to_csv('subset_output-111001001.csv', header=True, index_label='Id')
This results in an error of 0.11998 on the public leaderboard, for a position of 936th on 2/20/17.
Blending with XGBoost and MLP¶
We can try to do a bit better by fitting XGBoost and MLP regressors to the data and making additional predictions.
XGBoost¶
In [145]:
x_train = x_train_nogorb
y_train = y_train_nogorb
In [161]:
xgb_regr = xgb.XGBRegressor(
max_depth = 3,
min_child_weight = 6.25,
gamma = 0.0005,
subsample = 0.4,
colsample_bytree = 0.3,
reg_alpha = 0.1,
reg_lambda = 3.25,
learning_rate = 0.06,
n_estimators = 500,
objective='reg:linear',
seed = 42,
nthread = -1,
silent = 1)
xgb_params = {'max_depth': np.arange(1, 9, 1).tolist()}
xgb_df = single_search(xgb_regr, xgb_params)
xgb_df.plot(x = ['max_depth'], y = ['RMSE'])
xgb_df.sort_values(['RMSE'], ascending = False)
Out[161]:
In [206]:
xgb_nogorb_regr = xgb.XGBRegressor(
max_depth = 3,
min_child_weight = 6.25,
gamma = 0.0005,
subsample = 0.4,
colsample_bytree = 0.3,
reg_alpha = 0.1,
reg_lambda = 3.25,
learning_rate = 0.06,
n_estimators = 500,
objective='reg:linear',
seed = 42,
nthread = -1,
silent = 1)
In [163]:
x_train = x_train_hasgb
y_train = y_train_hasgb
In [179]:
xgb_regr = xgb.XGBRegressor(
max_depth = 2,
min_child_weight = 1,
gamma = 0.001,
subsample = 0.7,
colsample_bytree = 0.105,
reg_alpha = 0.35,
reg_lambda = 2.5,
learning_rate = 0.06,
n_estimators = 800,
objective='reg:linear',
seed = 42,
nthread = -1,
silent = 1)
xgb_params = {'max_depth': np.arange(1, 6, 1).tolist()}
xgb_df = single_search(xgb_regr, xgb_params)
xgb_df.plot(x = ['max_depth'], y = ['RMSE'])
xgb_df.sort_values(['RMSE'], ascending = False)
Out[179]:
In [207]:
xgb_hasgb_regr = xgb.XGBRegressor(
max_depth = 2,
min_child_weight = 1,
gamma = 0.001,
subsample = 0.7,
colsample_bytree = 0.105,
reg_alpha = 0.35,
reg_lambda = 2.5,
learning_rate = 0.06,
n_estimators = 800,
objective='reg:linear',
seed = 42,
nthread = -1,
silent = 1)
In [208]:
xgb_nogorb_regr.fit(x_train_nogorb, y_train_nogorb)
xgb_hasgb_regr.fit(x_train_hasgb, y_train_hasgb)
Out[208]:
Multilevel Perceptron¶
In [181]:
x_train = x_train_nogorb
y_train = y_train_nogorb
In [190]:
mlp_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(9, ),
alpha=0.0004,
tol=0.0004,
max_iter=250)
mlp_params = {'max_iter': np.arange(25, 325, 25).tolist()}
# mlp_params = {'tol': np.logspace(-4, -2, 11).tolist()}
mlp_df = single_search(mlp_regr, mlp_params)
mlp_df.plot(x = ['max_iter'], y = ['RMSE'])
mlp_df.sort_values(['RMSE'], ascending = False)
Out[190]:
In [191]:
mlp_nogorb_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(9, ),
alpha=0.0004,
tol=0.0004,
max_iter=250)
In [192]:
x_train = x_train_hasgb
y_train = y_train_hasgb
In [199]:
mlp_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(2, ),
alpha=0.0005,
tol=0.0004,
max_iter=250)
mlp_params = {'max_iter': np.arange(300, 425, 25).tolist()}
# mlp_params = {'tol': np.logspace(-4, -2, 11).tolist()}
mlp_df = single_search(mlp_regr, mlp_params)
mlp_df.plot(x = ['max_iter'], y = ['RMSE'])
mlp_df.sort_values(['RMSE'], ascending = False)
Out[199]:
In [200]:
mlp_hasgb_regr = MLPRegressor(activation='relu', solver='lbfgs', random_state=641,
hidden_layer_sizes=(2, ),
alpha=0.0005,
tol=0.0004,
max_iter=300)
In [203]:
mlp_nogorb_regr.fit(x_train_nogorb, y_train_nogorb)
mlp_hasgb_regr.fit(x_train_hasgb, y_train_hasgb)
Out[203]:
Blending Predictions¶
In [211]:
y_pred_hasgb_ll = ll_hasgb_regr.predict(x_test_hasgb)
y_pred_nogorb_ll = ll_nogorb_regr.predict(x_test_nogorb)
y_pred_hasgb_xgb = xgb_hasgb_regr.predict(x_test_hasgb)
y_pred_nogorb_xgb = xgb_nogorb_regr.predict(x_test_nogorb)
y_pred_hasgb_mlp = mlp_hasgb_regr.predict(x_test_hasgb)
y_pred_nogorb_mlp = mlp_nogorb_regr.predict(x_test_nogorb)
We will weight the predictions according to the cross-validation errors.
In [213]:
y_pred_hasgb = (y_pred_hasgb_ll / 0.0998 + y_pred_hasgb_xgb / 0.1053
+ y_pred_hasgb_mlp / 0.1082) / (1 / 0.0998 + 1 / 0.1053 + 1 / 0.1082)
y_pred_nogorb = (y_pred_nogorb_ll / 0.1044 + y_pred_nogorb_xgb / 0.11022
+ y_pred_nogorb_mlp / 0.1166) / (1 / 0.1044 + 1 / 0.11022 + 1 / 0.1166)
y_pred_hasgb = np.exp(y_pred_hasgb)
y_pred_nogorb = np.exp(y_pred_nogorb)
In [214]:
ids_hasgb = test_df_hasgb.loc[:,'HouseId'].tolist()
ids_nogorb = test_df_nogorb.loc[:,'HouseId'].tolist()
In [215]:
nogorb_preds = zip(ids_nogorb, y_pred_nogorb)
nogorb_preds_df = pd.DataFrame.from_dict(nogorb_preds)
nogorb_preds_df.columns = ["HouseId", "SalePrice"]
hasgb_preds = zip(ids_hasgb, y_pred_hasgb)
hasgb_preds_df = pd.DataFrame.from_dict(hasgb_preds)
hasgb_preds_df.columns = ["HouseId", "SalePrice"]
In [216]:
preds_df = pd.concat([nogorb_preds_df,hasgb_preds_df]).sort_values('HouseId').set_index('HouseId')
In [217]:
preds_df.to_csv('subset_blend_output-111001001.csv', header=True, index_label='Id')
This yielded a 0.12211 error on the public leaderboard, which was a worse result. This is likely due to comparably larger variance of the XGB and MLP results, as evidenced by the cross-validation error on the training set.
Elastic Net¶
Since the tree and NN models didn't seem to do so well, it might be useful to blend with a different linear model to reduce some variance error. The Elastic Net was the 2nd performer behind LassoLars, so it is a natural choice.
In [218]:
x_train = x_train_nogorb
y_train = y_train_nogorb
In [220]:
elnet_regr = linear_model.ElasticNet(alpha=0.0006, l1_ratio=0.5, max_iter=15000, random_state=7)
# elnet_params = {'alpha': np.arange(0.0001, 0.0008, 0.0001).tolist()}
elnet_params = {'l1_ratio': np.arange(0.1, 1, 0.1).tolist()}
elnet_df = single_search(elnet_regr, elnet_params)
# elnet_df.plot(x = ['alpha'], y = ['RMSE'])
elnet_df.plot(x = ['l1_ratio'], y = ['RMSE'])
elnet_df.sort_values(['RMSE'], ascending = False)
Out[220]:
In [221]:
elnet_nogorb_regr = linear_model.ElasticNet(alpha=0.0006, l1_ratio=0.5, max_iter=15000, random_state=7)
In [222]:
x_train = x_train_hasgb
y_train = y_train_hasgb
In [224]:
elnet_regr = linear_model.ElasticNet(alpha=0.00055, l1_ratio=0.5, max_iter=15000, random_state=7)
# elnet_params = {'alpha': np.arange(0.0001, 0.0008, 0.0001).tolist()}
elnet_params = {'l1_ratio': np.arange(0.1, 1, 0.1).tolist()}
elnet_df = single_search(elnet_regr, elnet_params)
# elnet_df.plot(x = ['alpha'], y = ['RMSE'])
elnet_df.plot(x = ['l1_ratio'], y = ['RMSE'])
elnet_df.sort_values(['RMSE'], ascending = False)
Out[224]:
In [225]:
elnet_hasgb_regr = linear_model.ElasticNet(alpha=0.00055, l1_ratio=0.6, max_iter=15000, random_state=7)
In [226]:
elnet_nogorb_regr.fit(x_train_nogorb, y_train_nogorb)
elnet_hasgb_regr.fit(x_train_hasgb, y_train_hasgb)
Out[226]:
In [227]:
y_pred_hasgb_elnet = elnet_hasgb_regr.predict(x_test_hasgb)
y_pred_nogorb_elnet = elnet_nogorb_regr.predict(x_test_nogorb)
In [228]:
y_pred_hasgb = (y_pred_hasgb_ll / 0.0998 + y_pred_hasgb_elnet / 0.1015 ) / (1 / 0.0998 + 1 / 0.1015)
y_pred_nogorb = (y_pred_nogorb_ll / 0.1044 + y_pred_nogorb_elnet / 0.1061) / (1 / 0.1044 + 1 / 0.1061)
y_pred_hasgb = np.exp(y_pred_hasgb)
y_pred_nogorb = np.exp(y_pred_nogorb)
In [229]:
nogorb_preds = zip(ids_nogorb, y_pred_nogorb)
nogorb_preds_df = pd.DataFrame.from_dict(nogorb_preds)
nogorb_preds_df.columns = ["HouseId", "SalePrice"]
hasgb_preds = zip(ids_hasgb, y_pred_hasgb)
hasgb_preds_df = pd.DataFrame.from_dict(hasgb_preds)
hasgb_preds_df.columns = ["HouseId", "SalePrice"]
In [230]:
preds_df = pd.concat([nogorb_preds_df,hasgb_preds_df]).sort_values('HouseId').set_index('HouseId')
In [231]:
preds_df.to_csv('subset_linearblend_output-111001001.csv', header=True, index_label='Id')
This result gets 0.12049 on the public leaderboard, which is better than the multimodel blend above, but not as good as the original LassoLars entry.
In [ ]: