Exercise 6.9

(a)

import numpy as np
import pandas as pd
from sklearn.cross_validation import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeCV, LassoCV
from sklearn import preprocessing

df = pd.read_csv('../data/College.csv')

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 777 entries, 0 to 776
Data columns (total 19 columns):
Unnamed: 0     777 non-null object
Private        777 non-null object
Apps           777 non-null int64
Accept         777 non-null int64
Enroll         777 non-null int64
Top10perc      777 non-null int64
Top25perc      777 non-null int64
F.Undergrad    777 non-null int64
P.Undergrad    777 non-null int64
Outstate       777 non-null int64
Room.Board     777 non-null int64
Books          777 non-null int64
Personal       777 non-null int64
PhD            777 non-null int64
Terminal       777 non-null int64
S.F.Ratio      777 non-null float64
perc.alumni    777 non-null int64
Expend         777 non-null int64
Grad.Rate      777 non-null int64
dtypes: float64(1), int64(16), object(2)
memory usage: 115.4+ KB

df.head()

df.describe()

For a ridge and lasso regression, we should normalize the predictors.

However, what should one do for categorical variables?

This is an interesting discussion, as per these 2 questions on 'Cross Validated': https://stats.stackexchange.com/questions/59392/should-you-ever-standardise-binary-variables https://stats.stackexchange.com/questions/69568/whether-to-rescale-indicator-binary-dummy-predictors-for-lasso

For the sake of simplicity we will simply drop the variable 'private_yes' (perhaps a future blog post on the issue?).

X = df.iloc[:,3:]
X = X.iloc[:,:-1]

X.head()

y = df['Apps']

y.head()

0    1660
1    2186
2    1428
3     417
4     193
Name: Apps, dtype: int64

X_train,  X_test,  y_train,  y_test  =  train_test_split(X, y, random_state=0)

(b)

lr = LinearRegression()
lr.fit(X_train, y_train)

/Users/disciplina/anaconda3/envs/islp/lib/python3.6/site-packages/scipy/linalg/basic.py:1226: RuntimeWarning: internal gelsd driver lwork query error, required iwork dimension not returned. This is likely the result of LAPACK bug 0038, fixed in LAPACK 3.2.2 (released July 21, 2010). Falling back to 'gelss' driver.
  warnings.warn(mesg, RuntimeWarning)





LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)

np.mean((lr.predict(X_test) - y_test) ** 2)

1040120.0912491741

lr.score(X_test, y_test)

0.8985132475318234

(c)

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

scaler = preprocessing.StandardScaler().fit(X_train)
X_train_scaled = scaler.transform(X_train) 
X_test_scaled = scaler.transform(X_test) 

rcv = RidgeCV(alphas=np.linspace(.01, 100, 1000),  cv=10)

rcv.fit(X_train_scaled, y_train) 

RidgeCV(alphas=array([1.00000e-02, 1.10090e-01, ..., 9.98999e+01, 1.00000e+02]),
    cv=10, fit_intercept=True, gcv_mode=None, normalize=False,
    scoring=None, store_cv_values=False)

rcv.score(X_test_scaled,y_test)

0.9014568714885173

np.mean((rcv.predict(scaler.transform(X_test)) - y_test) ** 2)

1009951.4008144322

(d)

lcv = LassoCV(alphas=np.linspace(.01, 100, 1000),  cv=10)

lcv.fit(X_train_scaled, y_train) 

LassoCV(alphas=array([1.00000e-02, 1.10090e-01, ..., 9.98999e+01, 1.00000e+02]),
    copy_X=True, cv=10, eps=0.001, fit_intercept=True, max_iter=1000,
    n_alphas=100, n_jobs=1, normalize=False, positive=False,
    precompute='auto', random_state=None, selection='cyclic', tol=0.0001,
    verbose=False)

np.mean((lcv.predict(scaler.transform(X_test)) - y_test) ** 2)

1023952.7019786208

lcv.score(X_test_scaled,y_test)

0.9000907344458458