123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297 |
- import pandas as pd
- import numpy as np
- import matplotlib.pyplot as plt
- import seaborn as sns
- from sklearn import ensemble, tree, linear_model, preprocessing
- from sklearn.preprocessing import LabelEncoder, RobustScaler
- from sklearn.linear_model import ElasticNet, Lasso
- from sklearn.kernel_ridge import KernelRidge
- from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
- from sklearn.model_selection import KFold, train_test_split, cross_val_score
- from sklearn.metrics import r2_score, mean_squared_error
- from sklearn.pipeline import make_pipeline
- from sklearn.utils import shuffle
- from scipy import stats
- from scipy.stats import norm, skew, boxcox
- from scipy.special import boxcox1p
- import xgboost as xgb
- import lightgbm as lgb
- import warnings
- warnings.filterwarnings('ignore')
- class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
- def __init__(self, models):
- self.models = models
-
- def fit(self, X, y):
- self.models_ = [clone(x) for x in self.models]
-
- for model in self.models_:
- model.fit(X, y)
- return self
-
- def predict(self, X):
- predictions = np.column_stack([
- model.predict(X) for model in self.models_
- ])
- return np.mean(predictions, axis=1)
- train = pd.read_csv("../../train.csv")
- test = pd.read_csv("../../test.csv")
- train_ID = train['Id']
- test_ID = test['Id']
- train.drop('Id', axis=1, inplace=True)
- test.drop('Id', axis=1, inplace=True)
- fig, ax = plt.subplots()
- ax.scatter(x = train['GrLivArea'], y = train['SalePrice'])
- plt.ylabel('SalePrice', fontsize=13)
- plt.xlabel('GrLivArea', fontsize=13)
- train = train.drop(train[(train['GrLivArea'] > 4000) & (train['SalePrice'] < 300000)].index)
- fig, ax = plt.subplots()
- ax.scatter(train['GrLivArea'], train['SalePrice'])
- plt.ylabel('SalePrice', fontsize=13)
- plt.xlabel('GrLivArea', fontsize=13)
- train["SalePrice"] = np.log1p(train["SalePrice"])
- sns.distplot(train['SalePrice'] , fit=norm);
- (mu, sigma) = norm.fit(train['SalePrice'])
- print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))
- plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
- loc='best')
- plt.ylabel('Frequency')
- plt.title('SalePrice distribution')
- fig = plt.figure()
- res = stats.probplot(train['SalePrice'], plot=plt)
- train_labels = train.pop('SalePrice')
- features = pd.concat([train, test], keys=['train', 'test'])
- total_missing_data = features.isnull().sum().sort_values(ascending=False)
- missing_data_percent = (features.isnull().sum()/features.isnull().count()).sort_values(ascending=False)
- missing_data = pd.concat([total_missing_data, missing_data_percent], axis=1, keys=['Total', 'Percent'])
- print(missing_data[missing_data['Percent']> 0])
- features.drop(['Utilities'], axis=1, inplace=True)
- features.OverallCond = features.OverallCond.astype(str)
- features['MSSubClass'] = features['MSSubClass'].astype(str)
- features['MSZoning'] = features['MSZoning'].fillna(features['MSZoning'].mode()[0])
- median = features.groupby('OverallQual')['LotFrontage'].transform('median')
- features['LotFrontage'] = features['LotFrontage'].fillna(median)
- features['Alley'] = features['Alley'].fillna('NOACCESS')
- median = features.groupby('OverallQual')['MasVnrArea'].transform('median')
- features['MasVnrArea'] = features['MasVnrArea'].fillna(median)
- features['MasVnrType'] = features['MasVnrType'].fillna(features['MasVnrType'].mode()[0])
- for col in ('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
- features[col] = features[col].fillna('NoBSMT')
- features['TotalBsmtSF'] = features['TotalBsmtSF'].fillna(0)
- features['Electrical'] = features['Electrical'].fillna(features['Electrical'].mode()[0])
- features['KitchenAbvGr'] = features['KitchenAbvGr'].astype(str)
- features['KitchenQual'] = features['KitchenQual'].fillna(features['KitchenQual'].mode()[0])
- features['FireplaceQu'] = features['FireplaceQu'].fillna('NoFP')
- for col in ('GarageType', 'GarageFinish', 'GarageQual', 'GarageYrBlt', 'GarageCond'):
- features[col] = features[col].fillna('NoGRG')
- for col in ('GarageCars', 'GarageArea'):
- features[col] = features[col].fillna(0.0)
- features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode()[0])
- features['PoolQC'] = features['PoolQC'].fillna('NoPool')
- features['MiscFeature'] = features['MiscFeature'].fillna('None')
- features['Fence'] = features['Fence'].fillna('NoFence')
- for col in ('BsmtHalfBath', 'BsmtFullBath'):
- features[col] = features[col].fillna(0)
- features['Functional'] = features['Functional'].fillna('Typ')
- for col in ('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF'):
- features[col] = features[col].fillna(0)
- for col in ('Exterior1st', 'Exterior2nd'):
- features[col] = features[col].fillna(features[col].mode()[0])
- features['YrSold'] = features['YrSold'].astype(str)
- features['MoSold'] = features['MoSold'].astype(str)
- features['TotalSF'] = features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
- numeric_feats = features.dtypes[features.dtypes != "object"].index
- skewed_feats = features[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
- print("\nSkew in numerical features:")
- skewness = pd.DataFrame({'Skew' :skewed_feats})
- skewness.head(10)
- skewness = skewness[abs(skewness) > 0.75]
- print("There are {} skewed numerical features to Box Cox transform\n".format(skewness.shape[0]))
- from scipy.special import boxcox1p
- skewed_features = skewness.index
- lam = 0.15
- for feat in skewed_features:
- features[feat] = boxcox1p(features[feat], lam)
- categorical_features = ('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
- 'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
- 'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
- 'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
- 'YrSold', 'MoSold')
- lbl = LabelEncoder()
- for col in categorical_features:
- lbl.fit(list(features[col].values))
- features[col] = lbl.transform(list(features[col].values))
- features = pd.get_dummies(features)
- train_features = features.loc['train'].select_dtypes(include=[np.number]).values
- test_features = features.loc['test'].select_dtypes(include=[np.number]).values
- n_folds = 5
- def rmsle_cv(model):
- kf = KFold(n_folds, shuffle=True, random_state=101010).get_n_splits(train_features)
- rmse= np.sqrt(-cross_val_score(model, train_features, train_labels, scoring="neg_mean_squared_error", cv = kf))
- return(rmse)
- enet_model = make_pipeline(RobustScaler(), ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=101010))
- gb_model = ensemble.GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
- max_depth=4, max_features='sqrt',
- min_samples_leaf=15, min_samples_split=10,
- loss='huber', random_state =101010)
- lasso_model = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, random_state=101010))
- krr_model = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
- averaged_models = AveragingModels(models = (gb_model, enet_model, lasso_model, krr_model))
- print("AVERAGED MODELS")
- score = rmsle_cv(averaged_models)
- print("\nRMSLE: {:.4f} (+/- {:.4f})\n".format(score.mean(), score.std()))
- averaged_models.fit(train_features, train_labels)
- final_labels = np.exp(averaged_models.predict(test_features))
- pd.DataFrame({'Id': test_ID, 'SalePrice': final_labels}).to_csv('submission-9.csv', index =False)
|