{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nnp.random.seed(seed=42)\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nids = test.Id\n\n#prepare for analysis\ntrain = pd.get_dummies(train)\ntest = pd.get_dummies(test)\ntrain,test = train.align(test,join='left', axis=1)\n\n# from sklearn.preprocessing import Imputer\n# imputer = Imputer()\n# train = imputer.fit_transform(train)\n# test = imputer.fit_transform(test)\n\ntrain = train.dropna(axis=1, how='any') #drop missing values\ntest = test.dropna(axis=1, how='any')\nprint(train.shape,test.shape)\n\nY = train[\"SalePrice\"]\nX = train.drop([\"SalePrice\"], axis=1)\n# print(\"Normalizing data...\")\n# from sklearn import preprocessing\n# X = preprocessing.scale(X)\n# test = preprocessing.scale(test)\n\nprint(\"PCA feature selection\")\nfrom sklearn.decomposition import PCA\npca = PCA(n_components=260)\nX = pca.fit_transform(X)\ntest = pca.fit_transform(test)\nprint(X.shape,test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e711f437-caa9-4a4e-a703-ab54a2712618","_uuid":"7a80e57578856a09be7328d9b9f0a2a9d851259c","trusted":false},"cell_type":"code","source":"print(\"Linear regression\")\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.feature_selection import RFE\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error,r2_score\nfrom math import sqrt\n\n# rms = sqrt(mean_squared_error(y_actual, y_predicted))\n\nX_train,X_val,y_train,y_val = train_test_split(X,Y,test_size=0.1,random_state=42)\n\n\nreg = LinearRegression(fit_intercept=True, normalize=False, copy_X=True, n_jobs=1)\nfit = reg.fit(X_train,y_train)\ny_predicted = reg.predict(X_val)\nprint('Train R2',reg.score(X_train,y_train))\nprint('Val R2', r2_score(y_val,y_predicted))\nprint('Val RMSE', sqrt(mean_squared_error(y_val, y_predicted)))\nprint('Predicting on test set for submission...')\npredictions=reg.predict(test)\n\nprint(\"Writing predictions...\")\nsolution = pd.DataFrame({\"id\":ids, \"SalePrice\":predictions})\nsolution.to_csv(\"lin_reg.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fbf8af2f-03ca-4a24-82e0-51b25e63bee4","_uuid":"fb6aa1f5054ee42ea9f1e6f2459204687b0aa708","trusted":false},"cell_type":"code","source":"print(\"Ridge Regression\")\nfrom sklearn.linear_model import Ridge\n\nreg = Ridge(alpha=0.1, fit_intercept=True, normalize=False, \n            copy_X=True, max_iter=None, tol=0.001, solver='auto', random_state=42)\n#solver : {‘auto’, ‘svd’, ‘cholesky’, ‘lsqr’, ‘sparse_cg’, ‘sag’, ‘saga’}\n\nfit = reg.fit(X_train,y_train)\ny_predicted = reg.predict(X_val)\n\nprint('Train R2',reg.score(X_train,y_train))\nprint('Val R2', r2_score(y_val,y_predicted))\nprint('Val RMSE', sqrt(mean_squared_error(y_val, y_predicted)))\nprint('Predicting on test set for submission...')\npredictions=reg.predict(test)\n\nprint(\"Writing predictions...\")\nsolution = pd.DataFrame({\"id\":ids, \"SalePrice\":predictions})\nsolution.to_csv(\"ridge_reg.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6c9593c6-175b-4700-9af8-47a2d1fb302a","_uuid":"125ff2e40f6c87add3745b89fbe886b7a49f905b","trusted":false},"cell_type":"code","source":"print(\"Bayesian Ridge (winning model so far...)\")\nfrom sklearn.linear_model import BayesianRidge\n\nreg = BayesianRidge(n_iter=300, tol=0.001, alpha_1=1e-06, alpha_2=1e-06, \n                    lambda_1=1e-06, lambda_2=1e-06, compute_score=True, fit_intercept=True, \n                    normalize=False, copy_X=True, verbose=True)\n\nfit = reg.fit(X_train,y_train)\ny_predicted = reg.predict(X_val)\n\nprint('Train R2',reg.score(X_train,y_train))\nprint('Val R2', r2_score(y_val,y_predicted))\nprint('Val RMSE', sqrt(mean_squared_error(y_val, y_predicted)))\nprint('Predicting on test set for submission...')\npredictions=reg.predict(test)\n\nprint(\"Writing predictions...\")\nsolution = pd.DataFrame({\"id\":ids, \"SalePrice\":predictions})\nsolution.to_csv(\"bayes_ridge_reg.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"864c2d38-48db-4153-8686-19cd6673c8cf","_uuid":"f1f41ccde2297ea3626f281baf34fbd048627ff5","trusted":false},"cell_type":"code","source":"print(\"ElasticNet regression\")\nfrom sklearn.linear_model import ElasticNet\n\nreg = ElasticNet(alpha=0.001, l1_ratio=0.8, fit_intercept=True, \n                 normalize=False, precompute=True, max_iter=1000000, copy_X=True,\n                 tol=0.00001, warm_start=False, positive=False, random_state=42, selection='random')\n\nfit = reg.fit(X_train,y_train)\ny_predicted = reg.predict(X_val)\n\nprint('Train R2',reg.score(X_train,y_train))\nprint('Val R2', r2_score(y_val,y_predicted))\nprint('Val RMSE', sqrt(mean_squared_error(y_val, y_predicted)))\nprint('Predicting on test set for submission...')\npredictions=reg.predict(test)\n\nprint(\"Writing predictions...\")\nsolution = pd.DataFrame({\"id\":ids, \"SalePrice\":predictions})\nsolution.to_csv(\"elasticnet_reg.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b29a981e-2ede-49fe-9f52-0ae95dbafeaf","_uuid":"0fb6a7de45d759649c595802b31a7f71cf66031d","trusted":false},"cell_type":"code","source":"print(\"Bayesian ARD Regression (too long and no inmediate improvements)\")\n\n# from sklearn.linear_model import ARDRegression\n\n# reg = ARDRegression(n_iter=300, tol=0.001, alpha_1=1e-06, alpha_2=1e-06, lambda_1=1e-06, \n#               lambda_2=1e-06, compute_score=False, threshold_lambda=10000.0, fit_intercept=True, \n#               normalize=False, copy_X=True, verbose=True)\n\n# fit = reg.fit(X_train,y_train)\n# y_predicted = reg.predict(X_val)\n\n# print('Train R2',reg.score(X_train,y_train))\n# print('Val R2', r2_score(y_val,y_predicted))\n# print('Val RMSE', sqrt(mean_squared_error(y_val, y_predicted)))\n# print('Predicting on test set for submission...')\n# predictions=reg.predict(test)\n\n# print(\"Writing predictions...\")\n# solution = pd.DataFrame({\"id\":ids, \"SalePrice\":predictions})\n# solution.to_csv(\"elasticnet_reg.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"47726ff0-ba63-458a-857d-33f016b355d0","scrolled":true,"_uuid":"e1c89d6e5229f94a5e57fdd537f66474e5666dc5","trusted":false},"cell_type":"code","source":"print(\"Simple NN\")\nprint(X_train.shape)\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Dropout\nfrom matplotlib import pyplot as plt \nfrom keras import backend\n\ndef rmse(y_true, y_pred):\n    return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))\n\n# create model\nmodel = Sequential()\n# model.add(Dropout(0.2, input_shape=(X_train.shape[1],)))\nmodel.add(Dense(260, input_dim=X_train.shape[1], activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(1))\nmodel.compile(loss='mse', optimizer='adam', metrics=[rmse])\n# train model\nhistory = model.fit(X_train, y_train, epochs=2000, batch_size=8, verbose=2, validation_data=[X_val,y_val])\n# plot metrics\nplt.plot(history.history['rmse'])\nplt.plot(history.history['val_rmse'])\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()\n\ny_predicted = model.predict(X_val)\nprint('Val R2', r2_score(y_val,y_predicted))\nprint('Val RMSE', sqrt(mean_squared_error(y_val, y_predicted)))\nprint('Predicting on test set for submission...')\npredictions=model.predict(test)\npredictions = predictions[:, 0]\n\nprint(\"Writing predictions...\")\nsolution = pd.DataFrame({\"id\":ids, \"SalePrice\":predictions})\nsolution.to_csv(\"nn.csv\", index = False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a8552859-33e0-4d6e-a051-8dee0e2de73c","_uuid":"aebb5f30080ec4951802cdca0f32e27411a304bc","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4baef42d-a123-45be-9f15-08d51228ffad","collapsed":true,"_uuid":"8001c9f4c8b64c18cc559559ad42e3dd42f31e2b","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}