{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# numpy , pandas\nimport numpy as np \nimport pandas as pd\n# scikit-learn\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import Lasso\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n# 可視化用ライブラリ\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n#pandasのカラムが100列まで見れるようにする\npd.set_option('display.max_columns', 100)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Enginnering","metadata":{}},{"cell_type":"code","source":"def add_new_columns(df):\n    # 建物内の総面積 = 1階の面積 + 2階の面積 + 地下の面積\n    df[\"TotalSF\"] = df[\"1stFlrSF\"] + df[\"2ndFlrSF\"] + df[\"TotalBsmtSF\"]\n    \n    # 一部屋あたりの平均面積 = 建物の総面積 / 部屋数\n    df['AreaPerRoom'] = df['TotalSF']/df['TotRmsAbvGrd']\n    \n    # 築年数 + 最新リフォーム年 : この値が大きいほど値段が高くなりそう\n    df['YearBuiltPlusRemod']=df['YearBuilt']+df['YearRemodAdd']\n    \n    # お風呂の総面積\n    # Full bath : 浴槽、シャワー、洗面台、便器全てが備わったバスルーム\n    # Half bath : 洗面台、便器が備わった部屋)(シャワールームがある場合もある)\n    # シャワーがない場合を想定してHalf Bathには0.5の係数をつける\n    df['TotalBathrooms'] = (df['FullBath'] + (0.5 * df['HalfBath']) + df['BsmtFullBath'] + (0.5 * df['BsmtHalfBath']))\n    \n    # 合計の屋根付きの玄関の総面積 \n    # Porch : 屋根付きの玄関 日本風にいうと縁側\n    df['TotalPorchSF'] = (df['OpenPorchSF'] + df['3SsnPorch'] + df['EnclosedPorch'] + df['ScreenPorch'] + df['WoodDeckSF'])\n    \n    # プールの有無\n    df['HasPool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)\n    \n    # 2階の有無\n    df['Has2ndFloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)\n    \n    # ガレージの有無\n    df['HasGarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)\n    \n    # 地下室の有無\n    df['HasBsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lasso_tuning(train_x,train_y):\n    # alphaパラメータのリスト\n    param_list = [0.001, 0.01, 0.1, 1.0, 10.0,100.0,1000.0] \n    \n    for cnt,alpha in enumerate(param_list):\n        # パラメータを設定したラッソ回帰モデル\n        lasso = Lasso(alpha=alpha) \n        # パイプライン生成\n        pipeline = make_pipeline(StandardScaler(), lasso)\n        \n        # 学習データ内でホールドアウト検証のために分割 テストデータの割合は0.3 seed値を0に固定\n        X_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size=0.3, random_state=0)\n        \n        # 学習\n        pipeline.fit(X_train,y_train)\n        \n        # RMSE(平均誤差)を計算\n        train_rmse = np.sqrt(mean_squared_error(y_train, pipeline.predict(X_train)))\n        test_rmse = np.sqrt(mean_squared_error(y_test, pipeline.predict(X_test)))\n        # ベストパラメータを更新\n        if cnt == 0:\n            best_score = test_rmse\n            best_param = alpha\n        elif best_score > test_rmse:\n            best_score = test_rmse\n            best_param = alpha\n    \n    # ベストパラメータのalphaと、そのときのMSEを出力\n    print('alpha : ' + str(best_param))\n    print('test score is : ' +str(round(best_score,4)))\n    \n    # ベストパラメータを返却\n    return best_param","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"# 学習データの読み込み\ntrain_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',index_col=0)\n# 先頭5行をみてみる。\ntrain_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 売却価格のヒストグラム\nsns.distplot(train_df['SalePrice'])\n# 売却価格の概要をみてみる\nprint(train_df[\"SalePrice\"].describe())\nprint(f\"歪度: {round(train_df['SalePrice'].skew(),4)}\" )\nprint(f\"尖度: {round(train_df['SalePrice'].kurt(),4)}\" )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# 予測用データセットの読み込み\ntest_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',index_col=0)\nall_df = pd.concat([train_df.drop(columns='SalePrice'),test_df])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num2str_list = ['MSSubClass','YrSold','MoSold']\nfor column in num2str_list:\n    all_df[column] = all_df[column].astype(str)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 変数の型ごとに欠損値の扱いが異なるため、列ごとに処理\nfor column in all_df.columns:\n    # dtypeがobjectの場合、文字列の変数\n    if all_df[column].dtype=='O':\n        all_df[column] = all_df[column].fillna('None')\n    # dtypeがint , floatの場合、数字の変数\n    else:\n        all_df[column] = all_df[column].fillna(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_new_columns(all_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.get_dummiesを使うとカテゴリ変数できる。\nall_df = pd.get_dummies(all_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データと予測データに分割して元のデータフレームに戻す。\ntrain_df = pd.merge(all_df.iloc[train_df.index[0]:train_df.index[-1]],train_df['SalePrice'],left_index=True,right_index=True)\ntest_df = all_df.iloc[train_df.index[-1]:]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df[(train_df['LotArea']<20000) & (train_df['SalePrice']<400000)& (train_df['YearBuilt']>1920)]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SalePriceLogに対数変換した値を入れる。説明の都合上新たなカラムを作るが、基本的にそのまま代入して良い。\ntrain_df['SalePriceLog'] = np.log(train_df['SalePrice'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データ、説明変数\ntrain_X = train_df.drop(columns = ['SalePrice','SalePriceLog'])\n# 学習データ、目的変数\ntrain_y = train_df['SalePriceLog']\n\n# 予測データ、説明変数\ntest_X = test_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Building","metadata":{}},{"cell_type":"markdown","source":"# Hyper Parameter Turning","metadata":{}},{"cell_type":"code","source":"best_alpha = lasso_tuning(train_X,train_y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ラッソ回帰モデルにベストパラメータを設定\nlasso = Lasso(alpha = best_alpha)\n# パイプラインの作成\npipeline = make_pipeline(StandardScaler(), lasso)\n# 学習\npipeline.fit(train_X,train_y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = pipeline.predict(test_X)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 指数変換\npred_exp = np.exp(pred)\n# 指数変換した予測結果をプロット\nsns.distplot(pred_exp)\n# 歪度と尖度\nprint(f\"歪度: {round(pd.Series(pred_exp).skew(),4)}\" )\nprint(f\"尖度: {round(pd.Series(pred_exp).kurt(),4)}\" )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 400,000より高い物件は除去\npred_exp_ex_outliars = pred_exp[pred_exp<400000]\n# 指数変換した予測結果をプロット\nsns.distplot(pred_exp_ex_outliars)\n# 歪度と尖度\nprint(f\"歪度: {round(pd.Series(pred_exp_ex_outliars).skew(),4)}\" )\nprint(f\"尖度: {round(pd.Series(pred_exp_ex_outliars).kurt(),4)}\" )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Submission","metadata":{}},{"cell_type":"code","source":"# sample_submission.csvの読み込み\nsubmission_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')\n# sample_submission.csvの形式を確認するために先頭五行を見てみる。\nsubmission_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 指数変換した値を代入\nsubmission_df['SalePrice'] = pred_exp","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]}]}