{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-11T14:57:46.228412Z","iopub.execute_input":"2022-07-11T14:57:46.22885Z","iopub.status.idle":"2022-07-11T14:57:46.236803Z","shell.execute_reply.started":"2022-07-11T14:57:46.228817Z","shell.execute_reply":"2022-07-11T14:57:46.235228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1.ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"# numpy , pandas\nimport numpy as np \nimport pandas as pd\n# scikit-learn\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import Lasso\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n# 可視化用ライブラリ\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n#pandasのカラムが100列まで見れるようにする\npd.set_option('display.max_columns', 100)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:46.238498Z","iopub.execute_input":"2022-07-11T14:57:46.239469Z","iopub.status.idle":"2022-07-11T14:57:46.253155Z","shell.execute_reply.started":"2022-07-11T14:57:46.239436Z","shell.execute_reply":"2022-07-11T14:57:46.25196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.データを読み込む","metadata":{}},{"cell_type":"code","source":"# 学習データの読み込み\ntrain_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv',index_col=0)\ntest_data = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv',index_col=0)\n# 先頭5行をみてみる。\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:46.254743Z","iopub.execute_input":"2022-07-11T14:57:46.255853Z","iopub.status.idle":"2022-07-11T14:57:46.357701Z","shell.execute_reply.started":"2022-07-11T14:57:46.255818Z","shell.execute_reply":"2022-07-11T14:57:46.356559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"3.EDA（探索的データ分析）","metadata":{}},{"cell_type":"code","source":"# 売却価格のヒストグラム\nsns.distplot(train_data['SalePrice'])\n# 売却価格の概要をみてみる\nprint(train_data[\"SalePrice\"].describe())\nprint(f\"歪度: {round(train_data['SalePrice'].skew(),4)}\" )\nprint(f\"尖度: {round(train_data['SalePrice'].kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:46.360105Z","iopub.execute_input":"2022-07-11T14:57:46.360453Z","iopub.status.idle":"2022-07-11T14:57:46.714675Z","shell.execute_reply.started":"2022-07-11T14:57:46.360422Z","shell.execute_reply":"2022-07-11T14:57:46.713546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#敷地面積と住宅価格の散布図\nvar = 'LotArea'\ndata = pd.concat([train_data['SalePrice'], train_data[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));\n\n#50000以上の外れ値を除外すればよさそう","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:46.716968Z","iopub.execute_input":"2022-07-11T14:57:46.717833Z","iopub.status.idle":"2022-07-11T14:57:46.973102Z","shell.execute_reply.started":"2022-07-11T14:57:46.717796Z","shell.execute_reply":"2022-07-11T14:57:46.972105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#リビングのサイズと住宅価格の散布図\nvar = 'GrLivArea'\ndata = pd.concat([train_data['SalePrice'], train_data[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:46.974599Z","iopub.execute_input":"2022-07-11T14:57:46.974946Z","iopub.status.idle":"2022-07-11T14:57:47.232736Z","shell.execute_reply.started":"2022-07-11T14:57:46.974916Z","shell.execute_reply":"2022-07-11T14:57:47.231588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#地下室の総面積と住宅価格の散布図\nvar = 'TotalBsmtSF'\ndata = pd.concat([train_data['SalePrice'], train_data[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:47.234079Z","iopub.execute_input":"2022-07-11T14:57:47.235278Z","iopub.status.idle":"2022-07-11T14:57:47.498138Z","shell.execute_reply.started":"2022-07-11T14:57:47.235242Z","shell.execute_reply":"2022-07-11T14:57:47.497061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#建築年と住宅価格の箱ひげ図\nvar = 'YearBuilt'\ndata = pd.concat([train_data['SalePrice'], train_data[var]], axis=1)\nf, ax = plt.subplots(figsize=(16, 8))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0, ymax=800000);\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:47.499891Z","iopub.execute_input":"2022-07-11T14:57:47.500576Z","iopub.status.idle":"2022-07-11T14:57:51.697523Z","shell.execute_reply.started":"2022-07-11T14:57:47.500531Z","shell.execute_reply":"2022-07-11T14:57:51.696392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#材料と仕上がりの品質と住宅価格の箱ひげ図\nvar = 'OverallQual'\ndata = pd.concat([train_data['SalePrice'], train_data[var]], axis=1)\nf, ax = plt.subplots(figsize=(16, 8))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0, ymax=800000);\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:51.701737Z","iopub.execute_input":"2022-07-11T14:57:51.702112Z","iopub.status.idle":"2022-07-11T14:57:52.081269Z","shell.execute_reply.started":"2022-07-11T14:57:51.702077Z","shell.execute_reply":"2022-07-11T14:57:52.080468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#correlation matrix\ncorrmat = train_data.corr()\nf, ax = plt.subplots(figsize=(12, 9))\nsns.heatmap(corrmat, vmax=.8, square=True);","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:52.082368Z","iopub.execute_input":"2022-07-11T14:57:52.083404Z","iopub.status.idle":"2022-07-11T14:57:53.069317Z","shell.execute_reply.started":"2022-07-11T14:57:52.083365Z","shell.execute_reply":"2022-07-11T14:57:53.067991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#saleprice correlation matrix\nk = 10 #number of variables for heatmap\ncols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index #目的変数と創刊の高い上位10%の変数を抽出\ncm = np.corrcoef(train_data[cols].values.T)\n#相関関係係数の算出\nsns.set(font_scale=1.25)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:53.071167Z","iopub.execute_input":"2022-07-11T14:57:53.071663Z","iopub.status.idle":"2022-07-11T14:57:53.777568Z","shell.execute_reply.started":"2022-07-11T14:57:53.071601Z","shell.execute_reply":"2022-07-11T14:57:53.776444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* TotalBsmtSF：地下室の総平方フィート \n* GarageCars：車の容量でのガレージのサイズ\n* GarageArea：ガレージのサイズ（平方フィート）\n* GrLivArea：地上（地上）のリビングエリアの平方フィート\n* TotRmsAbvGrd：グレード以上の部屋の合計（バスルームは含まれません）","metadata":{}},{"cell_type":"markdown","source":"* OverallQual、GrLivArea、TotalBsmtSFはSalePriceと強い相関がある\n* GarageCarsとGarageAreaは強い相関があり、多重共線性の点から一つに絞る必要がある\n* TotalBsmtSFと1stFloorは強い相関があり、多重共線性の点から一つに絞る必要がある\n* FullBathもSalePriceに相関がある\n* YearBuiltもSalePriceに相関がある。時系列解析を使えば、より効果的な特徴量に変える事ができるかもしれないが、ここでは行わないことにする。","metadata":{}},{"cell_type":"code","source":"sns.set()\ncols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']\nsns.pairplot(train_data[cols], size = 2)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:57:53.779024Z","iopub.execute_input":"2022-07-11T14:57:53.779367Z","iopub.status.idle":"2022-07-11T14:58:02.57508Z","shell.execute_reply.started":"2022-07-11T14:57:53.779336Z","shell.execute_reply":"2022-07-11T14:58:02.574302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* 「TotalBsmtSF」と「GrLiveArea」のグラフでは、点が直線を描き、境界線のように機能している。これは、地下は地上のリビングと同じサイズにすることは可能だが、リビングよりも大きな地下室を作ることはできないためと考えられる。\n* SalePriceとYearBuiltのグラフを見ると、現在に近づくにつれて、指数関数的に増加しているがわかる。","metadata":{}},{"cell_type":"markdown","source":"特徴量エンジニアリング","metadata":{}},{"cell_type":"markdown","source":"まず、教師データとテストデータを結合する","metadata":{}},{"cell_type":"code","source":"#教師データとテストデータ、双方にエンジニアリングを行うため、一旦ひとつに結合\nall_data = pd.concat([train_data.drop(columns='SalePrice'), test_data], sort=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.576251Z","iopub.execute_input":"2022-07-11T14:58:02.576937Z","iopub.status.idle":"2022-07-11T14:58:02.602411Z","shell.execute_reply.started":"2022-07-11T14:58:02.57689Z","shell.execute_reply":"2022-07-11T14:58:02.60151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"欠損値の処理","metadata":{}},{"cell_type":"code","source":"all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\nall_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)[:30]\nmissing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\nmissing_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.603615Z","iopub.execute_input":"2022-07-11T14:58:02.604041Z","iopub.status.idle":"2022-07-11T14:58:02.637053Z","shell.execute_reply.started":"2022-07-11T14:58:02.604008Z","shell.execute_reply":"2022-07-11T14:58:02.636266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"欠損値の補完","metadata":{}},{"cell_type":"markdown","source":"PoolQC\n\nデータの説明（kaggle参照）によると、NAはプールなしを意味する。よって、Noneで補完する","metadata":{}},{"cell_type":"code","source":"all_data[\"PoolQC\"] = all_data[\"PoolQC\"].fillna(\"None\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.638037Z","iopub.execute_input":"2022-07-11T14:58:02.638928Z","iopub.status.idle":"2022-07-11T14:58:02.64595Z","shell.execute_reply.started":"2022-07-11T14:58:02.638881Z","shell.execute_reply":"2022-07-11T14:58:02.644845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MiscFeature\n\nデータの説明によると、NAは「その他の機能はない」を意味する。よって、Noneで補完する。","metadata":{}},{"cell_type":"code","source":"all_data[\"MiscFeature\"] = all_data[\"MiscFeature\"].fillna(\"None\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.647394Z","iopub.execute_input":"2022-07-11T14:58:02.648141Z","iopub.status.idle":"2022-07-11T14:58:02.658429Z","shell.execute_reply.started":"2022-07-11T14:58:02.648097Z","shell.execute_reply":"2022-07-11T14:58:02.657526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Alley, Fence, FireplaceQu, GarageType, GarageFinish, GarageQual, GarageCond\n\n同様にして、Alley, Fence, FireplaceQu, GarageType, GarageFinish, GarageQual, GarageCondの欠損値もNoneで補完する\n\n\ncols = ['Alley', \"Fence\", \"FireplaceQu\", 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']\n for col in cols:\n     all_data[col] = all_data[col].fillna('None')\n\n","metadata":{}},{"cell_type":"code","source":"cols = ['Alley', \"Fence\", \"FireplaceQu\", 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']\nfor col in cols:\n     all_data[col] = all_data[col].fillna('None')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.660095Z","iopub.execute_input":"2022-07-11T14:58:02.660806Z","iopub.status.idle":"2022-07-11T14:58:02.674277Z","shell.execute_reply.started":"2022-07-11T14:58:02.660762Z","shell.execute_reply":"2022-07-11T14:58:02.673396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Utilities\n\nUtilitiesの中身は”NoSeWa”1つとNA2つで構成されており、”NoSeWa”は学習データにしか存在しない。よって、このカラムはテストデータの予測には役に立たないので、カラムごと削除する。","metadata":{}},{"cell_type":"code","source":"all_data = all_data.drop(['Utilities'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.675425Z","iopub.execute_input":"2022-07-11T14:58:02.676083Z","iopub.status.idle":"2022-07-11T14:58:02.685415Z","shell.execute_reply.started":"2022-07-11T14:58:02.676048Z","shell.execute_reply":"2022-07-11T14:58:02.684365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Functional\n\nデータの説明によると、NAは一般的であることを示している。よって、欠損値は文字列”Typ”で補完する。\n\n\nall_data[\"Functional\"] = all_data[\"Functional\"].fillna(\"Typ\")\n\n","metadata":{}},{"cell_type":"code","source":"all_data[\"Functional\"] = all_data[\"Functional\"].fillna(\"Typ\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.686505Z","iopub.execute_input":"2022-07-11T14:58:02.687201Z","iopub.status.idle":"2022-07-11T14:58:02.694613Z","shell.execute_reply.started":"2022-07-11T14:58:02.687164Z","shell.execute_reply":"2022-07-11T14:58:02.693472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Electrical\n\n欠損値は1つだけ。最頻値(SBrkr)で補完する。","metadata":{}},{"cell_type":"code","source":"all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.696414Z","iopub.execute_input":"2022-07-11T14:58:02.696844Z","iopub.status.idle":"2022-07-11T14:58:02.705055Z","shell.execute_reply.started":"2022-07-11T14:58:02.696802Z","shell.execute_reply":"2022-07-11T14:58:02.703879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"KitchenQual\n\n欠損値は1つだけ。最頻値(TA)で補完する。","metadata":{}},{"cell_type":"code","source":"all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.706627Z","iopub.execute_input":"2022-07-11T14:58:02.707057Z","iopub.status.idle":"2022-07-11T14:58:02.717659Z","shell.execute_reply.started":"2022-07-11T14:58:02.707007Z","shell.execute_reply":"2022-07-11T14:58:02.716917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Exterior1st, Exterior2ndExterior1st, Exterior2nd","metadata":{}},{"cell_type":"code","source":"all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode()[0])\nall_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.719256Z","iopub.execute_input":"2022-07-11T14:58:02.719928Z","iopub.status.idle":"2022-07-11T14:58:02.730765Z","shell.execute_reply.started":"2022-07-11T14:58:02.719886Z","shell.execute_reply":"2022-07-11T14:58:02.729829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SaleType\n\n欠損値は1つだけ。最頻値で補完する。","metadata":{}},{"cell_type":"code","source":"all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode()[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.732478Z","iopub.execute_input":"2022-07-11T14:58:02.733146Z","iopub.status.idle":"2022-07-11T14:58:02.741517Z","shell.execute_reply.started":"2022-07-11T14:58:02.7331Z","shell.execute_reply":"2022-07-11T14:58:02.740697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MSSubClass\n\n欠損値は建物クラスなしに近い。よって、欠損値をNoneで補完する。","metadata":{}},{"cell_type":"code","source":"all_data['MSSubClass'] = all_data['MSSubClass'].fillna(\"None\")","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.749424Z","iopub.execute_input":"2022-07-11T14:58:02.749803Z","iopub.status.idle":"2022-07-11T14:58:02.755827Z","shell.execute_reply.started":"2022-07-11T14:58:02.74977Z","shell.execute_reply":"2022-07-11T14:58:02.754604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ごり押しで削除\n文字列の変数の欠損は「'None'」、数字の変数の欠損は「0」","metadata":{}},{"cell_type":"code","source":"# 変数の型ごとに欠損値の扱いが異なるため、変数ごとに処理\nfor column in all_data.columns:\n    # dtypeがobjectの場合、文字列の変数\n    if all_data[column].dtype=='O':\n        all_data[column] = all_data[column].fillna('None')\n    # dtypeがint , floatの場合、数字の変数\n    else:\n        all_data[column] = all_data[column].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.757123Z","iopub.execute_input":"2022-07-11T14:58:02.757498Z","iopub.status.idle":"2022-07-11T14:58:02.80714Z","shell.execute_reply.started":"2022-07-11T14:58:02.757467Z","shell.execute_reply":"2022-07-11T14:58:02.806153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"全ての欠損値の処理が完了。","metadata":{}},{"cell_type":"code","source":"# Check remaining missing values if any\nall_data_na = (all_data.isnull().sum() / len(all_data)) * 100\nall_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\nmissing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\nmissing_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.808288Z","iopub.execute_input":"2022-07-11T14:58:02.808897Z","iopub.status.idle":"2022-07-11T14:58:02.835485Z","shell.execute_reply.started":"2022-07-11T14:58:02.808865Z","shell.execute_reply":"2022-07-11T14:58:02.834466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"数値化","metadata":{}},{"cell_type":"code","source":"# pd.get_dummiesを使うとカテゴリ変数化できる。\nall_data = pd.get_dummies(all_data)\nall_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.836943Z","iopub.execute_input":"2022-07-11T14:58:02.837361Z","iopub.status.idle":"2022-07-11T14:58:02.944893Z","shell.execute_reply.started":"2022-07-11T14:58:02.837328Z","shell.execute_reply":"2022-07-11T14:58:02.943705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"trainデータとtestデータに戻す。","metadata":{}},{"cell_type":"code","source":"# 学習データと予測データに分割して元のデータフレームに戻す。\ntrain_data = pd.merge(all_data.iloc[train_data.index[0]:train_data.index[-1]],train_data['SalePrice'],left_index=True,right_index=True)\ntest_data = all_data.iloc[train_data.index[-1]:]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.946697Z","iopub.execute_input":"2022-07-11T14:58:02.9471Z","iopub.status.idle":"2022-07-11T14:58:02.96481Z","shell.execute_reply.started":"2022-07-11T14:58:02.94706Z","shell.execute_reply":"2022-07-11T14:58:02.963853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"外れ値の除去","metadata":{}},{"cell_type":"code","source":"train_data = train_data[(train_data['LotArea']<20000) & (train_data['SalePrice']<400000)& (train_data['YearBuilt']>1920)]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check remaining missing values if any\ntrain_data_na = (train_data.isnull().sum() / len(train_data)) * 100\ntrain_data_na = train_data_na.drop(train_data_na[train_data_na == 0].index).sort_values(ascending=False)\nmissing_data = pd.DataFrame({'Missing Ratio' :train_data_na})\nmissing_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.966388Z","iopub.execute_input":"2022-07-11T14:58:02.966751Z","iopub.status.idle":"2022-07-11T14:58:02.989531Z","shell.execute_reply.started":"2022-07-11T14:58:02.96672Z","shell.execute_reply":"2022-07-11T14:58:02.988785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check remaining missing values if any\ntest_data_na = (test_data.isnull().sum() / len(test_data)) * 100\ntest_data_na = test_data_na.drop(test_data_na[test_data_na == 0].index).sort_values(ascending=False)\nmissing_data = pd.DataFrame({'Missing Ratio' :test_data_na})\nmissing_data","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:02.991109Z","iopub.execute_input":"2022-07-11T14:58:02.991712Z","iopub.status.idle":"2022-07-11T14:58:03.015427Z","shell.execute_reply.started":"2022-07-11T14:58:02.991666Z","shell.execute_reply":"2022-07-11T14:58:03.01432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"住宅価格を対数変換","metadata":{}},{"cell_type":"code","source":"# 対数変換前のヒストグラム、歪度、尖度\nsns.distplot(train_data['SalePrice'])\nprint(f\"歪度: {round(train_data['SalePrice'].skew(),4)}\" )\nprint(f\"尖度: {round(train_data['SalePrice'].kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:03.016921Z","iopub.execute_input":"2022-07-11T14:58:03.017965Z","iopub.status.idle":"2022-07-11T14:58:03.349605Z","shell.execute_reply.started":"2022-07-11T14:58:03.017924Z","shell.execute_reply":"2022-07-11T14:58:03.348379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"対数変換して正規分布に","metadata":{}},{"cell_type":"code","source":"# SalePriceLogに対数変換した値を入れる。説明の都合上新たなカラムを作るが、基本的にそのまま代入して良い。\n# np.log()は底がeの対数変換を行う。\ntrain_data['SalePriceLog'] = np.log(train_data['SalePrice'])\n# 対数変換後のヒストグラム、歪度、尖度\nsns.distplot(train_data['SalePriceLog'])\nprint(f\"歪度: {round(train_data['SalePriceLog'].skew(),4)}\" )\nprint(f\"尖度: {round(train_data['SalePriceLog'].kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:03.351277Z","iopub.execute_input":"2022-07-11T14:58:03.352143Z","iopub.status.idle":"2022-07-11T14:58:03.680787Z","shell.execute_reply.started":"2022-07-11T14:58:03.352099Z","shell.execute_reply":"2022-07-11T14:58:03.679708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"学習データの説明変数と目的変数、予測データの説明変数にデータフレームを分割する。","metadata":{}},{"cell_type":"markdown","source":"いろいろモデルを読み込む","metadata":{}},{"cell_type":"code","source":"# 学習データ、説明変数\ntrain_X = train_data.drop(columns = ['SalePrice','SalePriceLog'])\n# 学習データ、目的変数\ntrain_y = train_data['SalePriceLog']\n\n# 予測データ、目的変数\ntest_X = test_data\ndef lasso_tuning(train_x,train_y):\n    # alphaパラメータのリスト\n    param_list = [0.001, 0.01, 0.1, 1.0, 10.0,100.0,1000.0] \n\n    for cnt,alpha in enumerate(param_list):\n        # パラメータを設定したラッソ回帰モデル\n        lasso = Lasso(alpha=alpha) \n        # パイプライン生成\n        pipeline = make_pipeline(StandardScaler(), lasso)\n\n        # 学習データ内でホールドアウト検証のために分割 テストデータの割合は0.3 seed値を0に固定\n        X_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size=0.3, random_state=0)\n\n        # 学習\n        pipeline.fit(X_train,y_train)\n\n        # RMSE(平均誤差)を計算\n        train_rmse = np.sqrt(mean_squared_error(y_train, pipeline.predict(X_train)))\n        test_rmse = np.sqrt(mean_squared_error(y_test, pipeline.predict(X_test)))\n        # ベストパラメータを更新\n        if cnt == 0:\n            best_score = test_rmse\n            best_param = alpha\n        elif best_score > test_rmse:\n            best_score = test_rmse\n            best_param = alpha\n\n    # ベストパラメータのalphaと、そのときのMSEを出力\n    print('alpha : ' + str(best_param))\n    print('test score is : ' +str(round(best_score,4)))\n\n    # ベストパラメータを返却\n    return best_param\n\n# best_alphaにベストパラメータのalphaが渡される。\nbest_alpha = lasso_tuning(train_X,train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:03.682657Z","iopub.execute_input":"2022-07-11T14:58:03.683057Z","iopub.status.idle":"2022-07-11T14:58:04.420074Z","shell.execute_reply.started":"2022-07-11T14:58:03.683018Z","shell.execute_reply":"2022-07-11T14:58:04.41892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ラッソ回帰モデルにベストパラメータを設定\nlasso = Lasso(alpha = best_alpha)\n# パイプラインの作成\npipeline = make_pipeline(StandardScaler(), lasso)\n# 学習\npipeline.fit(train_X,train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:04.422189Z","iopub.execute_input":"2022-07-11T14:58:04.424263Z","iopub.status.idle":"2022-07-11T14:58:04.544083Z","shell.execute_reply.started":"2022-07-11T14:58:04.424214Z","shell.execute_reply":"2022-07-11T14:58:04.542892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 結果を予測\npred = pipeline.predict(test_X)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:04.545885Z","iopub.execute_input":"2022-07-11T14:58:04.54657Z","iopub.status.idle":"2022-07-11T14:58:04.578977Z","shell.execute_reply.started":"2022-07-11T14:58:04.54652Z","shell.execute_reply":"2022-07-11T14:58:04.577698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 予測結果のプロット\nsns.distplot(pred)\n# 歪度と尖度\nprint(f\"歪度: {round(pd.Series(pred).skew(),4)}\" )\nprint(f\"尖度: {round(pd.Series(pred).kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:04.580894Z","iopub.execute_input":"2022-07-11T14:58:04.581681Z","iopub.status.idle":"2022-07-11T14:58:04.988999Z","shell.execute_reply.started":"2022-07-11T14:58:04.581599Z","shell.execute_reply":"2022-07-11T14:58:04.987881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 指数変換\npred_exp = np.exp(pred)\n# 指数変換した予測結果をプロット\nsns.distplot(pred_exp)\n# 歪度と尖度\nprint(f\"歪度: {round(pd.Series(pred_exp).skew(),4)}\" )\nprint(f\"尖度: {round(pd.Series(pred_exp).kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:04.990753Z","iopub.execute_input":"2022-07-11T14:58:04.991326Z","iopub.status.idle":"2022-07-11T14:58:05.35212Z","shell.execute_reply.started":"2022-07-11T14:58:04.991295Z","shell.execute_reply":"2022-07-11T14:58:05.351047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 400,000より高い物件は除去\npred_exp_ex_outliars = pred_exp[pred_exp<400000]\n# 指数変換した予測結果をプロット\nsns.distplot(pred_exp_ex_outliars)\n# 歪度と尖度\nprint(f\"歪度: {round(pd.Series(pred_exp_ex_outliars).skew(),4)}\" )\nprint(f\"尖度: {round(pd.Series(pred_exp_ex_outliars).kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:05.353718Z","iopub.execute_input":"2022-07-11T14:58:05.354037Z","iopub.status.idle":"2022-07-11T14:58:05.638547Z","shell.execute_reply.started":"2022-07-11T14:58:05.354007Z","shell.execute_reply":"2022-07-11T14:58:05.637518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習データの住宅価格をプロット(外れ値除去済み)\nsns.distplot(train_data['SalePrice'])\n# 歪度と尖度\nprint(f\"歪度: {round(pd.Series(train_data['SalePrice']).skew(),4)}\" )\nprint(f\"尖度: {round(pd.Series(train_data['SalePrice']).kurt(),4)}\" )","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:05.640294Z","iopub.execute_input":"2022-07-11T14:58:05.640659Z","iopub.status.idle":"2022-07-11T14:58:05.978871Z","shell.execute_reply.started":"2022-07-11T14:58:05.640602Z","shell.execute_reply":"2022-07-11T14:58:05.977673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.csvの読み込み\nsubmission_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')\n# sample_submission.csvの形式を確認するために先頭五行を見てみる。\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:05.980382Z","iopub.execute_input":"2022-07-11T14:58:05.980825Z","iopub.status.idle":"2022-07-11T14:58:05.996682Z","shell.execute_reply.started":"2022-07-11T14:58:05.980792Z","shell.execute_reply":"2022-07-11T14:58:05.995458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.csvを出力\nsubmission_df.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:58:05.998665Z","iopub.execute_input":"2022-07-11T14:58:05.999325Z","iopub.status.idle":"2022-07-11T14:58:06.009879Z","shell.execute_reply.started":"2022-07-11T14:58:05.999291Z","shell.execute_reply":"2022-07-11T14:58:06.008716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}