{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom scipy.stats import norm\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline\n\n# -----------------------------------\n# 学習データ、テストデータの読み込み\n# -----------------------------------\n# 学習データ、テストデータの読み込み\ntrain = pd.read_csv('../input/house-prices-advanced-regression-techniques/train.csv')\ntest = pd.read_csv('../input/house-prices-advanced-regression-techniques/test.csv')\n\n# 学習データを特徴量と目的変数に分ける\ntrain_x = train.drop(['SalePrice'], axis=1)\ntrain_y = train['SalePrice']\n\n# テストデータは特徴量のみなので、そのままでよい\ntest_x = test.copy()\n\n# 全体統計\nall_df = pd.concat([train,test])\n\nall_df.shape","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-06T04:34:07.759127Z","iopub.execute_input":"2022-07-06T04:34:07.760029Z","iopub.status.idle":"2022-07-06T04:34:07.838117Z","shell.execute_reply.started":"2022-07-06T04:34:07.759967Z","shell.execute_reply":"2022-07-06T04:34:07.837028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# デフォルトだと全部でない(最大20列)ので、表示列数を拡張、あまり多すぎる場合はやらないほうがいい\npd.set_option('display.max_columns', 100)\npd.set_option('display.max_rows', 100)\n\n# 大量のデータをの部分を見る場合は、 all_df.iloc[\"行の範囲orリスト\" , \"列の範囲orリスト\"] を用いて20エントリずつスライスして見るのがおすすめ\nall_df.iloc[:,0:5] #0列目〜4列目の(セパレータの場所なので最終表示列+1をになっている)、列について、全行 (:)の内容を対象","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:40:57.505975Z","iopub.execute_input":"2022-07-06T04:40:57.506446Z","iopub.status.idle":"2022-07-06T04:40:57.527098Z","shell.execute_reply.started":"2022-07-06T04:40:57.506407Z","shell.execute_reply":"2022-07-06T04:40:57.526219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ntrain = train.shape[0]\nntest = test.shape[0]\nprint(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:07.971280Z","iopub.execute_input":"2022-07-06T04:34:07.971713Z","iopub.status.idle":"2022-07-06T04:34:07.977661Z","shell.execute_reply.started":"2022-07-06T04:34:07.971681Z","shell.execute_reply":"2022-07-06T04:34:07.976457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# データの中身表示 ⇒ 右側のDataから各項目をクリックするとNotebookでいい感じに表示してくれた(元データ見るならheadより便利)\n# 統計量表示\n\ntrain.iloc[:,0:40].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.065130Z","iopub.execute_input":"2022-07-06T04:34:08.065690Z","iopub.status.idle":"2022-07-06T04:34:08.118018Z","shell.execute_reply.started":"2022-07-06T04:34:08.065655Z","shell.execute_reply":"2022-07-06T04:34:08.117043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[:,40:60].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.156915Z","iopub.execute_input":"2022-07-06T04:34:08.157695Z","iopub.status.idle":"2022-07-06T04:34:08.213705Z","shell.execute_reply.started":"2022-07-06T04:34:08.157646Z","shell.execute_reply":"2022-07-06T04:34:08.212653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[:,60:].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.253008Z","iopub.execute_input":"2022-07-06T04:34:08.253585Z","iopub.status.idle":"2022-07-06T04:34:08.304801Z","shell.execute_reply.started":"2022-07-06T04:34:08.253553Z","shell.execute_reply":"2022-07-06T04:34:08.303746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train総エントリ数1460、countが少なすぎるものや、count - freqの差が小さすぎるものはサンプル数が少なすぎる=使えないかもしれない\n#Streetはサンプル数が少ないかもしれない、Utilitiesは1個のみ値が異なりあと同じなので利用しないほうがよさそう\ntrain.iloc[:,0:20].describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.358159Z","iopub.execute_input":"2022-07-06T04:34:08.358727Z","iopub.status.idle":"2022-07-06T04:34:08.400180Z","shell.execute_reply.started":"2022-07-06T04:34:08.358693Z","shell.execute_reply":"2022-07-06T04:34:08.399340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[:,20:40].describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.480729Z","iopub.execute_input":"2022-07-06T04:34:08.481171Z","iopub.status.idle":"2022-07-06T04:34:08.528631Z","shell.execute_reply.started":"2022-07-06T04:34:08.481125Z","shell.execute_reply":"2022-07-06T04:34:08.527394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[:,40:70].describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.569379Z","iopub.execute_input":"2022-07-06T04:34:08.569804Z","iopub.status.idle":"2022-07-06T04:34:08.610345Z","shell.execute_reply.started":"2022-07-06T04:34:08.569766Z","shell.execute_reply":"2022-07-06T04:34:08.609129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#PoolQC,MiscFeatueはつかえないかも\ntrain.iloc[:,70:].describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.824149Z","iopub.execute_input":"2022-07-06T04:34:08.824543Z","iopub.status.idle":"2022-07-06T04:34:08.848355Z","shell.execute_reply.started":"2022-07-06T04:34:08.824512Z","shell.execute_reply":"2022-07-06T04:34:08.847358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#数値データなのだけど、ラベルとして扱う物を文字列変換しておく\n#for c in ['MSSubClass','YrSold','MoSold']:\n#    train[c] = train[c].astype(str)\n#    test[c] = test[c].astype(str)\n#    all_df[c] = all_df[c].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:08.924088Z","iopub.execute_input":"2022-07-06T04:34:08.925264Z","iopub.status.idle":"2022-07-06T04:34:08.930060Z","shell.execute_reply.started":"2022-07-06T04:34:08.925208Z","shell.execute_reply":"2022-07-06T04:34:08.929317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ラベルデータの表示順序リストを作っておく (True : 順序の意味あり)\nLabelData = {\n    'MSSubClass':(False,['20','30','40','45','50','60','70','75','80','85','90','120','150','160','180','190']),\n    'MSZoning':(False,['A','C','FV','I','RH','RL','RP','RM']),\n    'Street':(False,['Grvl','Pave']),\n    'Alley':(False,['Grvl','Pave','NA','None']),\n    'LotShape':(True,['Reg','IR1','IR2','IR3']),\n    'LandContour':(True,['Lvl','Bnk','HLS','Low']),\n    'Utilities':(False,['AllPub','NoSewr','NoSeWa','ELO']),\n    'LotConfig':(False,['Inside','Corner','CulDSac','FR2','FR3']),\n    'LandSlope':(True,['Gtl','Mod','Sev']),\n    'Neighborhood':(False,['Blmngtn','Blueste','BrDale','BrkSide','ClearCr','CollgCr','Crawfor','Edwards','Gilbert','IDOTRR','MeadowV','Mitchel','Names','NoRidge','NPkVill','NridgHt','NWAmes','OldTown','SWISU','Sawyer','SawyerW','Somerst','StoneBr','Timber','Veenker']),\n    'Condition1':(False,['Artery','Feedr','Norm','RRNn','RRAn','PosN','PosA','RRNe','RRAe']),\n    'Condition2':(False,['Artery','Feedr','Norm','RRNn','RRAn','PosN','PosA','RRNe','RRAe']),\n    'BldgType':(False,['1Fam','2FmCon','Duplx','TwnhsE','TwnhsI']),\n    'HouseStyle':(False,['1Story','1.5Fin','1.5Unf','2Story','2.5Fin','2.5Unf','SFoyer','SLvl']),\n    'RoofStyle':(False,['Flat','Gable','Gambrel','Hip','Mansard','Shed']),\n    'RoofMatl':(False,['ClyTile','CompShg','Membran','Metal','Roll','Tar&Grv','WdShake','WdShngl']),\n    'Exterior1st':(False,['AsbShng','AsphShn','BrkComm','BrkFace','CBlock','CemntBd','HdBoard','ImStucc','MetalSd','Other','Plywood','PreCast','Stone','Stucco','VinylSd','Wd','WdShing']),\n    'Exterior2nd':(False,['AsbShng','AsphShn','BrkComm','BrkFace','CBlock','CemntBd','HdBoard','ImStucc','MetalSd','Other','Plywood','PreCast','Stone','Stucco','VinylSd','Wd','WdShing']),\n    'MasVnrType':(False,['BrkCmn','BrkFace','CBlock','None','Stone','None']),\n    'ExterQual':(True,['Ex','Gd','TA','Fa','Po']),\n    'ExterCond':(True,['Ex','Gd','TA','Fa','Po']),\n    'Foundation':(False,['BrkTil','CBlock','PConc','Slab','Stone','Wood']),\n    'BsmtQual':(True,['Ex','Gd','TA','Fa','Po','NA','None']),\n    'BsmtCond':(True,['Ex','Gd','TA','Fa','Po','NA','None']),\n    'BsmtExposure':(False,['Gd','Av','Mn','No','NA','None']),\n    'BsmtFinType1':(False,['GLQ','ALQ','BLQ','Rec','LwQ','Unf','NA','None']),\n    'BsmtFinType2':(False,['GLQ','ALQ','BLQ','Rec','LwQ','Unf','NA','None']),\n    'Heating':(False,['Floor','GasA','GasW','Grav','OthW','Wall']),\n    'HeatingQC':(True,['Ex','Gd','TA','Fa','Po']),\n    'CentralAir':(False,['N','Y']),\n    'Electrical':(False,['SBrkr','FuseA','FuseF','FuseP','Mix']),\n    'KitchenQual':(True,['Ex','Gd','TA','Fa','Po']),\n    'Functional':(False,['Typ','Min1','Min2','Mod','Maj1','Maj2','Sev','Sal']),\n    'FireplaceQu':(True,['Ex','Gd','TA','Fa','Po','NA','None']),\n    'GarageType':(False,['2Types','Attchd','Basment','BuiltIn','CarPort','Detchd','NA','None']),\n    'GarageFinish':(False,['Fin','RFn','Unf','NA','None']),\n    'GarageQual':(True,['Ex','Gd','TA','Fa','Po','NA','None']),\n    'GarageCond':(True,['Ex','Gd','TA','Fa','Po','NA','None']),\n    'PavedDrive':(False,['Y','P','N']),\n    'PoolQC':(True,['Ex','Gd','TA','Fa','NA','None']),\n    'Fence':(False,['GdPrv','MnPrv','GdWo','MnWw','NA','None']),\n    'MiscFeature':(False,['Elev','Gar2','Othr','Shed','TenC','NA','None']),\n    'MoSold':(False,['1','2','3','4','5','6','7','8','9','10','11','12']),\n    'YrSold':(False,['2006','2007','2008','2009','2010']),\n    'SaleType':(False,['WD','CWD','VWD','New','COD','Con','ConLw','ConLI','ConLD','Oth']),\n    'SaleCondition':(False,['Normal','Abnorml','AdjLand','Alloca','Family','Partial'])\n}\nlen(LabelData)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:09.012863Z","iopub.execute_input":"2022-07-06T04:34:09.013894Z","iopub.status.idle":"2022-07-06T04:34:09.051249Z","shell.execute_reply.started":"2022-07-06T04:34:09.013851Z","shell.execute_reply":"2022-07-06T04:34:09.050135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testデータだけの欠損がある。 Public/Private 分割方式の場合 Privateのみの欠陥値も想定される\n# この場合、トレーニングデータで予測可能なもっとも無難な値を入れる必要がありそう\n#  testだけにあるラベルデータも同様に予測不可能だけど、これは無いと信じる\nnullentry = pd.concat([train.isnull().sum(),test.isnull().sum(),train.dtypes],axis=1)\n#nullentoryがある場所だけ表示\nnullentry[ (nullentry[0]+nullentry[1]>0) & (nullentry[2] != object)]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:09.110284Z","iopub.execute_input":"2022-07-06T04:34:09.110691Z","iopub.status.idle":"2022-07-06T04:34:09.146380Z","shell.execute_reply.started":"2022-07-06T04:34:09.110659Z","shell.execute_reply":"2022-07-06T04:34:09.145429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nullentry[ (nullentry[0]+nullentry[1]>0) & (nullentry[2] == object)]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:09.193118Z","iopub.execute_input":"2022-07-06T04:34:09.193560Z","iopub.status.idle":"2022-07-06T04:34:09.210425Z","shell.execute_reply.started":"2022-07-06T04:34:09.193516Z","shell.execute_reply":"2022-07-06T04:34:09.209679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#欠損データを埋めておく\nfor c in [\"MasVnrArea\",\"BsmtFinSF1\",\"BsmtFinSF2\",\"BsmtUnfSF\",\"TotalBsmtSF\",\"BsmtFullBath\",\"BsmtHalfBath\",\"GarageYrBlt\",\"GarageCars\",\"GarageArea\"]:\n    all_df[c] = all_df[c].fillna(0)\n\n# 'NA'はすでにあるので、'None'で埋めておく\nfor c in [\"Alley\",\"BsmtQual\",\"BsmtCond\",\"BsmtExposure\",\"BsmtFinType1\",\"BsmtFinType2\",\"FireplaceQu\",\"GarageType\",\"GarageFinish\",\"GarageQual\",\"GarageCond\",\"PoolQC\",\"Fence\",\"MiscFeature\"]:\n    all_df[c] = all_df[c].fillna('None')\n\n# Noneを使っているのでNAで埋めておく\nall_df[\"MasVnrType\"] = all_df[\"MasVnrType\"].fillna('NA')\n    \n# 通りに面した長さ 0はないと思われるので地域の平均を利用\n#\"LotFrontage\"\nall_df[\"LotFrontage\"] = all_df.groupby(\"Neighborhood\")[\"LotFrontage\"].transform(lambda x: x.fillna(x.median()))\n# testだけ存在しないもの : trainの最頻値で埋める\n#\"MSZoning\",\"Utilities\",\"Exterior1st\",\"Exterior2nd\",\"KitchenQual\",\"Functional\",\"SaleType\"\n# 1個だけなのでそれっぽい値でごまかす\n#\"Electrical\"\nfor c in [\"MSZoning\",\"Utilities\",\"Exterior1st\",\"Exterior2nd\",\"KitchenQual\",\"Functional\",\"SaleType\",\"Electrical\"]:\n    all_df[c] = all_df[c].fillna(all_df[c].mode()[0])\n\ntrain = all_df[:ntrain]\ntest = all_df[ntrain:]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:09.270731Z","iopub.execute_input":"2022-07-06T04:34:09.271604Z","iopub.status.idle":"2022-07-06T04:34:09.321482Z","shell.execute_reply.started":"2022-07-06T04:34:09.271548Z","shell.execute_reply":"2022-07-06T04:34:09.320668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#エントリ数の極端に少ないラベル (Seriesの名前がデータの後ろにつくので注意)\n#サンプル数が少ないので変な影響を与えるかもしれない (全エントリの1% = 14.6)\n#Utilities,Condition2,RoofMatl,PoolQC,MiscFeatureはサンプル不足でつかえなさそう  /Heatingは微妙\nfor column in train:\n    if train[column].dtype == object:\n        vc = train[column].value_counts()\n        vcd = vc[vc<5]\n        if(len(vcd)>0):\n            print(vc.head())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:09.372352Z","iopub.execute_input":"2022-07-06T04:34:09.373610Z","iopub.status.idle":"2022-07-06T04:34:09.436767Z","shell.execute_reply.started":"2022-07-06T04:34:09.373550Z","shell.execute_reply":"2022-07-06T04:34:09.435657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#選択項目数がいように小さいものを含むリスト\n#Utilities,Condition2,RoofMatl(微妙),PoolQC,MiscFeature(微妙)\n# ⇒ あまり相関なさそうなのでDrop (項目数に対してPrice幅が大きい)\nfor column in train:\n    if train[column].dtype == object:\n        vc = train[column].value_counts()\n        vcd = vc[vc<5]\n        if(len(vcd)>0):\n            plt.subplots(figsize =(int(max(8,min(20,len(LabelData[column][1])*8/5))), 5))\n            plt.xticks(rotation=30)\n            sns.stripplot(x=column, y=\"SalePrice\", data=train, size = 5, jitter = True, order=LabelData[column][1]);\n            plt.show()  #グラフ表示位置をこのタイミングで確定させたいので呼んでいる\n            \n            print(train[column].value_counts(sort=False))","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:09.469296Z","iopub.execute_input":"2022-07-06T04:34:09.469709Z","iopub.status.idle":"2022-07-06T04:34:15.931632Z","shell.execute_reply.started":"2022-07-06T04:34:09.469678Z","shell.execute_reply":"2022-07-06T04:34:15.930262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#選択項目数が小さいものを含まないものの散布図\nfor column in train:\n    if train[column].dtype == object:\n        vc = train[column].value_counts()\n        vcd = vc[vc<5]\n        if(len(vcd)==0):\n            plt.subplots(figsize =(int(max(8,min(20,len(LabelData[column][1])*8/5))), 5))\n            sns.stripplot(x=column, y=\"SalePrice\", data=train, size = 5, jitter = True, order=LabelData[column][1]);\n            plt.show()  #グラフ表示位置をこのタイミングで確定させたいので呼んでいる\n            print(train[column].value_counts(sort=False).head())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:15.933810Z","iopub.execute_input":"2022-07-06T04:34:15.934294Z","iopub.status.idle":"2022-07-06T04:34:21.024627Z","shell.execute_reply.started":"2022-07-06T04:34:15.934247Z","shell.execute_reply":"2022-07-06T04:34:21.023453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#特徴量のおまとめ変数を作っておく\ntrain[\"TotalGrLivArea\"] = train['GrLivArea'] + train['TotalBsmtSF']\ntrain['TotalBath'] = train['BsmtFullBath'] + train['FullBath'] + (train['BsmtHalfBath']+train['HalfBath'])/2\ntrain['TotalPorch'] = train['WoodDeckSF'] + train['OpenPorchSF'] + train['EnclosedPorch'] + train['3SsnPorch'] + train['ScreenPorch']","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:21.026230Z","iopub.execute_input":"2022-07-06T04:34:21.026592Z","iopub.status.idle":"2022-07-06T04:34:21.037156Z","shell.execute_reply.started":"2022-07-06T04:34:21.026561Z","shell.execute_reply":"2022-07-06T04:34:21.035905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in train:\n    if train[column].dtype != object:\n        #NaNが混っているとグラフ化できないので、Noneに変換している\n        data = pd.concat([train['SalePrice'], train[column].fillna(0)], axis=1)\n        data.plot.scatter(x=column, y='SalePrice', ylim=(0,800000));\n        plt.show()  #グラフ表示位置をこのタイミングで確定させたいので呼んでいる\n        print(train[column].describe())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:21.039619Z","iopub.execute_input":"2022-07-06T04:34:21.039967Z","iopub.status.idle":"2022-07-06T04:34:31.341308Z","shell.execute_reply.started":"2022-07-06T04:34:21.039936Z","shell.execute_reply":"2022-07-06T04:34:31.340068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#値の地域性を表示してみる\nfor column in ['LotFrontage','LotArea','TotalBsmtSF','GrLivArea','TotalGrLivArea','GarageArea','TotalPorch','TotalBath']:\n    plt.subplots(figsize =(20, 5))\n    plt.xticks(rotation=30)\n    sns.stripplot(x=\"Neighborhood\", y=column, data=train, size = 5, jitter = True, order=LabelData[\"Neighborhood\"][1]);\n    plt.show()  #グラフ表示位置をこのタイミングで確定させたいので呼んでいる\n","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:31.342884Z","iopub.execute_input":"2022-07-06T04:34:31.343307Z","iopub.status.idle":"2022-07-06T04:34:35.872227Z","shell.execute_reply.started":"2022-07-06T04:34:31.343273Z","shell.execute_reply":"2022-07-06T04:34:35.871223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"TotalLiveArea\"]=train[\"TotalBsmtSF\"]+train[\"GrLivArea\"]\nabsSortCorr = abs(train.corr()['SalePrice']).sort_values(ascending=False)\nabsSortCorr[1:11]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:35.873761Z","iopub.execute_input":"2022-07-06T04:34:35.874171Z","iopub.status.idle":"2022-07-06T04:34:35.895559Z","shell.execute_reply.started":"2022-07-06T04:34:35.874136Z","shell.execute_reply":"2022-07-06T04:34:35.894666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor column in absSortCorr.index[1:11]:\n    if train[column].dtype != object:\n        #NaNが混っているとグラフ化できないので、Noneに変換している\n        data = pd.concat([train['SalePrice'], train[column].fillna(0)], axis=1)\n        data.plot.scatter(x=column, y='SalePrice', ylim=(0,800000));\n        plt.show()  #グラフ表示位置をこのタイミングで確定させたいので呼んでいる\n        print(train[column].describe())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:35.896574Z","iopub.execute_input":"2022-07-06T04:34:35.897239Z","iopub.status.idle":"2022-07-06T04:34:38.360268Z","shell.execute_reply.started":"2022-07-06T04:34:35.897207Z","shell.execute_reply":"2022-07-06T04:34:38.359221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rlist = ['TotalLiveArea','GrLivArea','GarageCars','GarageArea','TotalBsmtSF','1stFlrSF','TotRmsAbvGrd','MasVnrArea','Fireplaces','BsmtFinSF1']\nprint(absSortCorr[rlist])\nrtrain = train\n#上限カットオフを入れて、外れ値をごまかす\ncolcut = [('TotalLiveArea',5000),('GrLivArea',3500),('GarageCars',3),('GarageArea',950),('TotalBsmtSF',2500),('1stFlrSF',2500),('TotRmsAbvGrd',11),('MasVnrArea',1000),('Fireplaces',2),('BsmtFinSF1',1500)]\nfor (col,cutoff) in colcut:\n    rtrain[col] = rtrain[col].apply(lambda x: cutoff if x>cutoff else x)\n    \nrabsSortCorr = abs(rtrain.corr()['SalePrice']).sort_values(ascending=False)\nprint(rabsSortCorr[rlist])","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:38.361519Z","iopub.execute_input":"2022-07-06T04:34:38.362311Z","iopub.status.idle":"2022-07-06T04:34:38.403025Z","shell.execute_reply.started":"2022-07-06T04:34:38.362263Z","shell.execute_reply":"2022-07-06T04:34:38.402062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor column in absSortCorr.index[1:15]:\n    if rtrain[column].dtype != object:\n        #NaNが混っているとグラフ化できないので、Noneに変換している\n        data = pd.concat([rtrain['SalePrice'], rtrain[column].fillna(0)], axis=1)\n        data.plot.scatter(x=column, y='SalePrice', ylim=(0,800000));\n        plt.show()  #グラフ表示位置をこのタイミングで確定させたいので呼んでいる\n        print(rtrain[column].describe())","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:38.404017Z","iopub.execute_input":"2022-07-06T04:34:38.404373Z","iopub.status.idle":"2022-07-06T04:34:41.222325Z","shell.execute_reply.started":"2022-07-06T04:34:38.404341Z","shell.execute_reply":"2022-07-06T04:34:41.221047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"absCorr = abs(train.corr())\n# 説明変数どうしの相関が強いものは、同じ尺度を2重に判定することになっているかもしれない\nfor c in absCorr:\n    drip = absCorr.loc[(absCorr[c] > 0.8) & (absCorr[c] < 1) , c]\n    if len(drip) > 0 :\n        print(drip)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:41.226212Z","iopub.execute_input":"2022-07-06T04:34:41.226578Z","iopub.status.idle":"2022-07-06T04:34:41.274019Z","shell.execute_reply.started":"2022-07-06T04:34:41.226548Z","shell.execute_reply":"2022-07-06T04:34:41.272902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['SalePrice'], fit=norm);\nplt.figure()\nstats.probplot(train['SalePrice'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:41.275286Z","iopub.execute_input":"2022-07-06T04:34:41.275606Z","iopub.status.idle":"2022-07-06T04:34:41.828200Z","shell.execute_reply.started":"2022-07-06T04:34:41.275575Z","shell.execute_reply":"2022-07-06T04:34:41.827220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#log plot\ntrain['SalePriceLog'] = np.log(train['SalePrice'])\n\nsns.distplot(train['SalePriceLog'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['SalePriceLog'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:41.829848Z","iopub.execute_input":"2022-07-06T04:34:41.830323Z","iopub.status.idle":"2022-07-06T04:34:42.421552Z","shell.execute_reply.started":"2022-07-06T04:34:41.830278Z","shell.execute_reply":"2022-07-06T04:34:42.420287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['SalePriceLog'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:42.422815Z","iopub.execute_input":"2022-07-06T04:34:42.423174Z","iopub.status.idle":"2022-07-06T04:34:42.437394Z","shell.execute_reply.started":"2022-07-06T04:34:42.423144Z","shell.execute_reply":"2022-07-06T04:34:42.436077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var='GrLivArea'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:42.438806Z","iopub.execute_input":"2022-07-06T04:34:42.439248Z","iopub.status.idle":"2022-07-06T04:34:43.040542Z","shell.execute_reply.started":"2022-07-06T04:34:42.439213Z","shell.execute_reply":"2022-07-06T04:34:43.039410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transformed histogram and normal probability plot\nsns.distplot(train['GrLivArea'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['GrLivArea'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:43.042058Z","iopub.execute_input":"2022-07-06T04:34:43.042543Z","iopub.status.idle":"2022-07-06T04:34:43.579302Z","shell.execute_reply.started":"2022-07-06T04:34:43.042494Z","shell.execute_reply":"2022-07-06T04:34:43.578271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transformed histogram and normal probability plot\ntrain['GrLivAreaLog'] = np.log(train['GrLivArea'])\nsns.distplot(train['GrLivAreaLog'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['GrLivAreaLog'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:43.580625Z","iopub.execute_input":"2022-07-06T04:34:43.580996Z","iopub.status.idle":"2022-07-06T04:34:44.164529Z","shell.execute_reply.started":"2022-07-06T04:34:43.580949Z","shell.execute_reply":"2022-07-06T04:34:44.163231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var='GrLivAreaLog'\ndata = pd.concat([train['SalePriceLog'], train[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePriceLog');","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:44.166145Z","iopub.execute_input":"2022-07-06T04:34:44.166649Z","iopub.status.idle":"2022-07-06T04:34:44.429369Z","shell.execute_reply.started":"2022-07-06T04:34:44.166612Z","shell.execute_reply":"2022-07-06T04:34:44.428167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'TotalBsmtSF'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\ndata.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:44.431168Z","iopub.execute_input":"2022-07-06T04:34:44.431579Z","iopub.status.idle":"2022-07-06T04:34:44.631007Z","shell.execute_reply.started":"2022-07-06T04:34:44.431540Z","shell.execute_reply":"2022-07-06T04:34:44.629755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['TotalBsmtSF'], fit=norm);\nfig = plt.figure()\nres = stats.probplot(train['TotalBsmtSF'], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:44.632614Z","iopub.execute_input":"2022-07-06T04:34:44.633133Z","iopub.status.idle":"2022-07-06T04:34:45.104814Z","shell.execute_reply.started":"2022-07-06T04:34:44.633086Z","shell.execute_reply":"2022-07-06T04:34:45.103927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#box plot overallqual/saleprice\nvar = 'OverallQual'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\nf, ax = plt.subplots(figsize=(8, 6))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0);","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:45.105901Z","iopub.execute_input":"2022-07-06T04:34:45.106914Z","iopub.status.idle":"2022-07-06T04:34:45.425215Z","shell.execute_reply.started":"2022-07-06T04:34:45.106865Z","shell.execute_reply":"2022-07-06T04:34:45.424004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"var = 'YearBuilt'\ndata = pd.concat([train['SalePrice'], train[var]], axis=1)\nf, ax = plt.subplots(figsize=(16, 8))\nfig = sns.boxplot(x=var, y=\"SalePrice\", data=data)\nfig.axis(ymin=0, ymax=800000);\nplt.xticks(rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:45.426428Z","iopub.execute_input":"2022-07-06T04:34:45.426909Z","iopub.status.idle":"2022-07-06T04:34:49.439763Z","shell.execute_reply.started":"2022-07-06T04:34:45.426873Z","shell.execute_reply":"2022-07-06T04:34:49.438678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scatterplot\nsns.set()\ncols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']\nsns.pairplot(train[cols], size = 2.5)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:49.441220Z","iopub.execute_input":"2022-07-06T04:34:49.441588Z","iopub.status.idle":"2022-07-06T04:34:58.943177Z","shell.execute_reply.started":"2022-07-06T04:34:49.441556Z","shell.execute_reply":"2022-07-06T04:34:58.942076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SalePriceと相関が強いものの中での相関関係。 単純な相関から2重相関を抽出しているのと同じアプローチ\n#saleprice correlation matrix\nk = 14 #number of variables for heatmap\ncorrmat = train.corr()\ncols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index\ncm = np.corrcoef(train[cols].values.T)\nf, ax = plt.subplots(figsize=(12, 12))\nsns.set(font_scale=1.25)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:34:58.944501Z","iopub.execute_input":"2022-07-06T04:34:58.944841Z","iopub.status.idle":"2022-07-06T04:35:00.093114Z","shell.execute_reply.started":"2022-07-06T04:34:58.944810Z","shell.execute_reply":"2022-07-06T04:35:00.092072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['LotArea']>100000]","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:35:00.094404Z","iopub.execute_input":"2022-07-06T04:35:00.094768Z","iopub.status.idle":"2022-07-06T04:35:00.137216Z","shell.execute_reply.started":"2022-07-06T04:35:00.094737Z","shell.execute_reply":"2022-07-06T04:35:00.136372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby(\"Neighborhood\")[\"SalePrice\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:35:00.138404Z","iopub.execute_input":"2022-07-06T04:35:00.138839Z","iopub.status.idle":"2022-07-06T04:35:00.148316Z","shell.execute_reply.started":"2022-07-06T04:35:00.138808Z","shell.execute_reply":"2022-07-06T04:35:00.147605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['TotalGrLivArea']>6000].sort_values(\"SalePrice\")","metadata":{"execution":{"iopub.status.busy":"2022-07-06T04:41:15.587967Z","iopub.execute_input":"2022-07-06T04:41:15.588421Z","iopub.status.idle":"2022-07-06T04:41:15.667779Z","shell.execute_reply.started":"2022-07-06T04:41:15.588386Z","shell.execute_reply":"2022-07-06T04:41:15.666376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}