{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# 必要なライブラリの読み込み\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\npd.options.display.max_columns = 1000\nimport warnings\nwarnings.filterwarnings(action='ignore',category=DeprecationWarning)\nwarnings.filterwarnings(action='ignore',category=FutureWarning)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T07:29:45.388374Z","iopub.execute_input":"2022-07-15T07:29:45.389124Z","iopub.status.idle":"2022-07-15T07:29:46.489086Z","shell.execute_reply.started":"2022-07-15T07:29:45.389023Z","shell.execute_reply":"2022-07-15T07:29:46.487720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データ読み込み","metadata":{}},{"cell_type":"code","source":"# データセットの各種データを読み込み\nitem_categories = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/item_categories.csv\", sep=\",\")\nitems = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/items.csv\", sep=\",\")\nshops = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/shops.csv\", sep=\",\")\ntest = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/test.csv\", sep=\",\")\nsales_train = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/sales_train.csv\", sep=\",\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:46.491627Z","iopub.execute_input":"2022-07-15T07:29:46.492091Z","iopub.status.idle":"2022-07-15T07:29:49.198353Z","shell.execute_reply.started":"2022-07-15T07:29:46.492044Z","shell.execute_reply":"2022-07-15T07:29:49.197113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 各データセットの中身を確認","metadata":{}},{"cell_type":"code","source":"# View each dataset\ndisplay(item_categories.head(1))\ndisplay(items.head(1))\ndisplay(shops.head(1))\ndisplay(test.head(1))\ndisplay(sales_train.head(1))\n\n# Check for null values in sales_train\nprint(\"-------------------------------------------\")\nprint(\"\\nNull values:\")\ndisplay(sales_train.isnull().sum())\nprint(\"-------------------------------------------\")\n\n# Check columns in sales_train\ndisplay(sales_train.info())\nprint(\"-------------------------------------------\")\nprint(\"Max value in col 'item_cnt_day': \", sales_train['item_cnt_day'].max())\nprint(\"Min value in col 'item_cnt_day': \", sales_train['item_cnt_day'].min())\nprint(\"-------------------------------------------\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:49.199954Z","iopub.execute_input":"2022-07-15T07:29:49.200640Z","iopub.status.idle":"2022-07-15T07:29:49.428718Z","shell.execute_reply.started":"2022-07-15T07:29:49.200595Z","shell.execute_reply":"2022-07-15T07:29:49.427550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 観察結果\nsales_train にnull値は入っていない","metadata":{}},{"cell_type":"code","source":"# item_category_id の比率を表示 \nx=items.groupby(['item_category_id']).count()\nx=x.sort_values(by='item_id',ascending=False)\nx=x.iloc[0:10].reset_index()\nx\n# #plot\nplt.figure(figsize=(8,4))\nax= sns.barplot(x.item_category_id, x.item_id, alpha=0.8)\nplt.title(\"Items per Category\")\nplt.ylabel('# of items', fontsize=12)\nplt.xlabel('Category', fontsize=12)\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:49.430408Z","iopub.execute_input":"2022-07-15T07:29:49.430979Z","iopub.status.idle":"2022-07-15T07:29:49.690164Z","shell.execute_reply.started":"2022-07-15T07:29:49.430946Z","shell.execute_reply":"2022-07-15T07:29:49.688896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データの前処理\nまずは外れ値を見つける","metadata":{}},{"cell_type":"code","source":"# sales_train内部の'item_cnt_day','item_price'の外れ値を探す\ncols = ['item_cnt_day','item_price']\nfig, ax = plt.subplots(ncols = len(cols), figsize = (5 * len(cols),6), sharex = True)\nfig.subplots_adjust(wspace=0.5)\n\nfor i in range(len(cols)):\n  ax[i].boxplot(sales_train[cols[i]])\n  ax[i].set_xlabel(cols[i])\n  ax[i].set_ylabel(\"Count\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:49.691664Z","iopub.execute_input":"2022-07-15T07:29:49.692133Z","iopub.status.idle":"2022-07-15T07:29:50.671010Z","shell.execute_reply.started":"2022-07-15T07:29:49.692083Z","shell.execute_reply":"2022-07-15T07:29:50.669682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## わかったこと\nitem_cnt_day の外れ値は 2000 以上\n\nitem_price の外れ値は 300000 以上","metadata":{}},{"cell_type":"markdown","source":"### 外れ値は取り除く","metadata":{}},{"cell_type":"code","source":"# Remove Outliers\noutlier1 = sales_train[sales_train['item_cnt_day'] > 2000].index[0]\noutlier2 = sales_train[sales_train['item_price'] > 300000].index[0]\nsales_train.drop([outlier1,outlier2], axis = 0, inplace = True)\n\n# インデックスを修正\nsales_train.reset_index(inplace=True,drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:50.672776Z","iopub.execute_input":"2022-07-15T07:29:50.673193Z","iopub.status.idle":"2022-07-15T07:29:50.898419Z","shell.execute_reply.started":"2022-07-15T07:29:50.673153Z","shell.execute_reply":"2022-07-15T07:29:50.897307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:50.899719Z","iopub.execute_input":"2022-07-15T07:29:50.900042Z","iopub.status.idle":"2022-07-15T07:29:50.914537Z","shell.execute_reply.started":"2022-07-15T07:29:50.900014Z","shell.execute_reply":"2022-07-15T07:29:50.913275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 異常な値があるかを調べる","metadata":{}},{"cell_type":"code","source":"#  sales_train内部の'item_cnt_day','item_price'に異常値があるかを調べる\ncols = ['item_cnt_day','item_price']\nfig, ax = plt.subplots(ncols = len(cols), figsize = (5 * len(cols),6), sharex = True)\nfig.subplots_adjust(wspace=0.5)\n\nfor i in range(len(cols)):\n  ax[i].plot(sales_train[cols[i]])\n  ax[i].set_xlabel(cols[i])\n  ax[i].set_ylabel(\"Count\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:50.916086Z","iopub.execute_input":"2022-07-15T07:29:50.916624Z","iopub.status.idle":"2022-07-15T07:29:51.952367Z","shell.execute_reply.started":"2022-07-15T07:29:50.916583Z","shell.execute_reply":"2022-07-15T07:29:51.951236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## わかったこと\nitem_cnt_dayには負の値がいくつかある。おそらく返品処理などが負の値でカウントされていると考えられる。\n\n月別の集計を行うので、負の値はそのままにして、月別に集計したときに販売されたアイテムの数を正しく取得できるようにする。","metadata":{}},{"cell_type":"markdown","source":"## 特徴量分析\n","metadata":{}},{"cell_type":"code","source":"# データ中の最後の月を表す 'date_block_num' カラムの最大数を取得する\nsales_train_max_month = sales_train.date_block_num.max()\n\n# テストデータセットにカラム 'date_block_num' を追加する。値は、sales_train_max_month + 1 で翌月を表すことにする\ntest['date_block_num'] = sales_train_max_month + 1\n\n# 変更した sales_train と test のデータセットを連結した temp テーブルを作成する\nsales_temp = pd.concat([sales_train,test])\n\n# item_cnt_dayカラムで集計し、item_cnt_monthカラムにリネームして月次売上データを作成する\nsales_monthly = sales_temp.groupby(by = ['date_block_num','shop_id','item_id'], as_index=False).agg({'item_cnt_day':'sum'})\nsales_monthly = sales_monthly.rename(columns={'item_cnt_day':'item_cnt_month'})","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:51.953916Z","iopub.execute_input":"2022-07-15T07:29:51.954208Z","iopub.status.idle":"2022-07-15T07:29:53.193883Z","shell.execute_reply.started":"2022-07-15T07:29:51.954182Z","shell.execute_reply":"2022-07-15T07:29:53.192803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# item_cnt_month'の値を1つずつシフトして、新しいラグカラム'lag_item_cnt_month'を追加する。\nsales_monthly['lag_item_cnt_month'] = sales_monthly['item_cnt_month'].shift(1)\n\n# NaNは除去する\nsales_monthly.fillna(0, inplace=True)\nsales_monthly.isna().sum()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:53.198348Z","iopub.execute_input":"2022-07-15T07:29:53.198730Z","iopub.status.idle":"2022-07-15T07:29:53.247810Z","shell.execute_reply.started":"2022-07-15T07:29:53.198701Z","shell.execute_reply":"2022-07-15T07:29:53.246807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データの準備","metadata":{}},{"cell_type":"code","source":"# date_block_num は、学習データと検証データの月単位で分割する必要がある。\nsplit_ratio = 0.8\ntrain_valid_split = np.floor(sales_train_max_month*split_ratio)\ntrain_data = sales_monthly[sales_monthly['date_block_num'] <= train_valid_split]\nvalid_data = sales_monthly[(sales_monthly['date_block_num'] > train_valid_split) & (sales_monthly['date_block_num'] < sales_train_max_month+1)]\n\n# テストデータは、date_block_numがsales_train_max_month+1としている\ntest_data = sales_monthly[sales_monthly['date_block_num'] == sales_train_max_month+1]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:53.249274Z","iopub.execute_input":"2022-07-15T07:29:53.249811Z","iopub.status.idle":"2022-07-15T07:29:53.319726Z","shell.execute_reply.started":"2022-07-15T07:29:53.249771Z","shell.execute_reply":"2022-07-15T07:29:53.318781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練セット、検証セット、テストセットのX、y変数を作成する\nX_train = train_data.drop('item_cnt_month',axis=1)\ny_train = train_data['item_cnt_month']\n\nX_valid = valid_data.drop('item_cnt_month',axis=1)\ny_valid = valid_data['item_cnt_month']\n\nX_test = test_data.drop('item_cnt_month',axis=1)\ny_test = test_data['item_cnt_month']\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:53.321673Z","iopub.execute_input":"2022-07-15T07:29:53.322167Z","iopub.status.idle":"2022-07-15T07:29:53.353938Z","shell.execute_reply.started":"2022-07-15T07:29:53.322113Z","shell.execute_reply":"2022-07-15T07:29:53.352537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルの訓練して予測させる","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\n# トレーニングセットへの線形回帰モデルの適用\nmodel = LinearRegression()\nmodel.fit(X_train,y_train)\n\n# モデルを用いて、検証セットとテストセットのラベルを予測する \ntrain_pred =  model.predict(X_train)\nvalid_pred = model.predict(X_valid)\ntest_pred = model.predict(X_test)\n\n# Error metrics\nprint(f'Root Mean Square Train Data = {np.sqrt(mean_squared_error(y_train,train_pred))}')\nprint(f'Root Mean Square Validation Data = {np.sqrt(mean_squared_error(y_valid,valid_pred))}')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:53.356403Z","iopub.execute_input":"2022-07-15T07:29:53.357361Z","iopub.status.idle":"2022-07-15T07:29:53.810021Z","shell.execute_reply.started":"2022-07-15T07:29:53.357324Z","shell.execute_reply":"2022-07-15T07:29:53.808878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import cross_val_score\n\n# 訓練回帰\nreg1 = GradientBoostingRegressor(random_state=1)\nreg2 = RandomForestRegressor(random_state=1)\nreg3 = LinearRegression()\n\n# K-分割交差検証を用いてモデルの精度を確認\nfor reg, label in zip([reg1, reg2, reg3], ['GB Regression', 'Random Forest', 'Linear Regressor']):\n  scores = cross_val_score(reg, X_valid, y_valid, scoring='neg_mean_squared_error', cv=5)\n  print(\"Accuracy: %0.2f [%s]\" % (-scores.mean(), label))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:29:53.811809Z","iopub.execute_input":"2022-07-15T07:29:53.812506Z","iopub.status.idle":"2022-07-15T07:34:33.952630Z","shell.execute_reply.started":"2022-07-15T07:29:53.812447Z","shell.execute_reply":"2022-07-15T07:34:33.951441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## linear regression による予測結果を出力","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame(test['ID'])\nsubmission['item_cnt_month'] = model.predict(X_test)\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:34:33.954305Z","iopub.execute_input":"2022-07-15T07:34:33.955031Z","iopub.status.idle":"2022-07-15T07:34:34.500839Z","shell.execute_reply.started":"2022-07-15T07:34:33.954989Z","shell.execute_reply":"2022-07-15T07:34:34.499758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 予測結果確認","metadata":{}},{"cell_type":"code","source":"check_temp = pd.read_csv('submission.csv', sep=\",\")\ncheck_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:34:34.502269Z","iopub.execute_input":"2022-07-15T07:34:34.502610Z","iopub.status.idle":"2022-07-15T07:34:34.562406Z","shell.execute_reply.started":"2022-07-15T07:34:34.502579Z","shell.execute_reply":"2022-07-15T07:34:34.561498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_temp.max()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T07:34:34.563737Z","iopub.execute_input":"2022-07-15T07:34:34.564005Z","iopub.status.idle":"2022-07-15T07:34:34.572578Z","shell.execute_reply.started":"2022-07-15T07:34:34.563980Z","shell.execute_reply":"2022-07-15T07:34:34.571583Z"},"trusted":true},"execution_count":null,"outputs":[]}]}