{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 各種ライブラリを読み込む\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 100)\n\nfrom itertools import product\nfrom sklearn.preprocessing import LabelEncoder\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom xgboost import XGBRegressor\nfrom xgboost import plot_importance\n\ndef plot_features(booster, figsize):    \n    fig, ax = plt.subplots(1,1,figsize=figsize)\n    return plot_importance(booster=booster, ax=ax)\n\nfrom math import ceil\nfrom datetime import datetime, date\n\nimport time\nimport sys\nimport gc\nimport pickle\nsys.version_info","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:20.722331Z","iopub.execute_input":"2022-07-27T04:46:20.722811Z","iopub.status.idle":"2022-07-27T04:46:20.741333Z","shell.execute_reply.started":"2022-07-27T04:46:20.722771Z","shell.execute_reply":"2022-07-27T04:46:20.739883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データ読み込み","metadata":{}},{"cell_type":"code","source":"# データセットの各種データを読み込み\nitem_categories = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/item_categories.csv\", sep=\",\")\nitems = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/items.csv\", sep=\",\")\nshops = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/shops.csv\", sep=\",\")\ntest = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/test.csv\", sep=\",\")\nsales_train = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/sales_train.csv\", sep=\",\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:20.953408Z","iopub.execute_input":"2022-07-27T04:46:20.954562Z","iopub.status.idle":"2022-07-27T04:46:22.435626Z","shell.execute_reply.started":"2022-07-27T04:46:20.954512Z","shell.execute_reply":"2022-07-27T04:46:22.434350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 各データセットの中身を確認","metadata":{}},{"cell_type":"code","source":"# 要素名の確認用head(1)\ndisplay(item_categories.head(1))\ndisplay(items.head(1))\ndisplay(shops.head(1))\ndisplay(test.head(1))\ndisplay(sales_train.head(1))\n\n# sales_trainにnullがあるか確認\nprint(\"-------------------------------------------\")\nprint(\"\\nNull values:\")\ndisplay(sales_train.isnull().sum())\nprint(\"-------------------------------------------\")\n\n# sales_trainの要素名確認\ndisplay(sales_train.info())\nprint(\"-------------------------------------------\")\nprint(\"Max value in col 'item_cnt_day': \", sales_train['item_cnt_day'].max())\nprint(\"Min value in col 'item_cnt_day': \", sales_train['item_cnt_day'].min())\nprint(\"-------------------------------------------\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:22.437903Z","iopub.execute_input":"2022-07-27T04:46:22.438286Z","iopub.status.idle":"2022-07-27T04:46:22.660203Z","shell.execute_reply.started":"2022-07-27T04:46:22.438243Z","shell.execute_reply":"2022-07-27T04:46:22.658783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 観察結果\nsales_train にnull値は入っていない","metadata":{}},{"cell_type":"code","source":"# item_category_id の比率を表示 \nx=items.groupby(['item_category_id']).count()\nx=x.sort_values(by='item_id',ascending=False)\nx=x.iloc[0:10].reset_index()\nx\n# #plot\nplt.figure(figsize=(8,4))\nax= sns.barplot(x.item_category_id, x.item_id, alpha=0.8)\nplt.title(\"Items per Category\")\nplt.ylabel('# of items', fontsize=12)\nplt.xlabel('Category', fontsize=12)\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:22.662444Z","iopub.execute_input":"2022-07-27T04:46:22.662873Z","iopub.status.idle":"2022-07-27T04:46:22.920331Z","shell.execute_reply.started":"2022-07-27T04:46:22.662836Z","shell.execute_reply":"2022-07-27T04:46:22.919028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  訓練データのアイテムがテストデータのアイテムと共通しているかを調べる","metadata":{}},{"cell_type":"code","source":"test_shop_plot = test.shop_id.unique()\ntrain_plot = sales_train[sales_train.shop_id.isin(test_shop_plot)]\ntest_item_plot = test.item_id.unique()\ntrain_plot = sales_train[sales_train.item_id.isin(test_item_plot)]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:22.923166Z","iopub.execute_input":"2022-07-27T04:46:22.924104Z","iopub.status.idle":"2022-07-27T04:46:23.230349Z","shell.execute_reply.started":"2022-07-27T04:46:22.924059Z","shell.execute_reply":"2022-07-27T04:46:23.229002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_BLOCK_NUM = train_plot.date_block_num.max()\nMAX_ITEM = len(test_item_plot)\nMAX_CAT = len(item_categories)\nMAX_YEAR = 3\nMAX_MONTH = 4 # 7 8 9 10\nMAX_SHOP = len(test_shop_plot)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:23.231829Z","iopub.execute_input":"2022-07-27T04:46:23.232172Z","iopub.status.idle":"2022-07-27T04:46:23.241245Z","shell.execute_reply.started":"2022-07-27T04:46:23.232142Z","shell.execute_reply":"2022-07-27T04:46:23.240091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ショップとカテゴリの視点から調べる","metadata":{}},{"cell_type":"code","source":"grouped = pd.DataFrame(train_plot.groupby(['shop_id', 'date_block_num'])['item_cnt_day'].sum().reset_index())\nfig, axes = plt.subplots(nrows=5, ncols=2, sharex=True, sharey=True, figsize=(16,20))\nnum_graph = 10\nid_per_graph = ceil(grouped.shop_id.max() / num_graph)\ncount = 0\nfor i in range(5):\n    for j in range(2):\n        sns.pointplot(x='date_block_num', y='item_cnt_day', hue='shop_id', data=grouped[np.logical_and(count*id_per_graph <= grouped['shop_id'], grouped['shop_id'] < (count+1)*id_per_graph)], ax=axes[i][j])\n        count += 1\n# 月別のショップIDごとの売上比較表","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:23.242720Z","iopub.execute_input":"2022-07-27T04:46:23.243093Z","iopub.status.idle":"2022-07-27T04:46:32.501710Z","shell.execute_reply.started":"2022-07-27T04:46:23.243059Z","shell.execute_reply":"2022-07-27T04:46:32.500528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 上の表から分かること\ndate_block_numは(0~33 (mod 12)月)2年10ヶ月のデータがある\n明らかに年末年始にピークが見られるので、このピークのパターンから予測を立てたい。各アイテムの売上を予測するのは点数が膨大であるから難しいと考え、カテゴリに焦点を当てたい","metadata":{}},{"cell_type":"code","source":"# カテゴリ追加\ntrain_plot = train_plot.set_index('item_id').join(items.set_index('item_id')).drop('item_name', axis=1).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:32.503077Z","iopub.execute_input":"2022-07-27T04:46:32.503424Z","iopub.status.idle":"2022-07-27T04:46:33.055906Z","shell.execute_reply.started":"2022-07-27T04:46:32.503392Z","shell.execute_reply":"2022-07-27T04:46:33.054825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練データに年と月を追加\ntrain_plot['month'] = train_plot.date.apply(lambda x: datetime.strptime(x, '%d.%m.%Y').strftime('%m'))\ntrain_plot['year'] = train_plot.date.apply(lambda x: datetime.strptime(x, '%d.%m.%Y').strftime('%Y'))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:46:33.058740Z","iopub.execute_input":"2022-07-27T04:46:33.059100Z","iopub.status.idle":"2022-07-27T04:47:11.147067Z","shell.execute_reply.started":"2022-07-27T04:46:33.059069Z","shell.execute_reply":"2022-07-27T04:47:11.146037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 月別のitem_cnt_day売上1\n# 前月と比較した売上比に応じた縦線を追加\nfig, axes = plt.subplots(nrows=5, ncols=2, sharex=True, sharey=True, figsize=(16,20))\nnum_graph = 10\nid_per_graph = ceil(train_plot.item_category_id.max() / num_graph)\ncount = 0\nfor i in range(5):\n    for j in range(2):\n        sns.pointplot(x='month', y='item_cnt_day', hue='item_category_id', \n                      data=train_plot[np.logical_and(count*id_per_graph <= train_plot['item_category_id'], train_plot['item_category_id'] < (count+1)*id_per_graph)], \n                      ax=axes[i][j])\n        count += 1","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:47:11.148641Z","iopub.execute_input":"2022-07-27T04:47:11.149821Z","iopub.status.idle":"2022-07-27T04:47:48.495190Z","shell.execute_reply.started":"2022-07-27T04:47:11.149767Z","shell.execute_reply":"2022-07-27T04:47:48.493879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 月別のitem_cnt_day売上2\nfig, axes = plt.subplots(nrows=5, ncols=2, sharex=True, sharey=True, figsize=(16,20))\nnum_graph = 10\nid_per_graph = ceil(train_plot.item_category_id.max() / num_graph)\ncount = 0\nfor i in range(5):\n    for j in range(2):\n        sns.pointplot(x='date_block_num', y='item_cnt_day', hue='item_category_id', \n                      data=train_plot[np.logical_and(count*id_per_graph <= train_plot['item_category_id'], train_plot['item_category_id'] < (count+1)*id_per_graph)], \n                      ax=axes[i][j])\n        count += 1\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:47:48.496664Z","iopub.execute_input":"2022-07-27T04:47:48.497014Z","iopub.status.idle":"2022-07-27T04:48:52.128307Z","shell.execute_reply.started":"2022-07-27T04:47:48.496983Z","shell.execute_reply":"2022-07-27T04:48:52.127522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sales_train = sales_train.drop('date', axis=1)\n#sales_train = sales_train.drop('item_category_id', axis=1)\n#sales_train = sales_train.groupby(['shop_id', 'item_id', 'date_block_num', 'month', 'year']).sum()\n#sales_train = sales_train.sort_index()\n#sales_train","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:52.129603Z","iopub.execute_input":"2022-07-27T04:48:52.130541Z","iopub.status.idle":"2022-07-27T04:48:52.135665Z","shell.execute_reply.started":"2022-07-27T04:48:52.130502Z","shell.execute_reply":"2022-07-27T04:48:52.134534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データの前処理\nまずは外れ値を見つける","metadata":{}},{"cell_type":"code","source":"# sales_train内部の'item_cnt_day','item_price'の外れ値を探す\ncols = ['item_cnt_day','item_price']\nfig, ax = plt.subplots(ncols = len(cols), figsize = (5 * len(cols),6), sharex = True)\nfig.subplots_adjust(wspace=0.5)\n\nfor i in range(len(cols)):\n  ax[i].boxplot(sales_train[cols[i]])\n  ax[i].set_xlabel(cols[i])\n  ax[i].set_ylabel(\"Count\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:52.137834Z","iopub.execute_input":"2022-07-27T04:48:52.138304Z","iopub.status.idle":"2022-07-27T04:48:53.081978Z","shell.execute_reply.started":"2022-07-27T04:48:52.138261Z","shell.execute_reply":"2022-07-27T04:48:53.081121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## わかったこと\nitem_cnt_day の外れ値は 2000 以上\n\nitem_price の外れ値は 300000 以上","metadata":{}},{"cell_type":"markdown","source":"### 外れ値は取り除く","metadata":{}},{"cell_type":"code","source":"# 外れ値除去\noutlier1 = sales_train[sales_train['item_cnt_day'] > 2000].index[0]\noutlier2 = sales_train[sales_train['item_price'] > 300000].index[0]\nsales_train.drop([outlier1,outlier2], axis = 0, inplace = True)\n\n# インデックスを修正\nsales_train.reset_index(inplace=True,drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:53.083250Z","iopub.execute_input":"2022-07-27T04:48:53.083789Z","iopub.status.idle":"2022-07-27T04:48:53.306587Z","shell.execute_reply.started":"2022-07-27T04:48:53.083756Z","shell.execute_reply":"2022-07-27T04:48:53.305312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 中身確認\nsales_train[200000:200050]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:53.307984Z","iopub.execute_input":"2022-07-27T04:48:53.308417Z","iopub.status.idle":"2022-07-27T04:48:53.333062Z","shell.execute_reply.started":"2022-07-27T04:48:53.308383Z","shell.execute_reply":"2022-07-27T04:48:53.331795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 異常な値があるかを調べる\n異常な値とは、他とは明らかに特徴が異なるもの(値が負)など","metadata":{}},{"cell_type":"code","source":"#  sales_train内部の'item_cnt_day','item_price'に異常値があるかを調べる\ncols = ['item_cnt_day','item_price']\nfig, ax = plt.subplots(ncols = len(cols), figsize = (5 * len(cols),6), sharex = True)\nfig.subplots_adjust(wspace=0.5)\n\nfor i in range(len(cols)):\n  ax[i].plot(sales_train[cols[i]])\n  ax[i].set_xlabel(cols[i])\n  ax[i].set_ylabel(\"Count\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:53.334705Z","iopub.execute_input":"2022-07-27T04:48:53.335075Z","iopub.status.idle":"2022-07-27T04:48:54.368766Z","shell.execute_reply.started":"2022-07-27T04:48:53.335042Z","shell.execute_reply":"2022-07-27T04:48:54.367729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## わかったこと\nitem_cnt_dayには負の値がいくつかある。おそらく返品処理などが負の値でカウントされていると考えられる。\n\n月別の集計を行うので、負の値はそのままにして、月別に集計したときに販売されたアイテムの数を正しく取得できるようにする。","metadata":{}},{"cell_type":"markdown","source":"## 特徴量分析\n","metadata":{}},{"cell_type":"code","source":"display(sales_train.head(1))\ndisplay(test.head(1))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:54.370026Z","iopub.execute_input":"2022-07-27T04:48:54.370386Z","iopub.status.idle":"2022-07-27T04:48:54.390617Z","shell.execute_reply.started":"2022-07-27T04:48:54.370355Z","shell.execute_reply":"2022-07-27T04:48:54.389157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# データ中の最後の月を表す 'date_block_num' カラムの最大数を取得する\nsales_train_max_month = sales_train.date_block_num.max()\n\n# テストデータセットにカラム 'date_block_num' を追加する。値は、sales_train_max_month + 1 で翌月を表すことにする\ntest['date_block_num'] = sales_train_max_month + 1\ntest['item_price'] = sales_train['item_price']\n\n# 変更した sales_train と test のデータセットを連結した temp テーブルを作成する\nsales_temp = pd.concat([sales_train,test])\n\n# item_cnt_dayカラムで集計し、item_cnt_monthカラムにリネームして月次売上データを作成する\nsales_monthly = sales_temp.groupby(by = ['date_block_num','shop_id','item_id', 'item_price'], as_index=False).agg({'item_cnt_day':'sum'})\n#sales_monthly = sales_temp.groupby(by = ['date_block_num','item_id', 'item_price'], as_index=False).agg({'item_cnt_day':'sum'})\n\n#sales_monthly = sales_temp.groupby(by = ['date_block_num','shop_id','item_id'], as_index=False).agg({'item_cnt_day':'sum'})\nsales_monthly = sales_monthly.rename(columns={'item_cnt_day':'item_cnt_month'})","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:54.395790Z","iopub.execute_input":"2022-07-27T04:48:54.396525Z","iopub.status.idle":"2022-07-27T04:48:55.902399Z","shell.execute_reply.started":"2022-07-27T04:48:54.396465Z","shell.execute_reply":"2022-07-27T04:48:55.901224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales_monthly.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:55.903947Z","iopub.execute_input":"2022-07-27T04:48:55.904404Z","iopub.status.idle":"2022-07-27T04:48:55.917913Z","shell.execute_reply.started":"2022-07-27T04:48:55.904359Z","shell.execute_reply":"2022-07-27T04:48:55.916574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# item_cnt_month'の値を1つずつシフトして、新しいラグカラム'lag_item_cnt_month'を追加する。\nsales_monthly['lag_item_cnt_month'] = sales_monthly['item_cnt_month'].shift(1)\n\n# NaNは除去する\nsales_monthly.fillna(0, inplace=True)\nsales_monthly.isna().sum()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:55.919747Z","iopub.execute_input":"2022-07-27T04:48:55.920522Z","iopub.status.idle":"2022-07-27T04:48:56.000839Z","shell.execute_reply.started":"2022-07-27T04:48:55.920455Z","shell.execute_reply":"2022-07-27T04:48:55.999940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales_monthly['item_cnt_month'].max()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.002044Z","iopub.execute_input":"2022-07-27T04:48:56.003095Z","iopub.status.idle":"2022-07-27T04:48:56.017222Z","shell.execute_reply.started":"2022-07-27T04:48:56.003058Z","shell.execute_reply":"2022-07-27T04:48:56.015931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## データの準備","metadata":{}},{"cell_type":"code","source":"# date_block_num は、学習データと検証データの月単位で分割する必要がある。\nsplit_ratio = 0.8\ntrain_valid_split = np.floor(sales_train_max_month*split_ratio)\ntrain_data = sales_monthly[sales_monthly['date_block_num'] <= train_valid_split]\nvalid_data = sales_monthly[(sales_monthly['date_block_num'] > train_valid_split) & (sales_monthly['date_block_num'] < sales_train_max_month+1)]\n\n# テストデータは、date_block_numがsales_train_max_month+1としている\ntest_data = sales_monthly[sales_monthly['date_block_num'] == sales_train_max_month+1]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.019168Z","iopub.execute_input":"2022-07-27T04:48:56.019947Z","iopub.status.idle":"2022-07-27T04:48:56.106763Z","shell.execute_reply.started":"2022-07-27T04:48:56.019899Z","shell.execute_reply":"2022-07-27T04:48:56.105572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.108100Z","iopub.execute_input":"2022-07-27T04:48:56.108461Z","iopub.status.idle":"2022-07-27T04:48:56.127354Z","shell.execute_reply.started":"2022-07-27T04:48:56.108430Z","shell.execute_reply":"2022-07-27T04:48:56.126420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.128314Z","iopub.execute_input":"2022-07-27T04:48:56.128662Z","iopub.status.idle":"2022-07-27T04:48:56.148084Z","shell.execute_reply.started":"2022-07-27T04:48:56.128630Z","shell.execute_reply":"2022-07-27T04:48:56.146644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 訓練セット、検証セット、テストセットのX、y変数を作成する\nX_train = train_data.drop('item_cnt_month',axis=1)\ny_train = train_data['item_cnt_month']\n\nX_valid = valid_data.drop('item_cnt_month',axis=1)\ny_valid = valid_data['item_cnt_month']\n\nX_test = test_data.drop('item_cnt_month',axis=1)\ny_test = test_data['item_cnt_month']\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.149934Z","iopub.execute_input":"2022-07-27T04:48:56.150526Z","iopub.status.idle":"2022-07-27T04:48:56.186775Z","shell.execute_reply.started":"2022-07-27T04:48:56.150405Z","shell.execute_reply":"2022-07-27T04:48:56.185801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df[df['age'] < 25]\nX_train[X_train['item_price']<0]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.188011Z","iopub.execute_input":"2022-07-27T04:48:56.188862Z","iopub.status.idle":"2022-07-27T04:48:56.204678Z","shell.execute_reply.started":"2022-07-27T04:48:56.188825Z","shell.execute_reply":"2022-07-27T04:48:56.203417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.206023Z","iopub.execute_input":"2022-07-27T04:48:56.206372Z","iopub.status.idle":"2022-07-27T04:48:56.214548Z","shell.execute_reply.started":"2022-07-27T04:48:56.206340Z","shell.execute_reply":"2022-07-27T04:48:56.213300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.215798Z","iopub.execute_input":"2022-07-27T04:48:56.216822Z","iopub.status.idle":"2022-07-27T04:48:56.237493Z","shell.execute_reply.started":"2022-07-27T04:48:56.216782Z","shell.execute_reply":"2022-07-27T04:48:56.236368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデルの訓練して予測させる","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\n# トレーニングセットへの線形回帰モデルの適用\nmodel = LinearRegression()\nmodel.fit(X_train,y_train)\n\n# モデルを用いて、検証セットとテストセットのラベルを予測する \ntrain_pred =  model.predict(X_train)\nvalid_pred = model.predict(X_valid)\ntest_pred = model.predict(X_test)\n\n# Error metrics\n# 平均二乗誤差\nprint(f'Root Mean Square Train Data = {np.sqrt(mean_squared_error(y_train,train_pred))}')\nprint(f'Root Mean Square Validation Data = {np.sqrt(mean_squared_error(y_valid,valid_pred))}')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.238742Z","iopub.execute_input":"2022-07-27T04:48:56.239506Z","iopub.status.idle":"2022-07-27T04:48:56.621266Z","shell.execute_reply.started":"2022-07-27T04:48:56.239449Z","shell.execute_reply":"2022-07-27T04:48:56.619920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import cross_val_score\n\n# 回帰の訓練\nreg1 = GradientBoostingRegressor(random_state=1)\nreg2 = RandomForestRegressor(random_state=1)\nreg3 = LinearRegression()\n\n# K-分割交差検証を用いてモデルの精度を確認\nfor reg, label in zip([reg1, reg2, reg3], ['GB Regression', 'Random Forest', 'Linear Regressor']):\n  scores = cross_val_score(reg, X_valid, y_valid, scoring='neg_mean_squared_error', cv=5)\n  print(\"Accuracy: %0.2f [%s]\" % (-scores.mean(), label))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:48:56.623483Z","iopub.execute_input":"2022-07-27T04:48:56.624390Z","iopub.status.idle":"2022-07-27T04:55:44.400547Z","shell.execute_reply.started":"2022-07-27T04:48:56.624327Z","shell.execute_reply":"2022-07-27T04:55:44.396688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## linear regression による予測結果を出力","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame(test['ID'])\nsubmission['item_cnt_month'] = model.predict(X_test)\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:44.402813Z","iopub.execute_input":"2022-07-27T04:55:44.404260Z","iopub.status.idle":"2022-07-27T04:55:44.982602Z","shell.execute_reply.started":"2022-07-27T04:55:44.404197Z","shell.execute_reply":"2022-07-27T04:55:44.981464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 予測結果確認","metadata":{}},{"cell_type":"code","source":"check_temp = pd.read_csv('submission.csv', sep=\",\")\ncheck_temp","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:44.984193Z","iopub.execute_input":"2022-07-27T04:55:44.984562Z","iopub.status.idle":"2022-07-27T04:55:45.051391Z","shell.execute_reply.started":"2022-07-27T04:55:44.984525Z","shell.execute_reply":"2022-07-27T04:55:45.050618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_temp[check_temp['item_cnt_month']>4]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:45.052658Z","iopub.execute_input":"2022-07-27T04:55:45.053648Z","iopub.status.idle":"2022-07-27T04:55:45.066289Z","shell.execute_reply.started":"2022-07-27T04:55:45.053605Z","shell.execute_reply":"2022-07-27T04:55:45.065456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,8))\nplt.title('predict display')\nplt.xlabel('ID')\nplt.ylabel('pred_item_cnt_month')\nplt.plot(submission['item_cnt_month']);\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:45.070127Z","iopub.execute_input":"2022-07-27T04:55:45.070719Z","iopub.status.idle":"2022-07-27T04:55:45.404381Z","shell.execute_reply.started":"2022-07-27T04:55:45.070685Z","shell.execute_reply":"2022-07-27T04:55:45.403415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_x = model.predict(X_test)\npred_x","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:45.405818Z","iopub.execute_input":"2022-07-27T04:55:45.406337Z","iopub.status.idle":"2022-07-27T04:55:45.420636Z","shell.execute_reply.started":"2022-07-27T04:55:45.406302Z","shell.execute_reply":"2022-07-27T04:55:45.419411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data['item_cnt_month'])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:45.422908Z","iopub.execute_input":"2022-07-27T04:55:45.423727Z","iopub.status.idle":"2022-07-27T04:55:45.432075Z","shell.execute_reply.started":"2022-07-27T04:55:45.423683Z","shell.execute_reply":"2022-07-27T04:55:45.430918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 別の分析方法","metadata":{}},{"cell_type":"markdown","source":"## XGBoostによる予測","metadata":{}},{"cell_type":"code","source":"# データセットの各種データを読み込み\ncats = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/item_categories.csv\", sep=\",\")\nitems = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/items.csv\", sep=\",\")\nshops = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/shops.csv\", sep=\",\")\ntest = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/test.csv\", sep=\",\").set_index('ID')\ntrain = pd.read_csv(\"../input/competitive-data-science-predict-future-sales/sales_train.csv\", sep=\",\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:45.433783Z","iopub.execute_input":"2022-07-27T04:55:45.434484Z","iopub.status.idle":"2022-07-27T04:55:46.892484Z","shell.execute_reply.started":"2022-07-27T04:55:45.434423Z","shell.execute_reply":"2022-07-27T04:55:46.891455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 外れ値の確認","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nplt.xlim(-100, 3000)\nsns.boxplot(x=train.item_cnt_day)\n\nplt.figure(figsize=(10,4))\nplt.xlim(train.item_price.min(), train.item_price.max()*1.1)\nsns.boxplot(x=train.item_price)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:46.894021Z","iopub.execute_input":"2022-07-27T04:55:46.894348Z","iopub.status.idle":"2022-07-27T04:55:48.492579Z","shell.execute_reply.started":"2022-07-27T04:55:46.894319Z","shell.execute_reply":"2022-07-27T04:55:48.491329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 確認結果\nsales_trainの\n\nitem_cnt_day：2000を超えてる要素がある\n\nitem_price：300,000を超えている要素がある\n","metadata":{}},{"cell_type":"markdown","source":"## 外れ値は除外する\n価格と売上がおかしい商品があると考え、価格が100000以上、売上が1001以上のアイテムを削除する。","metadata":{}},{"cell_type":"code","source":"train = train[train.item_price<100000]\ntrain = train[train.item_cnt_day<1001]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:48.494258Z","iopub.execute_input":"2022-07-27T04:55:48.494623Z","iopub.status.idle":"2022-07-27T04:55:48.823123Z","shell.execute_reply.started":"2022-07-27T04:55:48.494592Z","shell.execute_reply":"2022-07-27T04:55:48.821877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"価格が0以下の商品が1つあるので中央値で埋める","metadata":{}},{"cell_type":"code","source":"median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()\ntrain.loc[train.item_price<0, 'item_price'] = median","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:48.824752Z","iopub.execute_input":"2022-07-27T04:55:48.825096Z","iopub.status.idle":"2022-07-27T04:55:48.868845Z","shell.execute_reply.started":"2022-07-27T04:55:48.825064Z","shell.execute_reply":"2022-07-27T04:55:48.867775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"いくつかのショップが重複しているので修正","metadata":{}},{"cell_type":"code","source":"# Якутск Орджоникидзе, 56\ntrain.loc[train.shop_id == 0, 'shop_id'] = 57\ntest.loc[test.shop_id == 0, 'shop_id'] = 57\n# Якутск ТЦ \"Центральный\"\ntrain.loc[train.shop_id == 1, 'shop_id'] = 58\ntest.loc[test.shop_id == 1, 'shop_id'] = 58\n# Жуковский ул. Чкалова 39м²\ntrain.loc[train.shop_id == 10, 'shop_id'] = 11\ntest.loc[test.shop_id == 10, 'shop_id'] = 11\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:48.870254Z","iopub.execute_input":"2022-07-27T04:55:48.870751Z","iopub.status.idle":"2022-07-27T04:55:48.940321Z","shell.execute_reply.started":"2022-07-27T04:55:48.870716Z","shell.execute_reply":"2022-07-27T04:55:48.938915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## shop・category・itemの前処理\n* 各shop_nameは都市名で始まっている。\n* 各カテゴリはその名前にtypeとsubtypeを含む。\n","metadata":{}},{"cell_type":"code","source":"shops","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:48.941943Z","iopub.execute_input":"2022-07-27T04:55:48.942735Z","iopub.status.idle":"2022-07-27T04:55:48.958456Z","shell.execute_reply.started":"2022-07-27T04:55:48.942646Z","shell.execute_reply":"2022-07-27T04:55:48.957503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cats","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:48.960011Z","iopub.execute_input":"2022-07-27T04:55:48.960754Z","iopub.status.idle":"2022-07-27T04:55:48.980070Z","shell.execute_reply.started":"2022-07-27T04:55:48.960715Z","shell.execute_reply":"2022-07-27T04:55:48.978510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:48.981762Z","iopub.execute_input":"2022-07-27T04:55:48.982600Z","iopub.status.idle":"2022-07-27T04:55:48.998907Z","shell.execute_reply.started":"2022-07-27T04:55:48.982552Z","shell.execute_reply":"2022-07-27T04:55:48.997536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shopの都市名をcityで区切る\nshops.loc[shops.shop_name == 'Сергиев Посад ТЦ \"7Я\"', 'shop_name'] = 'СергиевПосад ТЦ \"7Я\"'\nshops['city'] = shops['shop_name'].str.split(' ').map(lambda x: x[0])\nshops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'\nshops['city_code'] = LabelEncoder().fit_transform(shops['city'])\nshops = shops[['shop_id','city_code']]\n\n# categoryのカテゴリ名と商品名を区切る\ncats['split'] = cats['item_category_name'].str.split('-')\ncats['type'] = cats['split'].map(lambda x: x[0].strip())\ncats['type_code'] = LabelEncoder().fit_transform(cats['type'])\n# もしsubtypeがnanならtypeをいれる\ncats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x) > 1 else x[0].strip())\ncats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])\ncats = cats[['item_category_id','type_code', 'subtype_code']]\n\n# アイテム名は取り除く\nitems.drop(['item_name'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:49.009424Z","iopub.execute_input":"2022-07-27T04:55:49.009859Z","iopub.status.idle":"2022-07-27T04:55:49.035858Z","shell.execute_reply.started":"2022-07-27T04:55:49.009825Z","shell.execute_reply":"2022-07-27T04:55:49.034621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## テストセットについて\n34ヶ月間の42店舗と5100品目の214200組がテストセット含まれている\n\n363個のアイテムはテストセットのみに含まれている＝新しい\n\n","metadata":{}},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:49.037456Z","iopub.execute_input":"2022-07-27T04:55:49.038162Z","iopub.status.idle":"2022-07-27T04:55:49.050701Z","shell.execute_reply.started":"2022-07-27T04:55:49.038121Z","shell.execute_reply":"2022-07-27T04:55:49.049670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(set(test.item_id) - set(test.item_id).intersection(set(train.item_id)))), len(list(set(test.item_id))), len(test)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:49.052204Z","iopub.execute_input":"2022-07-27T04:55:49.052584Z","iopub.status.idle":"2022-07-27T04:55:49.498324Z","shell.execute_reply.started":"2022-07-27T04:55:49.052551Z","shell.execute_reply":"2022-07-27T04:55:49.497193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'date_block_num','shop_id','item_id'を抽出\nts = time.time()\nmatrix = []\ncols = ['date_block_num','shop_id','item_id']\nfor i in range(34):\n    sales = train[train.date_block_num==i]\n    matrix.append(np.array(list(product([i], sales.shop_id.unique(), sales.item_id.unique())), dtype='int16'))\n    \nmatrix = pd.DataFrame(np.vstack(matrix), columns=cols)\nmatrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)\nmatrix['shop_id'] = matrix['shop_id'].astype(np.int8)\nmatrix['item_id'] = matrix['item_id'].astype(np.int16)\nmatrix.sort_values(cols,inplace=True)\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:55:49.500176Z","iopub.execute_input":"2022-07-27T04:55:49.500841Z","iopub.status.idle":"2022-07-27T04:56:02.690190Z","shell.execute_reply.started":"2022-07-27T04:55:49.500805Z","shell.execute_reply":"2022-07-27T04:56:02.688847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"収益=商品価格＊日別売上\n\nで定める","metadata":{}},{"cell_type":"code","source":"train['revenue'] = train['item_price'] *  train['item_cnt_day']\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:02.691554Z","iopub.execute_input":"2022-07-27T04:56:02.691904Z","iopub.status.idle":"2022-07-27T04:56:02.712889Z","shell.execute_reply.started":"2022-07-27T04:56:02.691870Z","shell.execute_reply":"2022-07-27T04:56:02.711542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"item_cnt_dayは月毎に集計にするようにしてitem_cnt_monthを追加","metadata":{}},{"cell_type":"code","source":"ts = time.time()\ngroup = train.groupby(['date_block_num','shop_id','item_id']).agg({'item_cnt_day': ['sum']})\ngroup.columns = ['item_cnt_month']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=cols, how='left')\nmatrix['item_cnt_month'] = (matrix['item_cnt_month']\n                                .fillna(0)\n                                .clip(0,20) # NB clip target here\n                                .astype(np.float16))\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:02.714529Z","iopub.execute_input":"2022-07-27T04:56:02.715496Z","iopub.status.idle":"2022-07-27T04:56:08.655096Z","shell.execute_reply.started":"2022-07-27T04:56:02.715422Z","shell.execute_reply":"2022-07-27T04:56:08.654131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"時系列処理のためにテストセットの要素を追加","metadata":{}},{"cell_type":"code","source":"test['date_block_num'] = 34 #34ヶ月間\ntest['date_block_num'] = test['date_block_num'].astype(np.int8)\ntest['shop_id'] = test['shop_id'].astype(np.int8)\ntest['item_id'] = test['item_id'].astype(np.int16)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:08.656457Z","iopub.execute_input":"2022-07-27T04:56:08.657400Z","iopub.status.idle":"2022-07-27T04:56:08.666788Z","shell.execute_reply.started":"2022-07-27T04:56:08.657327Z","shell.execute_reply":"2022-07-27T04:56:08.665750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"testと抽出したmatrixを連結","metadata":{}},{"cell_type":"code","source":"ts = time.time()\nmatrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)\nmatrix.fillna(0, inplace=True) # 34 month\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:08.668171Z","iopub.execute_input":"2022-07-27T04:56:08.668578Z","iopub.status.idle":"2022-07-27T04:56:08.764677Z","shell.execute_reply.started":"2022-07-27T04:56:08.668542Z","shell.execute_reply":"2022-07-27T04:56:08.763106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"各ラベルをキャスト","metadata":{}},{"cell_type":"code","source":"ts = time.time()\nmatrix = pd.merge(matrix, shops, on=['shop_id'], how='left')\nmatrix = pd.merge(matrix, items, on=['item_id'], how='left')\nmatrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')\nmatrix['city_code'] = matrix['city_code'].astype(np.int8)\nmatrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)\nmatrix['type_code'] = matrix['type_code'].astype(np.int8)\nmatrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:08.766119Z","iopub.execute_input":"2022-07-27T04:56:08.766511Z","iopub.status.idle":"2022-07-27T04:56:13.561860Z","shell.execute_reply.started":"2022-07-27T04:56:08.766452Z","shell.execute_reply":"2022-07-27T04:56:13.560261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 時系列をずらす関数定義\ndef lag_feature(df, lags, col):\n    tmp = df[['date_block_num','shop_id','item_id',col]]\n    for i in lags:\n        shifted = tmp.copy()\n        shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]\n        shifted['date_block_num'] += i\n        df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:13.563426Z","iopub.execute_input":"2022-07-27T04:56:13.563953Z","iopub.status.idle":"2022-07-27T04:56:13.572018Z","shell.execute_reply.started":"2022-07-27T04:56:13.563902Z","shell.execute_reply":"2022-07-27T04:56:13.570480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1, 2, 3, 6, 12ヶ月ずつずらす=lag","metadata":{}},{"cell_type":"markdown","source":"### lagを追加していく","metadata":{}},{"cell_type":"code","source":"ts = time.time()\nmatrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month')\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:13.573381Z","iopub.execute_input":"2022-07-27T04:56:13.573746Z","iopub.status.idle":"2022-07-27T04:56:56.524234Z","shell.execute_reply.started":"2022-07-27T04:56:13.573717Z","shell.execute_reply":"2022-07-27T04:56:56.522951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num'], how='left')\nmatrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_avg_item_cnt')\nmatrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:56:56.525935Z","iopub.execute_input":"2022-07-27T04:56:56.526626Z","iopub.status.idle":"2022-07-27T04:57:09.108420Z","shell.execute_reply.started":"2022-07-27T04:56:56.526578Z","shell.execute_reply":"2022-07-27T04:57:09.107036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'item_id']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_item_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')\nmatrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')\nmatrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:57:09.110046Z","iopub.execute_input":"2022-07-27T04:57:09.110433Z","iopub.status.idle":"2022-07-27T04:58:01.338016Z","shell.execute_reply.started":"2022-07-27T04:57:09.110401Z","shell.execute_reply":"2022-07-27T04:58:01.336811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'shop_id']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_shop_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')\nmatrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')\nmatrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:58:01.340095Z","iopub.execute_input":"2022-07-27T04:58:01.341008Z","iopub.status.idle":"2022-07-27T04:58:57.570767Z","shell.execute_reply.started":"2022-07-27T04:58:01.340960Z","shell.execute_reply":"2022-07-27T04:58:57.569463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'item_category_id']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_cat_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')\nmatrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')\nmatrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:58:57.572223Z","iopub.execute_input":"2022-07-27T04:58:57.572629Z","iopub.status.idle":"2022-07-27T04:59:18.005101Z","shell.execute_reply.started":"2022-07-27T04:58:57.572592Z","shell.execute_reply":"2022-07-27T04:59:18.003777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'shop_id', 'type_code']).agg({'item_cnt_month': ['mean']})\ngroup.columns = ['date_shop_type_avg_item_cnt']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')\nmatrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')\nmatrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:59:18.006672Z","iopub.execute_input":"2022-07-27T04:59:18.007056Z","iopub.status.idle":"2022-07-27T04:59:39.615686Z","shell.execute_reply.started":"2022-07-27T04:59:18.007022Z","shell.execute_reply":"2022-07-27T04:59:39.614821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code']).agg({'item_cnt_month': ['mean']})\ngroup.columns = ['date_shop_subtype_avg_item_cnt']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')\nmatrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')\nmatrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T04:59:39.616826Z","iopub.execute_input":"2022-07-27T04:59:39.617458Z","iopub.status.idle":"2022-07-27T05:00:01.874229Z","shell.execute_reply.started":"2022-07-27T04:59:39.617405Z","shell.execute_reply":"2022-07-27T05:00:01.873011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'city_code']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_city_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')\nmatrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')\nmatrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:00:01.875832Z","iopub.execute_input":"2022-07-27T05:00:01.876183Z","iopub.status.idle":"2022-07-27T05:00:24.098513Z","shell.execute_reply.started":"2022-07-27T05:00:01.876151Z","shell.execute_reply":"2022-07-27T05:00:24.097226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'item_id', 'city_code']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_item_city_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')\nmatrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')\nmatrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:00:24.099832Z","iopub.execute_input":"2022-07-27T05:00:24.100834Z","iopub.status.idle":"2022-07-27T05:00:55.161515Z","shell.execute_reply.started":"2022-07-27T05:00:24.100798Z","shell.execute_reply":"2022-07-27T05:00:55.160437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'type_code']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_type_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')\nmatrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')\nmatrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:00:55.163201Z","iopub.execute_input":"2022-07-27T05:00:55.163712Z","iopub.status.idle":"2022-07-27T05:01:18.699691Z","shell.execute_reply.started":"2022-07-27T05:00:55.163669Z","shell.execute_reply":"2022-07-27T05:01:18.698539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ngroup = matrix.groupby(['date_block_num', 'subtype_code']).agg({'item_cnt_month': ['mean']})\ngroup.columns = [ 'date_subtype_avg_item_cnt' ]\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')\nmatrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)\nmatrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')\nmatrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:01:18.701131Z","iopub.execute_input":"2022-07-27T05:01:18.701534Z","iopub.status.idle":"2022-07-27T05:01:42.754718Z","shell.execute_reply.started":"2022-07-27T05:01:18.701495Z","shell.execute_reply":"2022-07-27T05:01:42.753361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"過去6ヶ月間の価格推移を見ていく","metadata":{}},{"cell_type":"code","source":"ts = time.time()\ngroup = train.groupby(['item_id']).agg({'item_price': ['mean']})\ngroup.columns = ['item_avg_item_price']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['item_id'], how='left')\nmatrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)\n\ngroup = train.groupby(['date_block_num','item_id']).agg({'item_price': ['mean']})\ngroup.columns = ['date_item_avg_item_price']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')\nmatrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)\n\nlags = [1,2,3,4,5,6]\nmatrix = lag_feature(matrix, lags, 'date_item_avg_item_price')\n\nfor i in lags:\n    matrix['delta_price_lag_'+str(i)] = \\\n        (matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price']) / matrix['item_avg_item_price']\n\ndef select_trend(row):\n    for i in lags:\n        if row['delta_price_lag_'+str(i)]:\n            return row['delta_price_lag_'+str(i)]\n    return 0\n    \nmatrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)\nmatrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)\nmatrix['delta_price_lag'].fillna(0, inplace=True)\n\n# https://stackoverflow.com/questions/31828240/first-non-null-value-per-row-from-a-list-of-pandas-columns/31828559\n# matrix['price_trend'] = matrix[['delta_price_lag_1','delta_price_lag_2','delta_price_lag_3']].bfill(axis=1).iloc[:, 0]\n# Invalid dtype for backfill_2d [float16]\n\nfetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']\nfor i in lags:\n    fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]\n    fetures_to_drop += ['delta_price_lag_'+str(i)]\n\nmatrix.drop(fetures_to_drop, axis=1, inplace=True)\n\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:01:42.756363Z","iopub.execute_input":"2022-07-27T05:01:42.756738Z","iopub.status.idle":"2022-07-27T05:06:22.119124Z","shell.execute_reply.started":"2022-07-27T05:01:42.756703Z","shell.execute_reply":"2022-07-27T05:06:22.117752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"直近1ヶ月のショップ売上高の推移","metadata":{}},{"cell_type":"code","source":"ts = time.time()\ngroup = train.groupby(['date_block_num','shop_id']).agg({'revenue': ['sum']})\ngroup.columns = ['date_shop_revenue']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')\nmatrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)\n\ngroup = group.groupby(['shop_id']).agg({'date_shop_revenue': ['mean']})\ngroup.columns = ['shop_avg_revenue']\ngroup.reset_index(inplace=True)\n\nmatrix = pd.merge(matrix, group, on=['shop_id'], how='left')\nmatrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)\n\nmatrix['delta_revenue'] = (matrix['date_shop_revenue'] - matrix['shop_avg_revenue']) / matrix['shop_avg_revenue']\nmatrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)\n\nmatrix = lag_feature(matrix, [1], 'delta_revenue')\n\nmatrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:06:22.120712Z","iopub.execute_input":"2022-07-27T05:06:22.121725Z","iopub.status.idle":"2022-07-27T05:06:54.985689Z","shell.execute_reply.started":"2022-07-27T05:06:22.121684Z","shell.execute_reply":"2022-07-27T05:06:54.984538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1~12月の日数を追加。うるう年は考慮しない","metadata":{}},{"cell_type":"code","source":"matrix['month'] = matrix['date_block_num'] % 12\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:06:54.987540Z","iopub.execute_input":"2022-07-27T05:06:54.987912Z","iopub.status.idle":"2022-07-27T05:06:55.052614Z","shell.execute_reply.started":"2022-07-27T05:06:54.987880Z","shell.execute_reply":"2022-07-27T05:06:55.051319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])\nmatrix['days'] = matrix['month'].map(days).astype(np.int8)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:06:55.054696Z","iopub.execute_input":"2022-07-27T05:06:55.055170Z","iopub.status.idle":"2022-07-27T05:06:55.486264Z","shell.execute_reply.started":"2022-07-27T05:06:55.055123Z","shell.execute_reply":"2022-07-27T05:06:55.484977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ショップとアイテムのペア、およびアイテムのみの最終販売からの月数を計算","metadata":{}},{"cell_type":"code","source":"ts = time.time()\ncache = {}\nmatrix['item_shop_last_sale'] = -1\nmatrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)\nfor idx, row in matrix.iterrows():    \n    key = str(row.item_id)+' '+str(row.shop_id)\n    if key not in cache:\n        if row.item_cnt_month!=0:\n            cache[key] = row.date_block_num\n    else:\n        last_date_block_num = cache[key]\n        matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num\n        cache[key] = row.date_block_num         \ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:06:55.487956Z","iopub.execute_input":"2022-07-27T05:06:55.488311Z","iopub.status.idle":"2022-07-27T05:25:23.408586Z","shell.execute_reply.started":"2022-07-27T05:06:55.488281Z","shell.execute_reply":"2022-07-27T05:25:23.407553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\ncache = {}\nmatrix['item_last_sale'] = -1\nmatrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)\nfor idx, row in matrix.iterrows():    \n    key = row.item_id\n    if key not in cache:\n        if row.item_cnt_month!=0:\n            cache[key] = row.date_block_num\n    else:\n        last_date_block_num = cache[key]\n        if row.date_block_num>last_date_block_num:\n            matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num\n            cache[key] = row.date_block_num         \ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:25:23.409945Z","iopub.execute_input":"2022-07-27T05:25:23.410864Z","iopub.status.idle":"2022-07-27T05:37:37.543782Z","shell.execute_reply.started":"2022-07-27T05:25:23.410827Z","shell.execute_reply":"2022-07-27T05:37:37.542321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"各ショップ/アイテムのペア、アイテムのみの初回販売からの月数を計算","metadata":{}},{"cell_type":"code","source":"ts = time.time()\nmatrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')\nmatrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:37.545529Z","iopub.execute_input":"2022-07-27T05:37:37.546066Z","iopub.status.idle":"2022-07-27T05:37:39.331496Z","shell.execute_reply.started":"2022-07-27T05:37:37.546029Z","shell.execute_reply":"2022-07-27T05:37:39.330382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 最終準備\n\nラグ値として12を使用しているため、最初の12ヶ月を削除する。また、今月の計算値（テストセットで計算できない値）を持つ列もすべて削除する","metadata":{}},{"cell_type":"code","source":"ts = time.time()\nmatrix = matrix[matrix.date_block_num > 11]\ntime.time() - ts\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:39.332734Z","iopub.execute_input":"2022-07-27T05:37:39.333058Z","iopub.status.idle":"2022-07-27T05:37:41.363145Z","shell.execute_reply.started":"2022-07-27T05:37:39.333028Z","shell.execute_reply":"2022-07-27T05:37:41.362156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ラグを利用することでヌケが大量に出る","metadata":{}},{"cell_type":"code","source":"ts = time.time()\ndef fill_na(df):\n    for col in df.columns:\n        if ('_lag_' in col) & (df[col].isnull().any()):\n            if ('item_cnt' in col):\n                df[col].fillna(0, inplace=True)         \n    return df\n\nmatrix = fill_na(matrix)\ntime.time() - ts\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:41.364455Z","iopub.execute_input":"2022-07-27T05:37:41.364807Z","iopub.status.idle":"2022-07-27T05:37:43.855043Z","shell.execute_reply.started":"2022-07-27T05:37:41.364777Z","shell.execute_reply":"2022-07-27T05:37:43.854139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix.columns\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:43.856456Z","iopub.execute_input":"2022-07-27T05:37:43.857032Z","iopub.status.idle":"2022-07-27T05:37:43.864002Z","shell.execute_reply.started":"2022-07-27T05:37:43.856995Z","shell.execute_reply":"2022-07-27T05:37:43.862927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix.info()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:43.865497Z","iopub.execute_input":"2022-07-27T05:37:43.866491Z","iopub.status.idle":"2022-07-27T05:37:43.885461Z","shell.execute_reply.started":"2022-07-27T05:37:43.866439Z","shell.execute_reply":"2022-07-27T05:37:43.884554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matrix.to_pickle('data.pkl')\ndel matrix\ndel cache\ndel group\ndel items\ndel shops\ndel cats\ndel train\n# leave test for submission\ngc.collect();\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:43.886747Z","iopub.execute_input":"2022-07-27T05:37:43.887285Z","iopub.status.idle":"2022-07-27T05:37:45.781480Z","shell.execute_reply.started":"2022-07-27T05:37:43.887250Z","shell.execute_reply":"2022-07-27T05:37:45.780350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_pickle('data.pkl')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:45.783053Z","iopub.execute_input":"2022-07-27T05:37:45.783407Z","iopub.status.idle":"2022-07-27T05:37:46.508383Z","shell.execute_reply.started":"2022-07-27T05:37:45.783375Z","shell.execute_reply":"2022-07-27T05:37:46.507048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data[[\n    'date_block_num',\n    'shop_id',\n    'item_id',\n    'item_cnt_month',\n    'city_code',\n    'item_category_id',\n    'type_code',\n    'subtype_code',\n    'item_cnt_month_lag_1',\n    'item_cnt_month_lag_2',\n    'item_cnt_month_lag_3',\n    'item_cnt_month_lag_6',\n    'item_cnt_month_lag_12',\n    'date_avg_item_cnt_lag_1',\n    'date_item_avg_item_cnt_lag_1',\n    'date_item_avg_item_cnt_lag_2',\n    'date_item_avg_item_cnt_lag_3',\n    'date_item_avg_item_cnt_lag_6',\n    'date_item_avg_item_cnt_lag_12',\n    'date_shop_avg_item_cnt_lag_1',\n    'date_shop_avg_item_cnt_lag_2',\n    'date_shop_avg_item_cnt_lag_3',\n    'date_shop_avg_item_cnt_lag_6',\n    'date_shop_avg_item_cnt_lag_12',\n    'date_cat_avg_item_cnt_lag_1',\n    #'date_shop_cat_avg_item_cnt_lag_1',\n    #'date_shop_type_avg_item_cnt_lag_1',\n    #'date_shop_subtype_avg_item_cnt_lag_1',\n    'date_city_avg_item_cnt_lag_1',\n    'date_item_city_avg_item_cnt_lag_1',\n    #'date_type_avg_item_cnt_lag_1',\n    #'date_subtype_avg_item_cnt_lag_1',\n    'delta_price_lag',\n    'month',\n    'days',\n    'item_shop_last_sale',\n    'item_last_sale',\n    'item_shop_first_sale',\n    'item_first_sale',\n]]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:46.510146Z","iopub.execute_input":"2022-07-27T05:37:46.510536Z","iopub.status.idle":"2022-07-27T05:37:47.143453Z","shell.execute_reply.started":"2022-07-27T05:37:46.510465Z","shell.execute_reply":"2022-07-27T05:37:47.142446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)\nY_train = data[data.date_block_num < 33]['item_cnt_month']\nX_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)\nY_valid = data[data.date_block_num == 33]['item_cnt_month']\nX_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:47.144761Z","iopub.execute_input":"2022-07-27T05:37:47.145101Z","iopub.status.idle":"2022-07-27T05:37:50.877561Z","shell.execute_reply.started":"2022-07-27T05:37:47.145068Z","shell.execute_reply":"2022-07-27T05:37:50.876041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data\ngc.collect();\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:50.879033Z","iopub.execute_input":"2022-07-27T05:37:50.879400Z","iopub.status.idle":"2022-07-27T05:37:51.204089Z","shell.execute_reply.started":"2022-07-27T05:37:50.879365Z","shell.execute_reply":"2022-07-27T05:37:51.202663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts = time.time()\n\nmodel = XGBRegressor(\n    max_depth=8,\n    n_estimators=1000,\n    min_child_weight=300, \n    colsample_bytree=0.8, \n    subsample=0.8, \n    eta=0.3,    \n    seed=42)\n\nmodel.fit(\n    X_train, \n    Y_train, \n    eval_metric=\"rmse\", \n    eval_set=[(X_train, Y_train), (X_valid, Y_valid)], \n    verbose=True, \n    early_stopping_rounds = 10)\n\ntime.time() - ts","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:37:51.205731Z","iopub.execute_input":"2022-07-27T05:37:51.206495Z","iopub.status.idle":"2022-07-27T05:38:34.249245Z","shell.execute_reply.started":"2022-07-27T05:37:51.206429Z","shell.execute_reply":"2022-07-27T05:38:34.248378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = model.predict(X_valid).clip(0, 20)\nY_test = model.predict(X_test).clip(0, 20)\n\nsubmission = pd.DataFrame({\n    \"ID\": test.index, \n    \"item_cnt_month\": Y_test\n})\nsubmission.to_csv('xgb_submission.csv', index=False)\n\n# save predictions for an ensemble\npickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))\npickle.dump(Y_test, open('xgb_test.pickle', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:38:34.250350Z","iopub.execute_input":"2022-07-27T05:38:34.250960Z","iopub.status.idle":"2022-07-27T05:38:34.856338Z","shell.execute_reply.started":"2022-07-27T05:38:34.250910Z","shell.execute_reply":"2022-07-27T05:38:34.855096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:38:34.857986Z","iopub.execute_input":"2022-07-27T05:38:34.858842Z","iopub.status.idle":"2022-07-27T05:38:34.867390Z","shell.execute_reply.started":"2022-07-27T05:38:34.858798Z","shell.execute_reply":"2022-07-27T05:38:34.866265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:38:34.868743Z","iopub.execute_input":"2022-07-27T05:38:34.869481Z","iopub.status.idle":"2022-07-27T05:38:34.912910Z","shell.execute_reply.started":"2022-07-27T05:38:34.869434Z","shell.execute_reply":"2022-07-27T05:38:34.912020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred.min()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:38:34.914125Z","iopub.execute_input":"2022-07-27T05:38:34.914654Z","iopub.status.idle":"2022-07-27T05:38:34.929043Z","shell.execute_reply.started":"2022-07-27T05:38:34.914621Z","shell.execute_reply":"2022-07-27T05:38:34.927907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16,8))\nplt.title('predict display')\nplt.xlabel('X_train')\nplt.ylabel('y_train')\nplt.plot(Y_pred);\nplt.show","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:38:34.930309Z","iopub.execute_input":"2022-07-27T05:38:34.930667Z","iopub.status.idle":"2022-07-27T05:38:35.310839Z","shell.execute_reply.started":"2022-07-27T05:38:34.930635Z","shell.execute_reply":"2022-07-27T05:38:35.309777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 影響が強い要素\nplot_features(model, (10,14))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:38:35.312124Z","iopub.execute_input":"2022-07-27T05:38:35.313020Z","iopub.status.idle":"2022-07-27T05:38:35.877608Z","shell.execute_reply.started":"2022-07-27T05:38:35.312981Z","shell.execute_reply":"2022-07-27T05:38:35.876527Z"},"trusted":true},"execution_count":null,"outputs":[]}]}