{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.colors\n#import plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold \nfrom sklearn.metrics import roc_auc_score, roc_curve, auc\nimport catboost\nfrom catboost import Pool\nfrom catboost import CatBoostRegressor\nfrom xgboost import XGBRegressor\nfrom xgboost import plot_importance\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\n\nimport warnings, gc","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:14:39.882483Z","iopub.execute_input":"2022-07-27T09:14:39.882921Z","iopub.status.idle":"2022-07-27T09:14:39.898288Z","shell.execute_reply.started":"2022-07-27T09:14:39.882888Z","shell.execute_reply":"2022-07-27T09:14:39.897247Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Preprocessing**","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('../input/competitive-data-science-predict-future-sales/test.csv', dtype={'ID': 'int32', 'shop_id': 'int32', \n                                                  'item_id': 'int32'})\nitem_categories = pd.read_csv('../input/competitive-data-science-predict-future-sales/item_categories.csv', \n                              dtype={'item_category_name': 'str', 'item_category_id': 'int32'})\nitems = pd.read_csv('../input/competitive-data-science-predict-future-sales/items.csv', dtype={'item_name': 'str', 'item_id': 'int32', \n                                                 'item_category_id': 'int32'})\nshops = pd.read_csv('../input/competitive-data-science-predict-future-sales/shops.csv', dtype={'shop_name': 'str', 'shop_id': 'int32'})\nsales = pd.read_csv('../input/competitive-data-science-predict-future-sales/sales_train.csv', parse_dates=['date'], \n                    dtype={'date': 'str', 'date_block_num': 'int32', 'shop_id': 'int32', \n                          'item_id': 'int32', 'item_price': 'float32', 'item_cnt_day': 'int32'})\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:14:39.911260Z","iopub.execute_input":"2022-07-27T09:14:39.911822Z","iopub.status.idle":"2022-07-27T09:14:42.072192Z","shell.execute_reply.started":"2022-07-27T09:14:39.911788Z","shell.execute_reply":"2022-07-27T09:14:42.070532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = sales.join(items, on='item_id', rsuffix='_').join(shops, on='shop_id', rsuffix='_').join(item_categories, on='item_category_id', rsuffix='_').drop(['item_id_', 'shop_id_', 'item_category_id_'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:14:42.073928Z","iopub.execute_input":"2022-07-27T09:14:42.074327Z","iopub.status.idle":"2022-07-27T09:14:43.653577Z","shell.execute_reply.started":"2022-07-27T09:14:42.074295Z","shell.execute_reply":"2022-07-27T09:14:43.652226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.date     = train.date.apply(pd.to_datetime)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:14:43.655402Z","iopub.execute_input":"2022-07-27T09:14:43.655727Z","iopub.status.idle":"2022-07-27T09:15:08.364491Z","shell.execute_reply.started":"2022-07-27T09:14:43.655695Z","shell.execute_reply":"2022-07-27T09:15:08.363309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['day']   = train.date.apply(lambda x: x.day)\ntrain['year'] =train.date.apply(lambda x: x.year)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:15:08.365835Z","iopub.execute_input":"2022-07-27T09:15:08.366182Z","iopub.status.idle":"2022-07-27T09:15:45.394660Z","shell.execute_reply.started":"2022-07-27T09:15:08.366135Z","shell.execute_reply":"2022-07-27T09:15:45.393393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:15:45.396508Z","iopub.execute_input":"2022-07-27T09:15:45.396817Z","iopub.status.idle":"2022-07-27T09:15:45.403444Z","shell.execute_reply.started":"2022-07-27T09:15:45.396787Z","shell.execute_reply":"2022-07-27T09:15:45.402565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order=['day','year','date_block_num', 'shop_id',  'item_category_id','item_id', 'item_price',\n       'item_cnt_day']\ntrain=train[order]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:15:45.404651Z","iopub.execute_input":"2022-07-27T09:15:45.404997Z","iopub.status.idle":"2022-07-27T09:15:45.498002Z","shell.execute_reply.started":"2022-07-27T09:15:45.404967Z","shell.execute_reply":"2022-07-27T09:15:45.496666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['item_pricmean_inshop']=train.groupby(['item_id','shop_id'])['item_price'].transform('mean')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:15:45.500074Z","iopub.execute_input":"2022-07-27T09:15:45.500798Z","iopub.status.idle":"2022-07-27T09:15:45.890894Z","shell.execute_reply.started":"2022-07-27T09:15:45.500763Z","shell.execute_reply":"2022-07-27T09:15:45.889621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Two of the columns have negative data: \nitem_price: only one row, it's probably a wrong data, replace it with mean value of that product in the same shop.\nitem_cnt_day: there are 7000+ '-1', probably means returned products.","metadata":{}},{"cell_type":"code","source":"#deal with those negative data\nfor i in range(train.shape[0]):\n    if train.iloc[i,5]<0:\n        train.iloc[i,5]=train.iloc[i,8]\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:15:45.892585Z","iopub.execute_input":"2022-07-27T09:15:45.892961Z","iopub.status.idle":"2022-07-27T09:18:23.965142Z","shell.execute_reply.started":"2022-07-27T09:15:45.892930Z","shell.execute_reply":"2022-07-27T09:18:23.963933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we only need to predict the total sales in the next month, we change the dataframe from daily record into monthly record. In this way, we can also reduce the dimensions.","metadata":{}},{"cell_type":"code","source":"trainM=train.groupby(['date_block_num', 'shop_id', 'item_category_id','item_id','item_price'],as_index=False)\ntrainM=trainM.agg({'item_price':['sum', 'mean'], 'item_cnt_day':['sum', 'mean','count']})\ntrainM.columns = ['date_block_num', 'shop_id', 'item_category_id', 'item_id', 'item_price', 'mean_item_price','item_cnt', 'mean_item_cnt', 'transactions']","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:18:23.966471Z","iopub.execute_input":"2022-07-27T09:18:23.966799Z","iopub.status.idle":"2022-07-27T09:18:26.142240Z","shell.execute_reply.started":"2022-07-27T09:18:23.966768Z","shell.execute_reply":"2022-07-27T09:18:26.141146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainM.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:18:26.143736Z","iopub.execute_input":"2022-07-27T09:18:26.144236Z","iopub.status.idle":"2022-07-27T09:18:26.179439Z","shell.execute_reply.started":"2022-07-27T09:18:26.144185Z","shell.execute_reply":"2022-07-27T09:18:26.178368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is no any missing value in the dataframe. However, we still need to fill in some data in order to see the complete pattern of each shop and item.（fill in empty data will influence the mean value, so we calculate those mean value before the next step）","metadata":{}},{"cell_type":"code","source":"shop_ids = trainM['shop_id'].unique()\nitem_ids = trainM['item_id'].unique()\nempty_df = []\nfor i in range(34):\n    for shop in shop_ids:\n        for itemc,item in zip(items['item_category_id'],items['item_id']):\n            empty_df.append([i, shop,itemc,item])\n    \n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:18:26.181082Z","iopub.execute_input":"2022-07-27T09:18:26.181405Z","iopub.status.idle":"2022-07-27T09:19:27.938645Z","shell.execute_reply.started":"2022-07-27T09:18:26.181374Z","shell.execute_reply":"2022-07-27T09:19:27.937541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_df = pd.DataFrame(empty_df, columns=['date_block_num','shop_id','item_category_id','item_id'])\ntrainM = pd.merge(empty_df, trainM, on=['date_block_num','shop_id','item_category_id','item_id'], how='left')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:19:27.940323Z","iopub.execute_input":"2022-07-27T09:19:27.940647Z","iopub.status.idle":"2022-07-27T09:21:18.959075Z","shell.execute_reply.started":"2022-07-27T09:19:27.940616Z","shell.execute_reply":"2022-07-27T09:21:18.957571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test set\ntest['date_block_num']=34\nitems=items.drop('item_name',axis=1)\ntest=pd.merge(test,items,on=['item_id'])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:21:18.960745Z","iopub.execute_input":"2022-07-27T09:21:18.961073Z","iopub.status.idle":"2022-07-27T09:21:19.000848Z","shell.execute_reply.started":"2022-07-27T09:21:18.961042Z","shell.execute_reply":"2022-07-27T09:21:18.999754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trainM['cnt_lag1_inshop']=trainM.groupby(['shop_id','item_id'])['item_cnt'].shift(1)\n#trainM['cnt_lag2_inshop']=trainM.groupby(['shop_id','item_id'])['item_cnt'].shift(2)\ntrainM['cnt_lag1']=trainM.groupby('item_id')['item_cnt'].shift(1)\ntrainM['cnt_lag2']=trainM.groupby('item_id')['item_cnt'].shift(2)\ntrainM['cnt_lag12']=trainM.groupby('item_id')['item_cnt'].shift(12)\n\n#trainM['cnt_cum_inshop']=trainM.groupby(['shop_id','item_id'])['item_cnt'].cumsum()\n#trainM['cnt_cum_lag1_inshop']=trainM.groupby(['shop_id','item_id'])['cnt_cum_inshop'].shift(1)\n#trainM['cnt_cum_lag2_inshop']=trainM.groupby(['shop_id','item_id'])['cnt_cum_inshop'].shift(2)\n#trainM=trainM.drop(['cnt_cum_inshop'],axis=1)\n\ntrainM['cnt_cum']=trainM.groupby('item_id')['item_cnt'].cumsum()\ntrainM['cnt_cum_lag1']=trainM.groupby(['item_id'])['cnt_cum'].shift(1)\ntrainM=trainM.drop(['cnt_cum'],axis=1)\n\ntrainM['item_mean_allshop']=trainM[trainM['item_cnt']!=0].groupby(['date_block_num','item_id'])['mean_item_price'].transform('mean')\ntrainM['item_mean_diff']=trainM['mean_item_price']-trainM['item_mean_allshop']\n\ntrainM['item_cnt_mean_3y']=trainM.groupby('item_id')['item_cnt'].transform('mean')\ntrainM['item_pri_mean_3y']=trainM[trainM['item_cnt']!=0].groupby(['item_id'])['mean_item_price'].transform('mean')\n\ntrainM.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:21:19.002263Z","iopub.execute_input":"2022-07-27T09:21:19.002614Z","iopub.status.idle":"2022-07-27T09:22:20.268840Z","shell.execute_reply.started":"2022-07-27T09:21:19.002575Z","shell.execute_reply":"2022-07-27T09:22:20.267451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA**","metadata":{}},{"cell_type":"markdown","source":"Our final target is to predict the sale of particular item in particular shop in the next month, so EDA will focus more on the sales trend and pattern of individual shops and products.","metadata":{}},{"cell_type":"code","source":"shop_itemkind=train.groupby(['shop_id','item_id'],as_index=False)['item_price'].agg('mean')\nshop_itemkind=pd.DataFrame(shop_itemkind)\nshop_itemkind=shop_itemkind.groupby(['shop_id'],as_index=False)['item_id'].agg('count')\nshop_itemkind=pd.DataFrame(shop_itemkind)\n\nshop_cnt_sum=trainM.groupby(['shop_id'],as_index=False)['item_cnt'].agg('sum')\nshop_cnt_sum=pd.DataFrame(shop_cnt_sum)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:20.270483Z","iopub.execute_input":"2022-07-27T09:22:20.271266Z","iopub.status.idle":"2022-07-27T09:22:21.669700Z","shell.execute_reply.started":"2022-07-27T09:22:20.271217Z","shell.execute_reply":"2022-07-27T09:22:21.668660Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('ggplot')\nf, (ax1, ax2) = plt.subplots(2, 1,figsize=(60, 40),dpi=200)\nax1.bar(x = range(shop_itemkind.shape[0]),  \n        height =shop_itemkind.item_id,  \n        tick_label = shop_itemkind.shop_id,  \n        color = 'blueviolet',\n        width=0.8\n        )\nax1.set_ylabel('kinds of product')\nax1.set_xlabel('shop_id')\nax1.tick_params(labelsize=23)\nax1.set_title('The number of items each shop sells',fontsize=35)\n\n\nax2.bar(x = range(shop_cnt_sum.shape[0]),  \n        height =shop_cnt_sum.item_cnt,  \n        tick_label = shop_cnt_sum.shop_id,  \n        color = 'hotpink',\n        width=0.8\n        )\nax2.set_ylabel('total item_cnt')\nax2.set_xlabel('shop_id')\nax2.tick_params(labelsize=23)\nax2.set_title('Total sales of each shop',fontsize=35)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:21.671099Z","iopub.execute_input":"2022-07-27T09:22:21.671939Z","iopub.status.idle":"2022-07-27T09:22:26.730078Z","shell.execute_reply.started":"2022-07-27T09:22:21.671905Z","shell.execute_reply":"2022-07-27T09:22:26.728845Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It shows that shop31 and shop25 are very popular (the total number of sold items is very high) and the kinds of items they sell are the most various among all the shop. We can also see that the trends of the 2 barchart above are very similar, which imply that the more kinds of item a shop sell, the more popular the shop will be.","metadata":{}},{"cell_type":"markdown","source":"Now, we want to know whether the category of an item is closely relative to its price or sales.","metadata":{}},{"cell_type":"code","source":"item_cum=train.groupby(['item_id'],as_index=False)['item_cnt_day'].agg('sum')\nitem_cum=pd.DataFrame(item_cum)\n\nitemc_cum=train.groupby(['item_category_id'],as_index=False)['item_cnt_day'].agg('sum')\nitemc_cum=pd.DataFrame(itemc_cum)\n\nitemc_mean=train.groupby(['item_category_id'],as_index=False)['item_price'].agg('mean')\nitemc_mean=pd.DataFrame(itemc_mean)\n\nitemc_kind=items.groupby('item_category_id')['item_id'].agg('count')\nitemc_kind=pd.DataFrame(itemc_kind)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:26.731758Z","iopub.execute_input":"2022-07-27T09:22:26.732878Z","iopub.status.idle":"2022-07-27T09:22:26.966883Z","shell.execute_reply.started":"2022-07-27T09:22:26.732835Z","shell.execute_reply":"2022-07-27T09:22:26.965883Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('ggplot')\n\nf, (ax1, ax2,ax3) = plt.subplots(3, 1,figsize=(80, 60),dpi=200)\nax1.bar(x = range(itemc_cum.shape[0]),  \n        height =itemc_cum.item_cnt_day,  \n        tick_label = itemc_cum.item_category_id,  \n        color = 'seagreen',\n        width=0.8\n        )\nax1.set_ylabel('total cnt')\nax1.set_xlabel('item_category_id')\nax1.tick_params(labelsize=23)\nax1.set_title('Total sale of each item category',fontsize=30)\n\nax2.bar(x = range(itemc_mean.shape[0]),  \n        height =itemc_mean.item_price,  \n        tick_label = itemc_mean.item_category_id,  \n        color = 'skyblue',\n        width=0.8\n        )\nax2.set_ylabel('mean price')\nax2.set_xlabel('item_category_id')\nax2.tick_params(labelsize=23)\nax2.set_title('Mean price of each item category',fontsize=30)\n\nax3.bar(x = range(itemc_kind.shape[0]),  \n        height =itemc_kind.item_id,  \n        tick_label = itemc_cum.item_category_id,  \n        color = 'firebrick',\n        width=0.8\n        )\nax3.set_ylabel('item')\nax3.set_xlabel('item_category_id')\nax3.tick_params(labelsize=23)\nax3.set_title('The number of items in each item category',fontsize=30)\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:26.968362Z","iopub.execute_input":"2022-07-27T09:22:26.969495Z","iopub.status.idle":"2022-07-27T09:22:36.754961Z","shell.execute_reply.started":"2022-07-27T09:22:26.969456Z","shell.execute_reply":"2022-07-27T09:22:36.753360Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemc_kind.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:36.756414Z","iopub.execute_input":"2022-07-27T09:22:36.756743Z","iopub.status.idle":"2022-07-27T09:22:36.773990Z","shell.execute_reply.started":"2022-07-27T09:22:36.756712Z","shell.execute_reply":"2022-07-27T09:22:36.772693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"category40 is the most popular item_category(sell over 600000 in the given period,rank No.1), and its mean price is not so expensive as well as includes over 5000 different items.","metadata":{}},{"cell_type":"code","source":"itemc_eachmean=train.groupby(['item_category_id','item_id'],as_index=False)['item_price'].agg('mean')\nitemc_eachmean=pd.DataFrame(itemc_eachmean)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:36.776238Z","iopub.execute_input":"2022-07-27T09:22:36.776709Z","iopub.status.idle":"2022-07-27T09:22:37.022361Z","shell.execute_reply.started":"2022-07-27T09:22:36.776643Z","shell.execute_reply":"2022-07-27T09:22:37.021228Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, axes = plt.subplots(14, 6, figsize=(200, 250))\nrow=0\ncol=[0,1,2,3,4,5]*14\nfor i in range(0,84):\n    if (i!=0)&(i%6==0):\n        row+=1\n    df=itemc_eachmean[itemc_eachmean['item_category_id']==i]\n    xx=sns.barplot(x=\"item_id\", y=\"item_price\", data=df, ax=axes[row,col[i]])\n    xx.set_title(i,fontsize=60)\n    xx.tick_params(labelsize=40)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:22:37.023912Z","iopub.execute_input":"2022-07-27T09:22:37.024241Z","iopub.status.idle":"2022-07-27T09:28:10.814341Z","shell.execute_reply.started":"2022-07-27T09:22:37.024210Z","shell.execute_reply":"2022-07-27T09:28:10.812890Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemc_eachcum=train.groupby(['item_category_id','item_id'],as_index=False)['item_cnt_day'].agg('sum')\nitemc_eachcum=pd.DataFrame(itemc_eachcum)\n\nf, axes = plt.subplots(14, 6, figsize=(100, 250))\nrow=0\ncol=[0,1,2,3,4,5]*14\nfor i in range(0,84):\n    if (i!=0)&(i%6==0):\n        row+=1\n    df=itemc_eachcum[itemc_eachcum['item_category_id']==i]\n    xx=sns.barplot(x=\"item_id\", y=\"item_cnt_day\", data=df, ax=axes[row,col[i]])\n    xx.set_title(i,fontsize=50)\n    xx.tick_params(labelsize=40)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:28:10.816096Z","iopub.execute_input":"2022-07-27T09:28:10.816871Z","iopub.status.idle":"2022-07-27T09:33:51.506563Z","shell.execute_reply.started":"2022-07-27T09:28:10.816817Z","shell.execute_reply":"2022-07-27T09:33:51.505394Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Although some items are in the same item category, their average price and sales are not the same.","metadata":{}},{"cell_type":"code","source":"mon_cnt=train.groupby(['date_block_num'],as_index=False)['item_cnt_day'].agg('sum')\nshop_mon_cnt=train.groupby(['date_block_num','shop_id'],as_index=False)['item_cnt_day'].agg('sum')\nitemc_mon_cnt=train.groupby(['date_block_num','item_category_id'],as_index=False)['item_cnt_day'].agg('sum')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:33:51.508060Z","iopub.execute_input":"2022-07-27T09:33:51.508383Z","iopub.status.idle":"2022-07-27T09:33:51.975339Z","shell.execute_reply.started":"2022-07-27T09:33:51.508352Z","shell.execute_reply":"2022-07-27T09:33:51.974287Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\nwarnings.filterwarnings(\"ignore\")\ninit_notebook_mode(connected=True)\ntemp=dict(layout=go.Layout(font=dict(family=\"Franklin Gothic\", size=12), \n                           height=500, width=1000))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:33:51.977022Z","iopub.execute_input":"2022-07-27T09:33:51.977989Z","iopub.status.idle":"2022-07-27T09:33:51.985823Z","shell.execute_reply.started":"2022-07-27T09:33:51.977947Z","shell.execute_reply":"2022-07-27T09:33:51.984994Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig=go.Figure()\nfig.add_trace(go.Scatter(x=mon_cnt['date_block_num'], \n                         y=mon_cnt['item_cnt_day'], mode='lines',\n                         line=dict(color='darkred', width=3), \n                         hovertemplate = ''))\nfig.update_layout(template=temp, title=\"Total sales\", \n                  hovermode=\"x unified\", width=1000,height=400,\n                  xaxis_title='Month', yaxis_title='Sale')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:33:51.987116Z","iopub.execute_input":"2022-07-27T09:33:51.987487Z","iopub.status.idle":"2022-07-27T09:33:52.008968Z","shell.execute_reply.started":"2022-07-27T09:33:51.987456Z","shell.execute_reply":"2022-07-27T09:33:52.007628Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The total sale of all the items reached a peak at the 11st month and the 23rd month.","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(10, 6, figsize=(200, 100),sharex=True)\nrow=0\ncol=[0,1,2,3,4,5]*10\nfor i in range(0,60):\n    if (i!=0)&(i%6==0):\n        row+=1\n    df=shop_mon_cnt[shop_mon_cnt['shop_id']==i]\n    xx=sns.lineplot(x=\"date_block_num\", y=\"item_cnt_day\", data=df,linewidth=5,color='blue', ax=axes[row,col[i]])\n    xx.set_title(i,fontsize=60)\n    xx.tick_params(labelsize=60)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:33:52.010873Z","iopub.execute_input":"2022-07-27T09:33:52.012026Z","iopub.status.idle":"2022-07-27T09:34:06.216549Z","shell.execute_reply.started":"2022-07-27T09:33:52.011989Z","shell.execute_reply":"2022-07-27T09:34:06.215227Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most shops have been trading during the given period,but some shops only trades for a very short time,e.g. shop0,shop1,shop14...\nBesides, most of the shops have 2 peaks during the given period. Those peaks often appeared around 11st(Dec.2013) or 23rd (Dec.2014) month,which is similar to the trend of total sale. \n","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(14, 6, figsize=(200, 150),sharex=True)\nrow=0\ncol=[0,1,2,3,4,5]*14\nfor i in range(0,84):\n    if (i!=0)&(i%6==0):\n        row+=1\n    df=itemc_mon_cnt[itemc_mon_cnt['item_category_id']==i]\n    xx=sns.lineplot(x=\"date_block_num\", y=\"item_cnt_day\", data=df,linewidth=5,color='purple', ax=axes[row,col[i]])\n    xx.set_title(i,fontsize=60)\n    xx.tick_params(labelsize=60)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:06.218086Z","iopub.execute_input":"2022-07-27T09:34:06.218446Z","iopub.status.idle":"2022-07-27T09:34:32.824555Z","shell.execute_reply.started":"2022-07-27T09:34:06.218413Z","shell.execute_reply":"2022-07-27T09:34:32.821784Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemsale=train.groupby(['item_id'],as_index=False)['item_cnt_day'].agg('sum')\nitemsale=pd.DataFrame(itemsale)\nitemsale=itemsale.sort_values(by=['item_cnt_day'],ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:32.826899Z","iopub.execute_input":"2022-07-27T09:34:32.828138Z","iopub.status.idle":"2022-07-27T09:34:32.942686Z","shell.execute_reply.started":"2022-07-27T09:34:32.828098Z","shell.execute_reply":"2022-07-27T09:34:32.941735Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=itemsale['item_cnt_day'])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:32.944090Z","iopub.execute_input":"2022-07-27T09:34:32.944870Z","iopub.status.idle":"2022-07-27T09:34:33.143538Z","shell.execute_reply.started":"2022-07-27T09:34:32.944834Z","shell.execute_reply":"2022-07-27T09:34:33.142221Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def RANGE(x):\n    if x <= 10:\n        return 0\n    elif 10 < x <=100:\n        return 1\n    elif 100 < x <= 500:\n        return 2\n    elif 500 < x <= 1000:\n        return 3\n    elif 1000< x:\n        return 4","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.145325Z","iopub.execute_input":"2022-07-27T09:34:33.145645Z","iopub.status.idle":"2022-07-27T09:34:33.151842Z","shell.execute_reply.started":"2022-07-27T09:34:33.145614Z","shell.execute_reply":"2022-07-27T09:34:33.150543Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itemsale['cnt_range']=itemsale['item_cnt_day'].map(RANGE)\nitemsale['cnt_range']=itemsale['cnt_range'].astype('str')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.153898Z","iopub.execute_input":"2022-07-27T09:34:33.154464Z","iopub.status.idle":"2022-07-27T09:34:33.203463Z","shell.execute_reply.started":"2022-07-27T09:34:33.154414Z","shell.execute_reply":"2022-07-27T09:34:33.202405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\nwarnings.filterwarnings(\"ignore\")\ninit_notebook_mode(connected=True)\ntemp=dict(layout=go.Layout(font=dict(family=\"Franklin Gothic\", size=12), \n                           height=500, width=1000))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.205652Z","iopub.execute_input":"2022-07-27T09:34:33.206224Z","iopub.status.idle":"2022-07-27T09:34:33.215085Z","shell.execute_reply.started":"2022-07-27T09:34:33.206146Z","shell.execute_reply":"2022-07-27T09:34:33.214219Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target=itemsale.cnt_range.value_counts(normalize=True)\n#target.rename(index={0:'<10',1:'<100',2:'<500',3:'<1000',4:'>1000'},inplace=True)\nlabel=['[10,100)','<10','[100,500)','[500,1000)','>1000']\npal, color=['cornsilk','seashell','honeydew','aliceblue','mistyrose'], ['gold','sandybrown','darkseagreen','powderblue','tomato']\nfig=go.Figure()\nfig.add_trace(go.Pie(labels=label, values=target*100, hole=.45, \n                     showlegend=True,sort=False, \n                     marker=dict(colors=color,line=dict(color=pal,width=2.5)),\n                     hovertemplate = \"%{label} Accounts: %{value:.2f}%<extra></extra>\"))\nfig.update_layout(template=temp, title='Sales Distribution', \n                  legend=dict(traceorder='reversed',y=1.05,x=0),\n                  uniformtext_minsize=15, uniformtext_mode='hide',width=700)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.216506Z","iopub.execute_input":"2022-07-27T09:34:33.216872Z","iopub.status.idle":"2022-07-27T09:34:33.244863Z","shell.execute_reply.started":"2022-07-27T09:34:33.216841Z","shell.execute_reply":"2022-07-27T09:34:33.244011Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of the sales are between 0 and 100, accounts for 71% of all the items. Only 2.78% item sell over 1000 in the given period.","metadata":{}},{"cell_type":"code","source":"itempri=train.groupby(['item_id'],as_index=False)['item_price'].agg('mean')\nitempri=pd.DataFrame(itempri)\nitempri=itempri.sort_values(by=['item_price'],ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.246297Z","iopub.execute_input":"2022-07-27T09:34:33.246583Z","iopub.status.idle":"2022-07-27T09:34:33.351008Z","shell.execute_reply.started":"2022-07-27T09:34:33.246553Z","shell.execute_reply":"2022-07-27T09:34:33.349970Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=itempri['item_price'])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.352742Z","iopub.execute_input":"2022-07-27T09:34:33.353071Z","iopub.status.idle":"2022-07-27T09:34:33.544901Z","shell.execute_reply.started":"2022-07-27T09:34:33.353041Z","shell.execute_reply":"2022-07-27T09:34:33.543009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=itempri.iloc[1:-1,1])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.546302Z","iopub.execute_input":"2022-07-27T09:34:33.547890Z","iopub.status.idle":"2022-07-27T09:34:33.684591Z","shell.execute_reply.started":"2022-07-27T09:34:33.547857Z","shell.execute_reply":"2022-07-27T09:34:33.683058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def PRI(x):\n    if x < 100:\n        return 0\n    elif 100 <= x <500:\n        return 1\n    elif 500 <= x < 1000:\n        return 2\n    elif 1000 <= x < 5000:\n        return 3\n    elif 5000<= x:\n        return 4","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.686964Z","iopub.execute_input":"2022-07-27T09:34:33.688481Z","iopub.status.idle":"2022-07-27T09:34:33.696725Z","shell.execute_reply.started":"2022-07-27T09:34:33.688431Z","shell.execute_reply":"2022-07-27T09:34:33.695306Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itempri['pri_range']=itempri['item_price'].map(PRI)\nitempri['pri_range']=itempri['pri_range'].astype('str')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.699709Z","iopub.execute_input":"2022-07-27T09:34:33.700525Z","iopub.status.idle":"2022-07-27T09:34:33.753723Z","shell.execute_reply.started":"2022-07-27T09:34:33.700476Z","shell.execute_reply":"2022-07-27T09:34:33.752339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target=itempri.pri_range.value_counts(normalize=True)\n#target.rename(index={0:'<100',1:'<500',2:'<1000',3:'<5000',4:'>5000'},inplace=True)\nlabel=['[100,500)','[1000,5000)','[500,1000)','<100','>5000']\npal, color=['cornsilk','seashell','honeydew','aliceblue','mistyrose'], ['gold','sandybrown','darkseagreen','powderblue','tomato']\nfig=go.Figure()\nfig.add_trace(go.Pie(labels=label, values=target*100, hole=.45, \n                     showlegend=True,sort=False, \n                     marker=dict(colors=color,line=dict(color=pal,width=2.5)),\n                     hovertemplate = \"%{label} Accounts: %{value:.2f}%<extra></extra>\"))\nfig.update_layout(template=temp, title='Mean Price Distribution', \n                  legend=dict(traceorder='reversed',y=1.05,x=0),\n                  uniformtext_minsize=15, uniformtext_mode='hide',width=700)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:34:33.755574Z","iopub.execute_input":"2022-07-27T09:34:33.756048Z","iopub.status.idle":"2022-07-27T09:34:33.789600Z","shell.execute_reply.started":"2022-07-27T09:34:33.756004Z","shell.execute_reply":"2022-07-27T09:34:33.786310Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most items' mean prices are from 100 to 500, only 1.71% of these items' mean prices are greater than 5000.","metadata":{}}]}