{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('../input/store-sales-time-series-forecasting'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as mdates","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:00.334219Z","iopub.execute_input":"2022-07-07T13:37:00.335019Z","iopub.status.idle":"2022-07-07T13:37:00.364099Z","shell.execute_reply.started":"2022-07-07T13:37:00.334918Z","shell.execute_reply":"2022-07-07T13:37:00.363278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from warnings import simplefilter\nsimplefilter(\"ignore\")  # ignore warnings to clean up output cells\n\nimport gc # for garbage cleaning\n\n%config Completer.use_jedi = False # for autocomplete","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:00.393251Z","iopub.execute_input":"2022-07-07T13:37:00.393726Z","iopub.status.idle":"2022-07-07T13:37:00.407168Z","shell.execute_reply.started":"2022-07-07T13:37:00.393682Z","shell.execute_reply":"2022-07-07T13:37:00.405904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('../input/store-sales-time-series-forecasting'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:00.443465Z","iopub.execute_input":"2022-07-07T13:37:00.444617Z","iopub.status.idle":"2022-07-07T13:37:00.451861Z","shell.execute_reply.started":"2022-07-07T13:37:00.444562Z","shell.execute_reply":"2022-07-07T13:37:00.451012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/oil.csv\")\nholiday_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/holidays_events.csv\")\nstore_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/stores.csv\")\ntrain_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/train.csv\")\ntest_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/test.csv\")\ntransaction_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/transactions.csv\")\nsample = pd.read_csv(\"../input/store-sales-time-series-forecasting/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:00.494335Z","iopub.execute_input":"2022-07-07T13:37:00.495130Z","iopub.status.idle":"2022-07-07T13:37:03.259486Z","shell.execute_reply.started":"2022-07-07T13:37:00.495072Z","shell.execute_reply":"2022-07-07T13:37:03.258510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.261031Z","iopub.execute_input":"2022-07-07T13:37:03.261455Z","iopub.status.idle":"2022-07-07T13:37:03.281947Z","shell.execute_reply.started":"2022-07-07T13:37:03.261422Z","shell.execute_reply":"2022-07-07T13:37:03.280867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.283253Z","iopub.execute_input":"2022-07-07T13:37:03.283677Z","iopub.status.idle":"2022-07-07T13:37:03.295330Z","shell.execute_reply.started":"2022-07-07T13:37:03.283646Z","shell.execute_reply":"2022-07-07T13:37:03.294250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.297867Z","iopub.execute_input":"2022-07-07T13:37:03.298543Z","iopub.status.idle":"2022-07-07T13:37:03.314032Z","shell.execute_reply.started":"2022-07-07T13:37:03.298499Z","shell.execute_reply":"2022-07-07T13:37:03.312741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.315420Z","iopub.execute_input":"2022-07-07T13:37:03.316201Z","iopub.status.idle":"2022-07-07T13:37:03.339280Z","shell.execute_reply.started":"2022-07-07T13:37:03.316165Z","shell.execute_reply":"2022-07-07T13:37:03.338247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('there are {} different product families'.format(train_data.family.nunique()))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.340705Z","iopub.execute_input":"2022-07-07T13:37:03.341353Z","iopub.status.idle":"2022-07-07T13:37:03.525335Z","shell.execute_reply.started":"2022-07-07T13:37:03.341309Z","shell.execute_reply":"2022-07-07T13:37:03.524137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('there are {} different stores'.format(train_data.store_nbr.nunique()))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.526969Z","iopub.execute_input":"2022-07-07T13:37:03.527412Z","iopub.status.idle":"2022-07-07T13:37:03.552100Z","shell.execute_reply.started":"2022-07-07T13:37:03.527353Z","shell.execute_reply":"2022-07-07T13:37:03.551032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_train = train_data.copy()\n\ndel train_data\n\nprocess_train['date'] = pd.to_datetime(process_train['date'])\nprocess_train = process_train.set_index('date')\nprocess_train = process_train.drop('id',axis = 1)\nprocess_train[['store_nbr','family']].astype('category') #to reduce ram usage\nprocess_train","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:03.553233Z","iopub.execute_input":"2022-07-07T13:37:03.553611Z","iopub.status.idle":"2022-07-07T13:37:04.578805Z","shell.execute_reply.started":"2022-07-07T13:37:03.553584Z","shell.execute_reply":"2022-07-07T13:37:04.577634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:04.580698Z","iopub.execute_input":"2022-07-07T13:37:04.581176Z","iopub.status.idle":"2022-07-07T13:37:04.738997Z","shell.execute_reply.started":"2022-07-07T13:37:04.581129Z","shell.execute_reply":"2022-07-07T13:37:04.737992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_day_gap(df):\n    temp = df.reset_index().groupby(['date']).sales.sum()\n    return (temp.index[1:]-temp.index[:-1]).value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:04.742680Z","iopub.execute_input":"2022-07-07T13:37:04.743009Z","iopub.status.idle":"2022-07-07T13:37:04.748788Z","shell.execute_reply.started":"2022-07-07T13:37:04.742981Z","shell.execute_reply":"2022-07-07T13:37:04.747751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_day_gap(process_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:04.750536Z","iopub.execute_input":"2022-07-07T13:37:04.750832Z","iopub.status.idle":"2022-07-07T13:37:04.890486Z","shell.execute_reply.started":"2022-07-07T13:37:04.750807Z","shell.execute_reply":"2022-07-07T13:37:04.889395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = process_train.reset_index().groupby(['date']).sales.sum().to_frame()\ngap = (temp.index[1:]-temp.index[:-1]).to_list()\ngap.insert(0,'first day') \ntemp['gap'] = gap\n\nday_skip = temp.groupby('gap').get_group(temp.gap.unique()[2])\nday_skip","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:04.891787Z","iopub.execute_input":"2022-07-07T13:37:04.892081Z","iopub.status.idle":"2022-07-07T13:37:05.068327Z","shell.execute_reply.started":"2022-07-07T13:37:04.892055Z","shell.execute_reply":"2022-07-07T13:37:05.067440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del temp","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:05.069793Z","iopub.execute_input":"2022-07-07T13:37:05.070214Z","iopub.status.idle":"2022-07-07T13:37:05.076548Z","shell.execute_reply.started":"2022-07-07T13:37:05.070171Z","shell.execute_reply":"2022-07-07T13:37:05.075787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_train.groupby('date').sales.sum().sort_values().head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:05.077527Z","iopub.execute_input":"2022-07-07T13:37:05.077920Z","iopub.status.idle":"2022-07-07T13:37:05.151934Z","shell.execute_reply.started":"2022-07-07T13:37:05.077880Z","shell.execute_reply":"2022-07-07T13:37:05.150927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_fam_sale = process_train.groupby(['date','family']).sum().sales\nunstack = date_fam_sale.unstack()\nunstack = unstack.resample('1M').sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:05.153332Z","iopub.execute_input":"2022-07-07T13:37:05.153704Z","iopub.status.idle":"2022-07-07T13:37:05.618821Z","shell.execute_reply.started":"2022-07-07T13:37:05.153673Z","shell.execute_reply":"2022-07-07T13:37:05.617590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(20, 10))\nax.set(title=\"'Total Monthly Sales From each Family\")\nplt.stackplot(unstack.index,unstack.T,labels=unstack.T.index)\nax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))\nplt.axvline(x=pd.Timestamp('2016-04-16'),color='black',linestyle='--',linewidth=5,alpha=0.7)\nplt.text(pd.Timestamp('2016-04-20'),30000000,'  The Earthquake',rotation=360,c='black',size=17)\nplt.xticks(rotation=90)\nplt.legend(loc='lower center',bbox_to_anchor=(0.5,-0.2),ncol=11)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:05.621396Z","iopub.execute_input":"2022-07-07T13:37:05.622170Z","iopub.status.idle":"2022-07-07T13:37:06.747352Z","shell.execute_reply.started":"2022-07-07T13:37:05.622121Z","shell.execute_reply":"2022-07-07T13:37:06.746262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del unstack\ndel date_fam_sale","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:06.748480Z","iopub.execute_input":"2022-07-07T13:37:06.749398Z","iopub.status.idle":"2022-07-07T13:37:06.753037Z","shell.execute_reply.started":"2022-07-07T13:37:06.749351Z","shell.execute_reply":"2022-07-07T13:37:06.752200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"month_family = process_train.groupby('family').resample('M').sales.sum()\n\nfig, ax = plt.subplots(figsize=(7,7))\nplt.barh(month_family.groupby('family').sum().sort_values().index,month_family.groupby('family').sum().sort_values())\nax.set(title='Total Sales by Product Family')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:06.754176Z","iopub.execute_input":"2022-07-07T13:37:06.754876Z","iopub.status.idle":"2022-07-07T13:37:07.696441Z","shell.execute_reply.started":"2022-07-07T13:37:06.754833Z","shell.execute_reply":"2022-07-07T13:37:07.695440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sale = month_family.sum()\nfamily_sale = month_family.groupby('family').sum().sort_values()\nproportion = ((family_sale/total_sale)*100).sort_values(ascending=False)\nproportion = pd.DataFrame(proportion)\nproportion.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:07.698154Z","iopub.execute_input":"2022-07-07T13:37:07.698603Z","iopub.status.idle":"2022-07-07T13:37:07.711799Z","shell.execute_reply.started":"2022-07-07T13:37:07.698561Z","shell.execute_reply":"2022-07-07T13:37:07.710546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del family_sale","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:07.713324Z","iopub.execute_input":"2022-07-07T13:37:07.713661Z","iopub.status.idle":"2022-07-07T13:37:07.718498Z","shell.execute_reply.started":"2022-07-07T13:37:07.713616Z","shell.execute_reply":"2022-07-07T13:37:07.717664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(18, 7))\nax.set(title=\"'Total Sales Across All Stores\")\ntotal_sales = process_train.sales.groupby(\"date\").sum()\nplt.plot(total_sales)\n\nax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))\nplt.xticks(rotation=70)\nplt.axvline(x=pd.Timestamp('2016-04-16'),color='r',linestyle='--',linewidth=4,alpha=0.3)\nplt.text(pd.Timestamp('2016-04-20'),1400000,'The Earthquake',rotation=360,c='r')\n\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:07.719637Z","iopub.execute_input":"2022-07-07T13:37:07.720190Z","iopub.status.idle":"2022-07-07T13:37:08.580232Z","shell.execute_reply.started":"2022-07-07T13:37:07.720161Z","shell.execute_reply":"2022-07-07T13:37:08.579414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_sale_dict = {}\nfor i in process_train.store_nbr.unique():\n    daily_sale = process_train[process_train['store_nbr']==i]\n    daily_sale_dict[i] = daily_sale","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:08.581499Z","iopub.execute_input":"2022-07-07T13:37:08.582011Z","iopub.status.idle":"2022-07-07T13:37:08.945314Z","shell.execute_reply.started":"2022-07-07T13:37:08.581978Z","shell.execute_reply":"2022-07-07T13:37:08.944347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(30,30))\nfor i in daily_sale_dict.keys():\n    plt.subplot(8,7,i)\n    plt.title('Store {} sale'.format(i))\n    plt.tight_layout(pad=5)\n    sale = daily_sale_dict[i].sales\n    sale.plot()\n    plt.axvline(x=pd.Timestamp('2016-04-16'),color='r',linestyle='--',linewidth=2,alpha=0.3)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:08.946799Z","iopub.execute_input":"2022-07-07T13:37:08.947305Z","iopub.status.idle":"2022-07-07T13:37:54.029174Z","shell.execute_reply.started":"2022-07-07T13:37:08.947264Z","shell.execute_reply":"2022-07-07T13:37:54.028166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"by_fam_dic = {}\nfam_list = process_train.family.unique()\n\nfor fam in fam_list:\n    by_fam_dic[fam] = process_train[process_train['family']==fam].sales","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:37:54.030991Z","iopub.execute_input":"2022-07-07T13:37:54.031388Z","iopub.status.idle":"2022-07-07T13:38:00.346147Z","shell.execute_reply.started":"2022-07-07T13:37:54.031339Z","shell.execute_reply":"2022-07-07T13:38:00.344990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(30,50))\n\nfor i,fam in enumerate(by_fam_dic.keys()):\n    plt.subplot(11,3,i+1)\n    plt.title('{} sale'.format(fam))\n    plt.tight_layout(pad=5)\n    sale = by_fam_dic[fam]\n    sale.plot()\n    plt.axvline(x=pd.Timestamp('2016-04-16'),color='r',linestyle='--',linewidth=2,alpha=0.3)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:00.347709Z","iopub.execute_input":"2022-07-07T13:38:00.348034Z","iopub.status.idle":"2022-07-07T13:38:28.750201Z","shell.execute_reply.started":"2022-07-07T13:38:00.347994Z","shell.execute_reply":"2022-07-07T13:38:28.749178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del by_fam_dic\ndel fam_list","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:28.751761Z","iopub.execute_input":"2022-07-07T13:38:28.752137Z","iopub.status.idle":"2022-07-07T13:38:28.757400Z","shell.execute_reply.started":"2022-07-07T13:38:28.752103Z","shell.execute_reply":"2022-07-07T13:38:28.756412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store_data.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:28.758723Z","iopub.execute_input":"2022-07-07T13:38:28.759551Z","iopub.status.idle":"2022-07-07T13:38:28.777926Z","shell.execute_reply.started":"2022-07-07T13:38:28.759520Z","shell.execute_reply":"2022-07-07T13:38:28.776959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_df = process_train.merge(store_data,on='store_nbr')\njoin_df.set_index(process_train.index)\njoin_df['date'] = process_train.index\njoin_df = join_df.set_index('date')\n\njoin_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:28.785032Z","iopub.execute_input":"2022-07-07T13:38:28.785588Z","iopub.status.idle":"2022-07-07T13:38:30.369777Z","shell.execute_reply.started":"2022-07-07T13:38:28.785554Z","shell.execute_reply":"2022-07-07T13:38:30.368681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" def show_type_df(join_store_type_df):\n    mean_sales_type = join_store_type_df.groupby('type').sales.mean()\n    median_sales_type = join_store_type_df.groupby('type').sales.median()\n    number=join_store_type_df.groupby('type').store_nbr.nunique()\n\n    type_df = pd.DataFrame((mean_sales_type,median_sales_type,number))\n    type_df = type_df.T\n    type_df.columns = ['mean','median','number of store']\n\n    return type_df","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:30.371545Z","iopub.execute_input":"2022-07-07T13:38:30.372610Z","iopub.status.idle":"2022-07-07T13:38:30.379010Z","shell.execute_reply.started":"2022-07-07T13:38:30.372564Z","shell.execute_reply":"2022-07-07T13:38:30.377956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_type_df(join_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:30.380504Z","iopub.execute_input":"2022-07-07T13:38:30.380777Z","iopub.status.idle":"2022-07-07T13:38:31.049441Z","shell.execute_reply.started":"2022-07-07T13:38:30.380752Z","shell.execute_reply":"2022-07-07T13:38:31.048277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_cluster_summary(join_store_type_df):\n    mean_sales_cluster = join_store_type_df.groupby('cluster').sales.mean()\n    median_sales_cluster = join_store_type_df.groupby('cluster').sales.median()\n    number=join_store_type_df.groupby('cluster').store_nbr.nunique()\n\n    cluster_df = pd.DataFrame((mean_sales_cluster,median_sales_cluster,number))\n    cluster_df = cluster_df.T\n    cluster_df.columns = ['mean','median','number of store']\n\n    return cluster_df.sort_values('mean', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:31.050846Z","iopub.execute_input":"2022-07-07T13:38:31.051135Z","iopub.status.idle":"2022-07-07T13:38:31.056492Z","shell.execute_reply.started":"2022-07-07T13:38:31.051108Z","shell.execute_reply":"2022-07-07T13:38:31.055779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_cluster_summary(join_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:31.057506Z","iopub.execute_input":"2022-07-07T13:38:31.058227Z","iopub.status.idle":"2022-07-07T13:38:31.344324Z","shell.execute_reply.started":"2022-07-07T13:38:31.058194Z","shell.execute_reply":"2022-07-07T13:38:31.343160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_city_df(join_store_type_df):    \n    mean_sales_city = join_store_type_df.groupby('city').sales.mean()\n    median_sales_city = join_store_type_df.groupby('city').sales.median()\n    number=join_store_type_df.groupby('city').store_nbr.nunique()\n\n    city_df = pd.DataFrame((mean_sales_city,median_sales_city,number))\n    city_df = city_df.T\n    city_df.columns = ['mean','median','number of store']\n\n    return city_df.sort_values('mean', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:31.345846Z","iopub.execute_input":"2022-07-07T13:38:31.346268Z","iopub.status.idle":"2022-07-07T13:38:31.354009Z","shell.execute_reply.started":"2022-07-07T13:38:31.346225Z","shell.execute_reply":"2022-07-07T13:38:31.352781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_city_df(join_df)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:31.355350Z","iopub.execute_input":"2022-07-07T13:38:31.356039Z","iopub.status.idle":"2022-07-07T13:38:32.069807Z","shell.execute_reply.started":"2022-07-07T13:38:31.355996Z","shell.execute_reply":"2022-07-07T13:38:32.068799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del join_df","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:32.071305Z","iopub.execute_input":"2022-07-07T13:38:32.071824Z","iopub.status.idle":"2022-07-07T13:38:32.076048Z","shell.execute_reply.started":"2022-07-07T13:38:32.071787Z","shell.execute_reply":"2022-07-07T13:38:32.075223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\na = process_train[[\"store_nbr\", \"sales\"]]\na[\"ind\"] = 1\na[\"ind\"] = a.groupby(\"store_nbr\").ind.cumsum().values \na = pd.pivot(a, index = \"ind\", columns = \"store_nbr\", values = \"sales\").corr() \n\nmask = np.triu(a.corr())\n\nplt.figure(figsize=(20, 20))\nsns.heatmap(a,\n         annot=True,\n         fmt='.1f',\n         square=True,\n         mask=mask,\n         linewidths=1,\n         cbar=False)\nplt.title(\"Correlations among stores\",fontsize = 24)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:32.077346Z","iopub.execute_input":"2022-07-07T13:38:32.078367Z","iopub.status.idle":"2022-07-07T13:38:39.014694Z","shell.execute_reply.started":"2022-07-07T13:38:32.078331Z","shell.execute_reply":"2022-07-07T13:38:39.013622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:39.015811Z","iopub.execute_input":"2022-07-07T13:38:39.016103Z","iopub.status.idle":"2022-07-07T13:38:39.519233Z","shell.execute_reply.started":"2022-07-07T13:38:39.016077Z","shell.execute_reply":"2022-07-07T13:38:39.517898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store_20 = process_train.groupby(['store_nbr','date']).sales.sum().loc[20]\nstore_20.loc['2013-01-01':'2015-01-01']","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:39.520617Z","iopub.execute_input":"2022-07-07T13:38:39.521297Z","iopub.status.idle":"2022-07-07T13:38:39.702184Z","shell.execute_reply.started":"2022-07-07T13:38:39.521262Z","shell.execute_reply":"2022-07-07T13:38:39.701417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del store_20","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:39.703513Z","iopub.execute_input":"2022-07-07T13:38:39.703889Z","iopub.status.idle":"2022-07-07T13:38:39.708577Z","shell.execute_reply.started":"2022-07-07T13:38:39.703857Z","shell.execute_reply":"2022-07-07T13:38:39.707584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Spearman Rank Correlation = {}'.format(\n    process_train.sales.corr(process_train.onpromotion,method='spearman')))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:39.710072Z","iopub.execute_input":"2022-07-07T13:38:39.710415Z","iopub.status.idle":"2022-07-07T13:38:40.527489Z","shell.execute_reply.started":"2022-07-07T13:38:39.710355Z","shell.execute_reply":"2022-07-07T13:38:40.526155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime as dt\n\ndef show_dow_sales():\n    day_group = process_train.reset_index()[['date','sales']]\n    day_group = day_group.groupby('date')\n    day_group = day_group.sales.mean().to_frame()\n    day_group['dow'] = day_group.index.day_of_week\n    day_group = day_group.groupby('dow').sum()\n    plt.bar(day_group.index,day_group['sales'])\n    plt.title('Average sales on day of week')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:40.528775Z","iopub.execute_input":"2022-07-07T13:38:40.529083Z","iopub.status.idle":"2022-07-07T13:38:40.536183Z","shell.execute_reply.started":"2022-07-07T13:38:40.529056Z","shell.execute_reply":"2022-07-07T13:38:40.534844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_month_group_sale():\n    month_group = process_train['sales'].to_frame()\n    month_group['moy'] = month_group.index.month\n    month_group = month_group.groupby('moy').sales.mean().to_frame()\n    plt.bar(month_group.index,month_group['sales'])\n    plt.title('Average sales on Month of Year')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:40.537473Z","iopub.execute_input":"2022-07-07T13:38:40.537799Z","iopub.status.idle":"2022-07-07T13:38:40.551313Z","shell.execute_reply.started":"2022-07-07T13:38:40.537771Z","shell.execute_reply":"2022-07-07T13:38:40.550425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_dow_sales()\nshow_month_group_sale()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:40.552200Z","iopub.execute_input":"2022-07-07T13:38:40.552512Z","iopub.status.idle":"2022-07-07T13:38:41.349066Z","shell.execute_reply.started":"2022-07-07T13:38:40.552484Z","shell.execute_reply":"2022-07-07T13:38:41.347849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holiday_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/holidays_events.csv\",index_col='date',parse_dates=['date'])\nholiday_data","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.350625Z","iopub.execute_input":"2022-07-07T13:38:41.351623Z","iopub.status.idle":"2022-07-07T13:38:41.372198Z","shell.execute_reply.started":"2022-07-07T13:38:41.351575Z","shell.execute_reply":"2022-07-07T13:38:41.370955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holiday_data.locale_name.value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.373848Z","iopub.execute_input":"2022-07-07T13:38:41.374334Z","iopub.status.idle":"2022-07-07T13:38:41.383125Z","shell.execute_reply.started":"2022-07-07T13:38:41.374292Z","shell.execute_reply":"2022-07-07T13:38:41.381957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ny_dic = {'type': 'Holiday','locale':'National','locale_name':'Ecuador','description': 'New Year Day','transferred':'False'}\nny_date = pd.to_datetime(['2012-01-01','2013-01-01','2014-01-01','2015-01-01','2016-01-01','2017-01-01','2018-01-01'])\n\ncm_dic = {'type': 'Holiday','locale':'National','locale_name':'Ecuador','description': 'Christmas Day','transferred':'False'}\ncm_date = pd.to_datetime(['2012-12-25','2013-12-25','2014-12-25','2015-12-25','2016-12-25','2017-12-25','2018-12-25'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.384651Z","iopub.execute_input":"2022-07-07T13:38:41.385267Z","iopub.status.idle":"2022-07-07T13:38:41.394612Z","shell.execute_reply.started":"2022-07-07T13:38:41.385226Z","shell.execute_reply":"2022-07-07T13:38:41.393609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for date in ny_date:\n    holiday_data.loc[date] = ['Holiday','National', 'Ecuador', 'New Year day','False']\n    \nfor date in cm_date:\n    holiday_data.loc[date] = ['Holiday','National', 'Ecuador', 'Christmas day','False']","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.395851Z","iopub.execute_input":"2022-07-07T13:38:41.396330Z","iopub.status.idle":"2022-07-07T13:38:41.428826Z","shell.execute_reply.started":"2022-07-07T13:38:41.396291Z","shell.execute_reply":"2022-07-07T13:38:41.427743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holiday_data = holiday_data.sort_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.430580Z","iopub.execute_input":"2022-07-07T13:38:41.431018Z","iopub.status.idle":"2022-07-07T13:38:41.437404Z","shell.execute_reply.started":"2022-07-07T13:38:41.430973Z","shell.execute_reply":"2022-07-07T13:38:41.436211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar = pd.DataFrame(index = pd.date_range('2013-01-01','2017-08-31'))\ncalendar = calendar.join(holiday_data).fillna(0)\ndel holiday_data\ncalendar","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.439020Z","iopub.execute_input":"2022-07-07T13:38:41.439400Z","iopub.status.idle":"2022-07-07T13:38:41.469720Z","shell.execute_reply.started":"2022-07-07T13:38:41.439348Z","shell.execute_reply":"2022-07-07T13:38:41.468598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar['dow'] = calendar.index.dayofweek+1\ncalendar['workday'] = True\ncalendar.loc[calendar['dow']>5 , 'workday'] = False\n\ncalendar.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.471058Z","iopub.execute_input":"2022-07-07T13:38:41.472077Z","iopub.status.idle":"2022-07-07T13:38:41.489670Z","shell.execute_reply.started":"2022-07-07T13:38:41.472031Z","shell.execute_reply":"2022-07-07T13:38:41.488843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.loc[(calendar['type']=='Holiday') & (calendar['locale'].str.contains('National')), 'workday'] = False\ncalendar.loc[(calendar['type']=='Additional') & (calendar['locale'].str.contains('National')), 'workday'] = False\ncalendar.loc[(calendar['type']=='Bridge') & (calendar['locale'].str.contains('National')), 'workday'] = False\ncalendar.loc[(calendar['type']=='Transfer') & (calendar['locale'].str.contains('National')), 'workday'] = False\n\ncalendar.loc[calendar['type']=='Work Day' , 'workday'] = True","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.490823Z","iopub.execute_input":"2022-07-07T13:38:41.491170Z","iopub.status.idle":"2022-07-07T13:38:41.515513Z","shell.execute_reply.started":"2022-07-07T13:38:41.491138Z","shell.execute_reply":"2022-07-07T13:38:41.514467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.where(calendar['transferred'] == True).dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.516844Z","iopub.execute_input":"2022-07-07T13:38:41.517455Z","iopub.status.idle":"2022-07-07T13:38:41.538119Z","shell.execute_reply.started":"2022-07-07T13:38:41.517420Z","shell.execute_reply":"2022-07-07T13:38:41.537408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.loc[(calendar['transferred'] == True), 'workday'] = True","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.539277Z","iopub.execute_input":"2022-07-07T13:38:41.539592Z","iopub.status.idle":"2022-07-07T13:38:41.545199Z","shell.execute_reply.started":"2022-07-07T13:38:41.539565Z","shell.execute_reply":"2022-07-07T13:38:41.544316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.where(calendar['description'].str.contains('futbol')).dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.546455Z","iopub.execute_input":"2022-07-07T13:38:41.546890Z","iopub.status.idle":"2022-07-07T13:38:41.579540Z","shell.execute_reply.started":"2022-07-07T13:38:41.546863Z","shell.execute_reply":"2022-07-07T13:38:41.578657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar['is_football'] = 0\ncalendar['is_eq'] = 0\n\ncalendar.loc[(calendar['is_football'] == 0) & (calendar['description'].str.contains('futbol')), 'is_football'] = 1\ncalendar.loc[(calendar['is_eq'] == 0) & (calendar['description'].str.contains('Terremoto')), 'is_eq'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.580771Z","iopub.execute_input":"2022-07-07T13:38:41.581245Z","iopub.status.idle":"2022-07-07T13:38:41.594648Z","shell.execute_reply.started":"2022-07-07T13:38:41.581216Z","shell.execute_reply":"2022-07-07T13:38:41.593692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.where(calendar['is_football']==1).dropna().head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.596126Z","iopub.execute_input":"2022-07-07T13:38:41.597325Z","iopub.status.idle":"2022-07-07T13:38:41.629227Z","shell.execute_reply.started":"2022-07-07T13:38:41.597287Z","shell.execute_reply":"2022-07-07T13:38:41.628031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.where(calendar['is_eq']==1).dropna().head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.630479Z","iopub.execute_input":"2022-07-07T13:38:41.630793Z","iopub.status.idle":"2022-07-07T13:38:41.657405Z","shell.execute_reply.started":"2022-07-07T13:38:41.630765Z","shell.execute_reply":"2022-07-07T13:38:41.656438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.loc[calendar['is_football']==1,'description'] = 'football'\ncalendar.loc[calendar['is_eq']==1,'description'] = 'earthquake'","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.660704Z","iopub.execute_input":"2022-07-07T13:38:41.661014Z","iopub.status.idle":"2022-07-07T13:38:41.667714Z","shell.execute_reply.started":"2022-07-07T13:38:41.660985Z","shell.execute_reply":"2022-07-07T13:38:41.666869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales = process_train.groupby('date').sales.sum()\nevent = calendar[calendar['type']=='Event']\n\nevent_merge = event.merge(sales,how='left',left_index=True,right_index=True)\nevent_merge\n\ndel sales\ndel event","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.668764Z","iopub.execute_input":"2022-07-07T13:38:41.669686Z","iopub.status.idle":"2022-07-07T13:38:41.739003Z","shell.execute_reply.started":"2022-07-07T13:38:41.669638Z","shell.execute_reply":"2022-07-07T13:38:41.737697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('mean of daily sale across country: {}'.format(process_train.groupby('date').sales.sum().mean()))\nprint('--------------------')\n\nprint(('mean of sale across country in event day: {}'.format(event_merge.groupby('description').sales.mean())))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.742723Z","iopub.execute_input":"2022-07-07T13:38:41.743174Z","iopub.status.idle":"2022-07-07T13:38:41.810024Z","shell.execute_reply.started":"2022-07-07T13:38:41.743139Z","shell.execute_reply":"2022-07-07T13:38:41.808881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.812392Z","iopub.execute_input":"2022-07-07T13:38:41.812841Z","iopub.status.idle":"2022-07-07T13:38:41.826667Z","shell.execute_reply.started":"2022-07-07T13:38:41.812796Z","shell.execute_reply":"2022-07-07T13:38:41.825625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar['workday'] = calendar['workday'].map({False:0,True:1})\ncalendar['transferred'] = calendar['transferred'].map({'False':0,False:0,True:1})","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.828225Z","iopub.execute_input":"2022-07-07T13:38:41.828568Z","iopub.status.idle":"2022-07-07T13:38:41.840670Z","shell.execute_reply.started":"2022-07-07T13:38:41.828538Z","shell.execute_reply":"2022-07-07T13:38:41.839648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar['is_ny'] = 0\ncalendar['is_christmas'] = 0\ncalendar['is_shopping'] = 0\n\ncalendar.loc[calendar['description'] == 'New Year day', 'is_ny'] = 1\ncalendar.loc[calendar['description'] == 'Christmas day', 'is_christmas'] = 1\ncalendar.loc[calendar['description'] == 'Black Friday', 'is_shopping'] = 1\ncalendar.loc[calendar['description'] == 'Cyber Monday' , 'is_shopping'] = 1","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.841997Z","iopub.execute_input":"2022-07-07T13:38:41.842474Z","iopub.status.idle":"2022-07-07T13:38:41.855098Z","shell.execute_reply.started":"2022-07-07T13:38:41.842444Z","shell.execute_reply":"2022-07-07T13:38:41.854406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.loc['2014-12-25'].to_frame().T","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.856323Z","iopub.execute_input":"2022-07-07T13:38:41.856870Z","iopub.status.idle":"2022-07-07T13:38:41.883429Z","shell.execute_reply.started":"2022-07-07T13:38:41.856841Z","shell.execute_reply":"2022-07-07T13:38:41.882596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.884538Z","iopub.execute_input":"2022-07-07T13:38:41.885060Z","iopub.status.idle":"2022-07-07T13:38:41.902955Z","shell.execute_reply.started":"2022-07-07T13:38:41.885025Z","shell.execute_reply":"2022-07-07T13:38:41.902131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"locale_dummy = pd.get_dummies(calendar['locale_name'],prefix='holiday_')\ncalendar = locale_dummy.join(calendar,how='left')\ncalendar = calendar.drop('locale_name',axis=1)\n\ndel locale_dummy","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.904463Z","iopub.execute_input":"2022-07-07T13:38:41.906737Z","iopub.status.idle":"2022-07-07T13:38:41.919542Z","shell.execute_reply.started":"2022-07-07T13:38:41.906689Z","shell.execute_reply":"2022-07-07T13:38:41.918436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar_checkpoint = calendar","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.920588Z","iopub.execute_input":"2022-07-07T13:38:41.921354Z","iopub.status.idle":"2022-07-07T13:38:41.927287Z","shell.execute_reply.started":"2022-07-07T13:38:41.921324Z","shell.execute_reply":"2022-07-07T13:38:41.926538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar_checkpoint = calendar_checkpoint.drop('description',axis = 1)\ncalendar_checkpoint = calendar_checkpoint[~calendar_checkpoint.index.duplicated(keep='first')] \ncalendar_checkpoint = calendar_checkpoint.iloc[:,1:-1]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.928277Z","iopub.execute_input":"2022-07-07T13:38:41.928855Z","iopub.status.idle":"2022-07-07T13:38:41.942818Z","shell.execute_reply.started":"2022-07-07T13:38:41.928820Z","shell.execute_reply":"2022-07-07T13:38:41.941758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar_checkpoint","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.944593Z","iopub.execute_input":"2022-07-07T13:38:41.945369Z","iopub.status.idle":"2022-07-07T13:38:41.974974Z","shell.execute_reply.started":"2022-07-07T13:38:41.945326Z","shell.execute_reply":"2022-07-07T13:38:41.973979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del calendar\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:41.976167Z","iopub.execute_input":"2022-07-07T13:38:41.977255Z","iopub.status.idle":"2022-07-07T13:38:42.435971Z","shell.execute_reply.started":"2022-07-07T13:38:41.977219Z","shell.execute_reply":"2022-07-07T13:38:42.434722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data = pd.read_csv(\"../input/store-sales-time-series-forecasting/oil.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.437322Z","iopub.execute_input":"2022-07-07T13:38:42.437995Z","iopub.status.idle":"2022-07-07T13:38:42.451361Z","shell.execute_reply.started":"2022-07-07T13:38:42.437959Z","shell.execute_reply":"2022-07-07T13:38:42.450386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.head(4)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.452798Z","iopub.execute_input":"2022-07-07T13:38:42.453635Z","iopub.status.idle":"2022-07-07T13:38:42.465353Z","shell.execute_reply.started":"2022-07-07T13:38:42.453575Z","shell.execute_reply":"2022-07-07T13:38:42.464009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.date_range(start = '2013-01-01', end = '2017-08-15' ).difference(oil_data.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.468134Z","iopub.execute_input":"2022-07-07T13:38:42.468721Z","iopub.status.idle":"2022-07-07T13:38:42.477142Z","shell.execute_reply.started":"2022-07-07T13:38:42.468687Z","shell.execute_reply":"2022-07-07T13:38:42.476290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data['date'] = pd.to_datetime(oil_data['date'])\noil_data = oil_data.set_index('date')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.478537Z","iopub.execute_input":"2022-07-07T13:38:42.479527Z","iopub.status.idle":"2022-07-07T13:38:42.489166Z","shell.execute_reply.started":"2022-07-07T13:38:42.479481Z","shell.execute_reply":"2022-07-07T13:38:42.488121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data = oil_data.resample('1D').sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.490457Z","iopub.execute_input":"2022-07-07T13:38:42.490969Z","iopub.status.idle":"2022-07-07T13:38:42.503736Z","shell.execute_reply.started":"2022-07-07T13:38:42.490938Z","shell.execute_reply":"2022-07-07T13:38:42.502710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.505540Z","iopub.execute_input":"2022-07-07T13:38:42.506298Z","iopub.status.idle":"2022-07-07T13:38:42.523450Z","shell.execute_reply.started":"2022-07-07T13:38:42.506256Z","shell.execute_reply":"2022-07-07T13:38:42.522281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.date_range(start = '2013-01-01', end = '2017-08-15' ).difference(oil_data.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.524398Z","iopub.execute_input":"2022-07-07T13:38:42.525502Z","iopub.status.idle":"2022-07-07T13:38:42.535016Z","shell.execute_reply.started":"2022-07-07T13:38:42.525459Z","shell.execute_reply":"2022-07-07T13:38:42.533796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data['dcoilwtico'] = np.where(oil_data['dcoilwtico']==0, np.nan, oil_data['dcoilwtico'])\noil_data['interpolated_price'] = oil_data.dcoilwtico.interpolate()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.536435Z","iopub.execute_input":"2022-07-07T13:38:42.536912Z","iopub.status.idle":"2022-07-07T13:38:42.547940Z","shell.execute_reply.started":"2022-07-07T13:38:42.536883Z","shell.execute_reply":"2022-07-07T13:38:42.547133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data = oil_data.drop('dcoilwtico',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.550027Z","iopub.execute_input":"2022-07-07T13:38:42.551172Z","iopub.status.idle":"2022-07-07T13:38:42.558495Z","shell.execute_reply.started":"2022-07-07T13:38:42.551127Z","shell.execute_reply":"2022-07-07T13:38:42.557525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.560393Z","iopub.execute_input":"2022-07-07T13:38:42.561259Z","iopub.status.idle":"2022-07-07T13:38:42.573837Z","shell.execute_reply.started":"2022-07-07T13:38:42.561215Z","shell.execute_reply":"2022-07-07T13:38:42.572902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data['price_chg'] = oil_data.interpolated_price - oil_data.interpolated_price.shift(1)\noil_data['pct_chg'] = oil_data['price_chg']/oil_data.interpolated_price.shift(-1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.575902Z","iopub.execute_input":"2022-07-07T13:38:42.576949Z","iopub.status.idle":"2022-07-07T13:38:42.584782Z","shell.execute_reply.started":"2022-07-07T13:38:42.576906Z","shell.execute_reply":"2022-07-07T13:38:42.583834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.598371Z","iopub.execute_input":"2022-07-07T13:38:42.599092Z","iopub.status.idle":"2022-07-07T13:38:42.610320Z","shell.execute_reply.started":"2022-07-07T13:38:42.599055Z","shell.execute_reply":"2022-07-07T13:38:42.609470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(15,5))\nplt.plot(oil_data['interpolated_price'])\nplt.title('Oil Price over time')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.611551Z","iopub.execute_input":"2022-07-07T13:38:42.612488Z","iopub.status.idle":"2022-07-07T13:38:42.803982Z","shell.execute_reply.started":"2022-07-07T13:38:42.612452Z","shell.execute_reply":"2022-07-07T13:38:42.803017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_total_sales = total_sales.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.805314Z","iopub.execute_input":"2022-07-07T13:38:42.805641Z","iopub.status.idle":"2022-07-07T13:38:42.810264Z","shell.execute_reply.started":"2022-07-07T13:38:42.805612Z","shell.execute_reply":"2022-07-07T13:38:42.809220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_total_sales.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.811639Z","iopub.execute_input":"2022-07-07T13:38:42.813009Z","iopub.status.idle":"2022-07-07T13:38:42.823718Z","shell.execute_reply.started":"2022-07-07T13:38:42.812965Z","shell.execute_reply":"2022-07-07T13:38:42.822859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_total_sales = daily_total_sales.resample('1D').sum()\ndaily_total_sales","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.824662Z","iopub.execute_input":"2022-07-07T13:38:42.825316Z","iopub.status.idle":"2022-07-07T13:38:42.841897Z","shell.execute_reply.started":"2022-07-07T13:38:42.825284Z","shell.execute_reply":"2022-07-07T13:38:42.840636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_data.interpolated_price.loc['2013-01-01':'2017-08-15']","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.843409Z","iopub.execute_input":"2022-07-07T13:38:42.844028Z","iopub.status.idle":"2022-07-07T13:38:42.860701Z","shell.execute_reply.started":"2022-07-07T13:38:42.843987Z","shell.execute_reply":"2022-07-07T13:38:42.859600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(daily_total_sales,oil_data.interpolated_price.loc['2013-01-01':'2017-08-15'],alpha=0.4)\nplt.ylabel('oil price')\nplt.xlabel('daily total sales')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:42.862265Z","iopub.execute_input":"2022-07-07T13:38:42.862982Z","iopub.status.idle":"2022-07-07T13:38:43.067468Z","shell.execute_reply.started":"2022-07-07T13:38:42.862902Z","shell.execute_reply":"2022-07-07T13:38:43.066339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_total_sales = pd.DataFrame(daily_total_sales)\ndaily_total_sales['sales_chg'] = daily_total_sales['sales']-daily_total_sales['sales'].shift(1)\ndaily_total_sales['sales_pct_chg'] = daily_total_sales['sales_chg']/daily_total_sales['sales'].shift(-1)\n\ndaily_total_sales.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:43.068337Z","iopub.execute_input":"2022-07-07T13:38:43.068644Z","iopub.status.idle":"2022-07-07T13:38:43.086622Z","shell.execute_reply.started":"2022-07-07T13:38:43.068617Z","shell.execute_reply":"2022-07-07T13:38:43.085262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Spearman Rank Correlation = {}'.format(\n    process_train.sales.corr(process_train.onpromotion,method='spearman')))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:43.087954Z","iopub.execute_input":"2022-07-07T13:38:43.088263Z","iopub.status.idle":"2022-07-07T13:38:43.884119Z","shell.execute_reply.started":"2022-07-07T13:38:43.088235Z","shell.execute_reply":"2022-07-07T13:38:43.882805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Pearson Correlation between oil price change and total sales change = {}'.format(\noil_data.price_chg.corr(daily_total_sales.sales_chg,method='pearson')))\nprint('Spearman Rank Correlation between oil price change and total sales change = {}'.format(\noil_data.price_chg.corr(daily_total_sales.sales_chg,method='spearman')))\n\nprint('----------------------------------------------------------------------------')\n\nprint('Pearson Correlation between % oil price change and % total sales change = {}'.format(\noil_data.pct_chg.corr(daily_total_sales.sales_pct_chg,method='pearson')))\nprint('Spearman Rank Correlation between oil price change and total sales change = {}'.format(\noil_data.pct_chg.corr(daily_total_sales.sales_pct_chg,method='spearman')))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:43.885341Z","iopub.execute_input":"2022-07-07T13:38:43.885650Z","iopub.status.idle":"2022-07-07T13:38:43.902142Z","shell.execute_reply.started":"2022-07-07T13:38:43.885623Z","shell.execute_reply":"2022-07-07T13:38:43.901031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import  statsmodels.graphics.tsaplots\n\nax = statsmodels.graphics.tsaplots.plot_pacf(oil_data.interpolated_price.dropna(), \n                                                 lags=31,\n                                                 title = 'oil lag PACF')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:43.903307Z","iopub.execute_input":"2022-07-07T13:38:43.904279Z","iopub.status.idle":"2022-07-07T13:38:44.295809Z","shell.execute_reply.started":"2022-07-07T13:38:43.904247Z","shell.execute_reply":"2022-07-07T13:38:44.294750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_lag = [10,15,26,27]\nfor lag in oil_lag:\n    oil_data['price_lag_{}'.format(lag)] = oil_data.interpolated_price.shift(lag)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:44.297220Z","iopub.execute_input":"2022-07-07T13:38:44.297591Z","iopub.status.idle":"2022-07-07T13:38:44.307639Z","shell.execute_reply.started":"2022-07-07T13:38:44.297560Z","shell.execute_reply":"2022-07-07T13:38:44.306608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_for_lag_coor = oil_data.dropna()\noil_for_lag_coor.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:44.309119Z","iopub.execute_input":"2022-07-07T13:38:44.309450Z","iopub.status.idle":"2022-07-07T13:38:44.330146Z","shell.execute_reply.started":"2022-07-07T13:38:44.309421Z","shell.execute_reply":"2022-07-07T13:38:44.328840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_for_lag_coor = oil_for_lag_coor.merge(daily_total_sales,how='inner',left_index=True,right_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:44.331954Z","iopub.execute_input":"2022-07-07T13:38:44.332765Z","iopub.status.idle":"2022-07-07T13:38:44.339766Z","shell.execute_reply.started":"2022-07-07T13:38:44.332719Z","shell.execute_reply":"2022-07-07T13:38:44.338915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_for_lag_coor = oil_for_lag_coor[['price_lag_10','price_lag_15','price_lag_26','price_lag_27','sales']]\noil_for_lag_coor.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:44.340967Z","iopub.execute_input":"2022-07-07T13:38:44.341493Z","iopub.status.idle":"2022-07-07T13:38:44.364934Z","shell.execute_reply.started":"2022-07-07T13:38:44.341461Z","shell.execute_reply":"2022-07-07T13:38:44.364162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\nlag_col =  ['price_lag_10','price_lag_15','price_lag_26','price_lag_27']\n\nfor lag in lag_col:\n    plt.scatter(oil_for_lag_coor['sales'],oil_for_lag_coor[lag],alpha=0.30)\n    plt.title('Oil Lag {} vs Sales'.format(lag))\n    plt.xlabel('oil price lag {}'.format(lag))\n    plt.ylabel('amount sold')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:44.366144Z","iopub.execute_input":"2022-07-07T13:38:44.367087Z","iopub.status.idle":"2022-07-07T13:38:45.169003Z","shell.execute_reply.started":"2022-07-07T13:38:44.367053Z","shell.execute_reply":"2022-07-07T13:38:45.167619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del [oil_lag,oil_data,oil_for_lag_coor,lag_col]\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:45.170504Z","iopub.execute_input":"2022-07-07T13:38:45.170846Z","iopub.status.idle":"2022-07-07T13:38:45.854991Z","shell.execute_reply.started":"2022-07-07T13:38:45.170815Z","shell.execute_reply":"2022-07-07T13:38:45.853755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = transaction_data.copy()\ntransactions = transactions.set_index('date')\n\ndel transaction_data","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:45.856475Z","iopub.execute_input":"2022-07-07T13:38:45.856844Z","iopub.status.idle":"2022-07-07T13:38:45.869227Z","shell.execute_reply.started":"2022-07-07T13:38:45.856815Z","shell.execute_reply":"2022-07-07T13:38:45.868130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.index = pd.to_datetime(transactions.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:45.870898Z","iopub.execute_input":"2022-07-07T13:38:45.871721Z","iopub.status.idle":"2022-07-07T13:38:45.892828Z","shell.execute_reply.started":"2022-07-07T13:38:45.871647Z","shell.execute_reply":"2022-07-07T13:38:45.891718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:45.893918Z","iopub.execute_input":"2022-07-07T13:38:45.894834Z","iopub.status.idle":"2022-07-07T13:38:45.905667Z","shell.execute_reply.started":"2022-07-07T13:38:45.894799Z","shell.execute_reply":"2022-07-07T13:38:45.904486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transaction_sales_dic(transaction_df,sale_dic):\n    transaction_dic = {}\n    sale_dict = sale_dic.copy()\n\n    for i in transaction_df['store_nbr'].unique():\n        store_transacion = transaction_df.loc[transaction_df['store_nbr'] == i]\n        transaction_dic[i] = store_transacion['transactions']\n        \n    for i in sale_dict.keys():\n        sale_dict[i] = sale_dict[i].groupby(['date','store_nbr']).sales.sum()\n        sale_dict[i] = sale_dict[i].reset_index()\n        sale_dict[i] = sale_dict[i].drop('store_nbr', axis=1)\n        sale_dict[i] = sale_dict[i].groupby('date').sales.sum()\n            \n    return transaction_dic, sale_dict\n        \ndef  series_merge_inner_index(dic1, dic2):\n    merged_dic = {}\n    for key in dic1.keys():\n        merged_dic[key] = dic1[key].to_frame().merge(dic2[key].to_frame(), how='inner',\n                                                    left_index=True, right_index=True)\n    return merged_dic","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:45.907577Z","iopub.execute_input":"2022-07-07T13:38:45.908266Z","iopub.status.idle":"2022-07-07T13:38:45.920058Z","shell.execute_reply.started":"2022-07-07T13:38:45.908226Z","shell.execute_reply":"2022-07-07T13:38:45.918688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transaction_dic, sale_dic = transaction_sales_dic(transactions,daily_sale_dict)\nmerged_sales_transaction = series_merge_inner_index(transaction_dic, sale_dic)\nmerged_sales_transaction[1]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:45.921506Z","iopub.execute_input":"2022-07-07T13:38:45.922226Z","iopub.status.idle":"2022-07-07T13:38:46.434883Z","shell.execute_reply.started":"2022-07-07T13:38:45.922179Z","shell.execute_reply":"2022-07-07T13:38:46.433807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,7))\nfor key in merged_sales_transaction.keys():\n    plt.scatter(merged_sales_transaction[key].transactions,\n                merged_sales_transaction[key].sales)\n\nplt.title('Transaction vs Amount Sold')\nplt.xlabel('transactions number')\nplt.ylabel('amount sold')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:46.436161Z","iopub.execute_input":"2022-07-07T13:38:46.436491Z","iopub.status.idle":"2022-07-07T13:38:47.279761Z","shell.execute_reply.started":"2022-07-07T13:38:46.436462Z","shell.execute_reply":"2022-07-07T13:38:47.278672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import  statsmodels.graphics.tsaplots\n\nfor store_nbr in merged_sales_transaction.keys():\n    ax = statsmodels.graphics.tsaplots.plot_pacf(merged_sales_transaction[store_nbr].transactions, \n                                                 lags=range(16,31),\n                                                 title = 'Store {} transactions lag'.format(store_nbr))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:47.281136Z","iopub.execute_input":"2022-07-07T13:38:47.281501Z","iopub.status.idle":"2022-07-07T13:38:58.141902Z","shell.execute_reply.started":"2022-07-07T13:38:47.281471Z","shell.execute_reply":"2022-07-07T13:38:58.140694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import  statsmodels.graphics.tsaplots\n\ndaily_store_sale_dict = {}\nfor i in daily_sale_dict.keys():\n    daily_store_sale_dict[i] = daily_sale_dict[i].groupby(['date','store_nbr']).sales.sum().to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:58.143314Z","iopub.execute_input":"2022-07-07T13:38:58.143812Z","iopub.status.idle":"2022-07-07T13:38:58.368033Z","shell.execute_reply.started":"2022-07-07T13:38:58.143779Z","shell.execute_reply":"2022-07-07T13:38:58.367041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del daily_sale_dict","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:58.369307Z","iopub.execute_input":"2022-07-07T13:38:58.369620Z","iopub.status.idle":"2022-07-07T13:38:58.380163Z","shell.execute_reply.started":"2022-07-07T13:38:58.369592Z","shell.execute_reply":"2022-07-07T13:38:58.379158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_store_sale_dict[1].head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:58.381250Z","iopub.execute_input":"2022-07-07T13:38:58.381575Z","iopub.status.idle":"2022-07-07T13:38:58.398124Z","shell.execute_reply.started":"2022-07-07T13:38:58.381547Z","shell.execute_reply":"2022-07-07T13:38:58.397181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in daily_store_sale_dict.keys():\n    daily_store_sale_dict[i] = daily_store_sale_dict[i].droplevel(1) ","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:58.399176Z","iopub.execute_input":"2022-07-07T13:38:58.399984Z","iopub.status.idle":"2022-07-07T13:38:58.419318Z","shell.execute_reply.started":"2022-07-07T13:38:58.399951Z","shell.execute_reply":"2022-07-07T13:38:58.418637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in daily_store_sale_dict.keys():\n\n    ax = statsmodels.graphics.tsaplots.plot_pacf(daily_store_sale_dict[i],lags=14, \n                                                 title = 'store {} PA'.format(i))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:38:58.421555Z","iopub.execute_input":"2022-07-07T13:38:58.421954Z","iopub.status.idle":"2022-07-07T13:39:05.425069Z","shell.execute_reply.started":"2022-07-07T13:38:58.421914Z","shell.execute_reply":"2022-07-07T13:39:05.423884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_train","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:05.426946Z","iopub.execute_input":"2022-07-07T13:39:05.427714Z","iopub.status.idle":"2022-07-07T13:39:05.444794Z","shell.execute_reply.started":"2022-07-07T13:39:05.427671Z","shell.execute_reply":"2022-07-07T13:39:05.443899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:05.446309Z","iopub.execute_input":"2022-07-07T13:39:05.448729Z","iopub.status.idle":"2022-07-07T13:39:05.461999Z","shell.execute_reply.started":"2022-07-07T13:39:05.448688Z","shell.execute_reply":"2022-07-07T13:39:05.460456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['sales'] = 0\ntest_data = test_data[['id','date','store_nbr','family','sales','onpromotion']]\n\ntest_data","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:05.463192Z","iopub.execute_input":"2022-07-07T13:39:05.464186Z","iopub.status.idle":"2022-07-07T13:39:05.489295Z","shell.execute_reply.started":"2022-07-07T13:39:05.464087Z","shell.execute_reply":"2022-07-07T13:39:05.488241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_train['id'] = process_train.reset_index().index","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:05.491191Z","iopub.execute_input":"2022-07-07T13:39:05.491963Z","iopub.status.idle":"2022-07-07T13:39:05.539437Z","shell.execute_reply.started":"2022-07-07T13:39:05.491932Z","shell.execute_reply":"2022-07-07T13:39:05.538568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process_train = process_train.reset_index()\nprocess_train = process_train[['id','date','store_nbr','family','sales','onpromotion']]\nprocess_train","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:05.540662Z","iopub.execute_input":"2022-07-07T13:39:05.541539Z","iopub.status.idle":"2022-07-07T13:39:05.677485Z","shell.execute_reply.started":"2022-07-07T13:39:05.541493Z","shell.execute_reply":"2022-07-07T13:39:05.676175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_train = pd.concat([process_train,test_data])\nmerged_train = merged_train.set_index(['date','store_nbr','family'])\nmerged_train","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:05.678834Z","iopub.execute_input":"2022-07-07T13:39:05.679176Z","iopub.status.idle":"2022-07-07T13:39:18.373796Z","shell.execute_reply.started":"2022-07-07T13:39:05.679148Z","shell.execute_reply":"2022-07-07T13:39:18.372635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_train","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:18.375500Z","iopub.execute_input":"2022-07-07T13:39:18.376183Z","iopub.status.idle":"2022-07-07T13:39:18.395101Z","shell.execute_reply.started":"2022-07-07T13:39:18.376138Z","shell.execute_reply":"2022-07-07T13:39:18.394336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:18.396543Z","iopub.execute_input":"2022-07-07T13:39:18.397320Z","iopub.status.idle":"2022-07-07T13:39:18.531440Z","shell.execute_reply.started":"2022-07-07T13:39:18.397287Z","shell.execute_reply":"2022-07-07T13:39:18.530345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store_location = store_data.drop(['state','type','cluster'],axis=1)\nstore_location = store_location.set_index('store_nbr')\nstore_location = pd.get_dummies(store_location,prefix='store_loc_')\nstore_location","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:18.535857Z","iopub.execute_input":"2022-07-07T13:39:18.536760Z","iopub.status.idle":"2022-07-07T13:39:18.577816Z","shell.execute_reply.started":"2022-07-07T13:39:18.536720Z","shell.execute_reply":"2022-07-07T13:39:18.576444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = merged_train.reset_index().merge(store_location,how='outer',left_on='store_nbr',right_on=store_location.index)\ninputs","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:18.579862Z","iopub.execute_input":"2022-07-07T13:39:18.580205Z","iopub.status.idle":"2022-07-07T13:39:19.994920Z","shell.execute_reply.started":"2022-07-07T13:39:18.580175Z","shell.execute_reply":"2022-07-07T13:39:19.993854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del store_location\ndel merged_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:19.996445Z","iopub.execute_input":"2022-07-07T13:39:19.996882Z","iopub.status.idle":"2022-07-07T13:39:20.124014Z","shell.execute_reply.started":"2022-07-07T13:39:19.996840Z","shell.execute_reply":"2022-07-07T13:39:20.122854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.125409Z","iopub.execute_input":"2022-07-07T13:39:20.126538Z","iopub.status.idle":"2022-07-07T13:39:20.140093Z","shell.execute_reply.started":"2022-07-07T13:39:20.126493Z","shell.execute_reply":"2022-07-07T13:39:20.139130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales_to_scale = pd.DataFrame(index=pd.date_range(start='2013-01-01',end='2017-08-31'))\ntotal_sales_to_scale = total_sales_to_scale.merge(total_sales,how='left',left_index=True,right_index=True)\ntotal_sales_to_scale = total_sales_to_scale.rename(columns={'sales':'national_sales'})","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.141195Z","iopub.execute_input":"2022-07-07T13:39:20.143668Z","iopub.status.idle":"2022-07-07T13:39:20.150955Z","shell.execute_reply.started":"2022-07-07T13:39:20.143623Z","shell.execute_reply":"2022-07-07T13:39:20.150245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales_to_scale","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.151984Z","iopub.execute_input":"2022-07-07T13:39:20.152545Z","iopub.status.idle":"2022-07-07T13:39:20.166172Z","shell.execute_reply.started":"2022-07-07T13:39:20.152513Z","shell.execute_reply":"2022-07-07T13:39:20.165121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nmmScale = MinMaxScaler()\nmmScale.fit(total_sales_to_scale['national_sales'].to_numpy().reshape(-1,1))\n\ntotal_sales_to_scale['scaled_nat_sales'] = mmScale.transform(total_sales_to_scale['national_sales'].to_numpy().reshape(-1,1))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.168422Z","iopub.execute_input":"2022-07-07T13:39:20.169797Z","iopub.status.idle":"2022-07-07T13:39:20.239736Z","shell.execute_reply.started":"2022-07-07T13:39:20.169749Z","shell.execute_reply":"2022-07-07T13:39:20.238668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales_to_scale","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.241270Z","iopub.execute_input":"2022-07-07T13:39:20.242230Z","iopub.status.idle":"2022-07-07T13:39:20.257589Z","shell.execute_reply.started":"2022-07-07T13:39:20.242184Z","shell.execute_reply":"2022-07-07T13:39:20.256452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import  statsmodels.graphics.tsaplots\n\nax = statsmodels.graphics.tsaplots.plot_pacf(total_sales_to_scale['scaled_nat_sales'].dropna(), \n                                                 lags=range(16,31),\n                                                 title = 'PACF of national sales')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.260325Z","iopub.execute_input":"2022-07-07T13:39:20.261113Z","iopub.status.idle":"2022-07-07T13:39:20.461860Z","shell.execute_reply.started":"2022-07-07T13:39:20.261075Z","shell.execute_reply":"2022-07-07T13:39:20.460736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags= [16,17,18,19,20,21,22,23,24,27,28]\nfor lag in lags:\n    total_sales_to_scale['nat_scaled_sales_lag{}'.format(lag)] = total_sales_to_scale['scaled_nat_sales'].shift(lag)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.463122Z","iopub.execute_input":"2022-07-07T13:39:20.463450Z","iopub.status.idle":"2022-07-07T13:39:20.476781Z","shell.execute_reply.started":"2022-07-07T13:39:20.463419Z","shell.execute_reply":"2022-07-07T13:39:20.475547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales_to_scale = total_sales_to_scale.drop(['national_sales','scaled_nat_sales'],axis=1) ","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.478180Z","iopub.execute_input":"2022-07-07T13:39:20.478522Z","iopub.status.idle":"2022-07-07T13:39:20.490013Z","shell.execute_reply.started":"2022-07-07T13:39:20.478492Z","shell.execute_reply":"2022-07-07T13:39:20.489216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_sales_to_scale.reset_index().tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.491817Z","iopub.execute_input":"2022-07-07T13:39:20.492765Z","iopub.status.idle":"2022-07-07T13:39:20.520810Z","shell.execute_reply.started":"2022-07-07T13:39:20.492720Z","shell.execute_reply":"2022-07-07T13:39:20.519687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = inputs.merge(total_sales_to_scale.reset_index(),how='left',left_on='date',right_on='index')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:20.522195Z","iopub.execute_input":"2022-07-07T13:39:20.522495Z","iopub.status.idle":"2022-07-07T13:39:21.480730Z","shell.execute_reply.started":"2022-07-07T13:39:20.522470Z","shell.execute_reply":"2022-07-07T13:39:21.479554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del total_sales_to_scale","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:21.482412Z","iopub.execute_input":"2022-07-07T13:39:21.483017Z","iopub.status.idle":"2022-07-07T13:39:21.488241Z","shell.execute_reply.started":"2022-07-07T13:39:21.482972Z","shell.execute_reply":"2022-07-07T13:39:21.487064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:21.489308Z","iopub.execute_input":"2022-07-07T13:39:21.489751Z","iopub.status.idle":"2022-07-07T13:39:21.503172Z","shell.execute_reply.started":"2022-07-07T13:39:21.489723Z","shell.execute_reply":"2022-07-07T13:39:21.502019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.drop(['index'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:21.504451Z","iopub.execute_input":"2022-07-07T13:39:21.504962Z","iopub.status.idle":"2022-07-07T13:39:22.101076Z","shell.execute_reply.started":"2022-07-07T13:39:21.504934Z","shell.execute_reply":"2022-07-07T13:39:22.099647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.102677Z","iopub.execute_input":"2022-07-07T13:39:22.103010Z","iopub.status.idle":"2022-07-07T13:39:22.755122Z","shell.execute_reply.started":"2022-07-07T13:39:22.102982Z","shell.execute_reply":"2022-07-07T13:39:22.753967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [1, 2, 3, 4, 5, 6, 7, 8, 13, 14]\nfor lag in lags:\n    inputs['store_fam_sales_lag_{}'.format(lag)] = inputs['sales'].shift(lag)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.756544Z","iopub.execute_input":"2022-07-07T13:39:22.756871Z","iopub.status.idle":"2022-07-07T13:39:22.892696Z","shell.execute_reply.started":"2022-07-07T13:39:22.756842Z","shell.execute_reply":"2022-07-07T13:39:22.891273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.894168Z","iopub.execute_input":"2022-07-07T13:39:22.895012Z","iopub.status.idle":"2022-07-07T13:39:22.902234Z","shell.execute_reply.started":"2022-07-07T13:39:22.894972Z","shell.execute_reply":"2022-07-07T13:39:22.901245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.903349Z","iopub.execute_input":"2022-07-07T13:39:22.903668Z","iopub.status.idle":"2022-07-07T13:39:22.918886Z","shell.execute_reply.started":"2022-07-07T13:39:22.903641Z","shell.execute_reply":"2022-07-07T13:39:22.918139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store_nbr = range(1,55)\ndates = pd.date_range('2013-01-01','2017-08-31')\nmul_index = pd.MultiIndex.from_product([dates,store_nbr],names=['date','store_nbr'])\ndf = pd.DataFrame(index=mul_index)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.920121Z","iopub.execute_input":"2022-07-07T13:39:22.920468Z","iopub.status.idle":"2022-07-07T13:39:22.933228Z","shell.execute_reply.started":"2022-07-07T13:39:22.920437Z","shell.execute_reply":"2022-07-07T13:39:22.932440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.936347Z","iopub.execute_input":"2022-07-07T13:39:22.936759Z","iopub.status.idle":"2022-07-07T13:39:22.957513Z","shell.execute_reply.started":"2022-07-07T13:39:22.936727Z","shell.execute_reply":"2022-07-07T13:39:22.956647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.958861Z","iopub.execute_input":"2022-07-07T13:39:22.959782Z","iopub.status.idle":"2022-07-07T13:39:22.974065Z","shell.execute_reply.started":"2022-07-07T13:39:22.959747Z","shell.execute_reply":"2022-07-07T13:39:22.972832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction = df.reset_index().merge(transactions.reset_index(),\n                                        how='left',\n                                        left_on=['date','store_nbr'],\n                                        right_on=['date','store_nbr']\n                                       )","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:22.974973Z","iopub.execute_input":"2022-07-07T13:39:22.975520Z","iopub.status.idle":"2022-07-07T13:39:23.007526Z","shell.execute_reply.started":"2022-07-07T13:39:22.975487Z","shell.execute_reply":"2022-07-07T13:39:23.006612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction.fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:23.008851Z","iopub.execute_input":"2022-07-07T13:39:23.009407Z","iopub.status.idle":"2022-07-07T13:39:23.485214Z","shell.execute_reply.started":"2022-07-07T13:39:23.009363Z","shell.execute_reply":"2022-07-07T13:39:23.484012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction.loc[30020:30026]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:23.486669Z","iopub.execute_input":"2022-07-07T13:39:23.486998Z","iopub.status.idle":"2022-07-07T13:39:23.499816Z","shell.execute_reply.started":"2022-07-07T13:39:23.486970Z","shell.execute_reply":"2022-07-07T13:39:23.498614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:23.501163Z","iopub.execute_input":"2022-07-07T13:39:23.501572Z","iopub.status.idle":"2022-07-07T13:39:23.516952Z","shell.execute_reply.started":"2022-07-07T13:39:23.501541Z","shell.execute_reply":"2022-07-07T13:39:23.515467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [21,22,28]\nfor lag in lags:\n    df_transaction['trans_lag_{}'.format(lag)] = df_transaction['transactions'].shift(lag)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:23.519035Z","iopub.execute_input":"2022-07-07T13:39:23.519493Z","iopub.status.idle":"2022-07-07T13:39:23.528775Z","shell.execute_reply.started":"2022-07-07T13:39:23.519449Z","shell.execute_reply":"2022-07-07T13:39:23.527615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction = df_transaction.drop('transactions',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:23.530303Z","iopub.execute_input":"2022-07-07T13:39:23.530901Z","iopub.status.idle":"2022-07-07T13:39:23.540814Z","shell.execute_reply.started":"2022-07-07T13:39:23.530865Z","shell.execute_reply":"2022-07-07T13:39:23.539556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction = df_transaction.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:23.542412Z","iopub.execute_input":"2022-07-07T13:39:23.543439Z","iopub.status.idle":"2022-07-07T13:39:24.011884Z","shell.execute_reply.started":"2022-07-07T13:39:23.543392Z","shell.execute_reply":"2022-07-07T13:39:24.010949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transaction.loc[30030:30040]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:24.013523Z","iopub.execute_input":"2022-07-07T13:39:24.014259Z","iopub.status.idle":"2022-07-07T13:39:24.029199Z","shell.execute_reply.started":"2022-07-07T13:39:24.014213Z","shell.execute_reply":"2022-07-07T13:39:24.028122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:24.030661Z","iopub.execute_input":"2022-07-07T13:39:24.031313Z","iopub.status.idle":"2022-07-07T13:39:24.898893Z","shell.execute_reply.started":"2022-07-07T13:39:24.031273Z","shell.execute_reply":"2022-07-07T13:39:24.897824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = inputs.merge(df_transaction, how='left', left_on = ['date','store_nbr'],right_on = ['date','store_nbr'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:24.900166Z","iopub.execute_input":"2022-07-07T13:39:24.900507Z","iopub.status.idle":"2022-07-07T13:39:26.437248Z","shell.execute_reply.started":"2022-07-07T13:39:24.900478Z","shell.execute_reply":"2022-07-07T13:39:26.435956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:26.438879Z","iopub.execute_input":"2022-07-07T13:39:26.439856Z","iopub.status.idle":"2022-07-07T13:39:27.340115Z","shell.execute_reply.started":"2022-07-07T13:39:26.439815Z","shell.execute_reply":"2022-07-07T13:39:27.339026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calendar_checkpoint.reset_index().tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:39:27.341893Z","iopub.execute_input":"2022-07-07T13:39:27.342325Z","iopub.status.idle":"2022-07-07T13:39:27.362083Z","shell.execute_reply.started":"2022-07-07T13:39:27.342282Z","shell.execute_reply":"2022-07-07T13:39:27.360974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = inputs.merge(calendar_checkpoint,how='left',left_on=['date'],right_on=calendar_checkpoint.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:42:13.768250Z","iopub.execute_input":"2022-07-07T13:42:13.769337Z","iopub.status.idle":"2022-07-07T13:42:15.675810Z","shell.execute_reply.started":"2022-07-07T13:42:13.769297Z","shell.execute_reply":"2022-07-07T13:42:15.674623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:42:28.710747Z","iopub.execute_input":"2022-07-07T13:42:28.711127Z","iopub.status.idle":"2022-07-07T13:42:28.719679Z","shell.execute_reply.started":"2022-07-07T13:42:28.711097Z","shell.execute_reply":"2022-07-07T13:42:28.718573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows',None)\ninputs.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:42:47.681324Z","iopub.execute_input":"2022-07-07T13:42:47.682197Z","iopub.status.idle":"2022-07-07T13:42:48.464249Z","shell.execute_reply.started":"2022-07-07T13:42:47.682146Z","shell.execute_reply":"2022-07-07T13:42:48.462985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.reset_option('display.max_rows','display.max_columns')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:43:09.839955Z","iopub.execute_input":"2022-07-07T13:43:09.840325Z","iopub.status.idle":"2022-07-07T13:43:09.845435Z","shell.execute_reply.started":"2022-07-07T13:43:09.840294Z","shell.execute_reply":"2022-07-07T13:43:09.844122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.dropna(inplace = True)\ninputs.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:43:20.591286Z","iopub.execute_input":"2022-07-07T13:43:20.591874Z","iopub.status.idle":"2022-07-07T13:43:24.742170Z","shell.execute_reply.started":"2022-07-07T13:43:20.591827Z","shell.execute_reply":"2022-07-07T13:43:24.741098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = inputs.set_index('date')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:43:31.035980Z","iopub.execute_input":"2022-07-07T13:43:31.036626Z","iopub.status.idle":"2022-07-07T13:43:31.471260Z","shell.execute_reply.started":"2022-07-07T13:43:31.036592Z","shell.execute_reply":"2022-07-07T13:43:31.470454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:43:46.163639Z","iopub.execute_input":"2022-07-07T13:43:46.164134Z","iopub.status.idle":"2022-07-07T13:43:46.195750Z","shell.execute_reply.started":"2022-07-07T13:43:46.164091Z","shell.execute_reply":"2022-07-07T13:43:46.194343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = inputs.loc['2013-01-01':'2017-08-15', 'sales']\ny_train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:44:14.658104Z","iopub.execute_input":"2022-07-07T13:44:14.658519Z","iopub.status.idle":"2022-07-07T13:44:14.760841Z","shell.execute_reply.started":"2022-07-07T13:44:14.658483Z","shell.execute_reply":"2022-07-07T13:44:14.759884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = inputs.loc['2013-01-01':'2017-08-15'].drop(['sales','id'],axis=1)\nx_train = x_train.reset_index()\nx_train = x_train.set_index(['date','store_nbr','family'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:44:36.612178Z","iopub.execute_input":"2022-07-07T13:44:36.612967Z","iopub.status.idle":"2022-07-07T13:44:40.934755Z","shell.execute_reply.started":"2022-07-07T13:44:36.612917Z","shell.execute_reply":"2022-07-07T13:44:40.933576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:45:02.211099Z","iopub.execute_input":"2022-07-07T13:45:02.212439Z","iopub.status.idle":"2022-07-07T13:45:02.236317Z","shell.execute_reply.started":"2022-07-07T13:45:02.212369Z","shell.execute_reply":"2022-07-07T13:45:02.235163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:45:19.563146Z","iopub.execute_input":"2022-07-07T13:45:19.563579Z","iopub.status.idle":"2022-07-07T13:45:24.948469Z","shell.execute_reply.started":"2022-07-07T13:45:19.563543Z","shell.execute_reply":"2022-07-07T13:45:24.947652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test = inputs.loc['2017-08-16': ]\ntest_id = x_test['id'] #Keep for later\n\nx_test.drop(['sales','id'],axis = 1,inplace = True)\n\nx_test = x_test.reset_index()\nx_test = x_test.set_index(['date','store_nbr','family'])\nx_test","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:45:37.501103Z","iopub.execute_input":"2022-07-07T13:45:37.501947Z","iopub.status.idle":"2022-07-07T13:45:37.589758Z","shell.execute_reply.started":"2022-07-07T13:45:37.501907Z","shell.execute_reply":"2022-07-07T13:45:37.588711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:47:14.388171Z","iopub.execute_input":"2022-07-07T13:47:14.388638Z","iopub.status.idle":"2022-07-07T13:47:14.402919Z","shell.execute_reply.started":"2022-07-07T13:47:14.388599Z","shell.execute_reply":"2022-07-07T13:47:14.402056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_id","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:47:24.280082Z","iopub.execute_input":"2022-07-07T13:47:24.280522Z","iopub.status.idle":"2022-07-07T13:47:24.290699Z","shell.execute_reply.started":"2022-07-07T13:47:24.280485Z","shell.execute_reply":"2022-07-07T13:47:24.289404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.to_csv('submissiontoday.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T13:48:23.322906Z","iopub.execute_input":"2022-07-07T13:48:23.323294Z","iopub.status.idle":"2022-07-07T13:48:23.367575Z","shell.execute_reply.started":"2022-07-07T13:48:23.323244Z","shell.execute_reply":"2022-07-07T13:48:23.366256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}