{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-29T07:12:07.904938Z","iopub.execute_input":"2022-07-29T07:12:07.905342Z","iopub.status.idle":"2022-07-29T07:12:07.914926Z","shell.execute_reply.started":"2022-07-29T07:12:07.905307Z","shell.execute_reply":"2022-07-29T07:12:07.913549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ライブラリのインポート","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import accuracy_score\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:07.917278Z","iopub.execute_input":"2022-07-29T07:12:07.918132Z","iopub.status.idle":"2022-07-29T07:12:09.822130Z","shell.execute_reply.started":"2022-07-29T07:12:07.918083Z","shell.execute_reply":"2022-07-29T07:12:09.820931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データ読み込み","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/store-sales-time-series-forecasting/train.csv', index_col = 'id')\noil = pd.read_csv('../input/store-sales-time-series-forecasting/oil.csv')\nholidays = pd.read_csv('../input/store-sales-time-series-forecasting/holidays_events.csv')\ntest = pd.read_csv('../input/store-sales-time-series-forecasting/test.csv', index_col = 'id')\nstores = pd.read_csv('/kaggle/input/store-sales-time-series-forecasting/stores.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:09.823482Z","iopub.execute_input":"2022-07-29T07:12:09.823811Z","iopub.status.idle":"2022-07-29T07:12:15.207744Z","shell.execute_reply.started":"2022-07-29T07:12:09.823782Z","shell.execute_reply":"2022-07-29T07:12:15.206642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['date'] = pd.to_datetime(train['date'])\noil['date'] = pd.to_datetime(oil['date'])\nholidays['date'] = pd.to_datetime(holidays['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:15.210254Z","iopub.execute_input":"2022-07-29T07:12:15.211347Z","iopub.status.idle":"2022-07-29T07:12:15.656805Z","shell.execute_reply.started":"2022-07-29T07:12:15.211295Z","shell.execute_reply":"2022-07-29T07:12:15.655742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<u>EDA</u>**","metadata":{}},{"cell_type":"markdown","source":"## 売上推移（2013/1/1~2017/8/15）","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,8))\ntrain.groupby(['date'])['sales'].sum().plot(color='dodgerblue',marker='o',markerfacecolor='dodgerblue')\nplt.title(f'Slaes(2013-01-01 ~ 2017-08-15)',fontsize=30)\nplt.xlabel('date',fontsize=30)\nplt.ylabel('sale',fontsize=30)\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:15.657942Z","iopub.execute_input":"2022-07-29T07:12:15.658268Z","iopub.status.idle":"2022-07-29T07:12:16.147257Z","shell.execute_reply.started":"2022-07-29T07:12:15.658239Z","shell.execute_reply":"2022-07-29T07:12:16.145628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 石油と売上の関係","metadata":{}},{"cell_type":"markdown","source":"### 欠損値を確認","metadata":{}},{"cell_type":"code","source":"oil.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:16.148661Z","iopub.execute_input":"2022-07-29T07:12:16.149089Z","iopub.status.idle":"2022-07-29T07:12:16.161693Z","shell.execute_reply.started":"2022-07-29T07:12:16.149049Z","shell.execute_reply":"2022-07-29T07:12:16.160358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 欠損値を処理","metadata":{}},{"cell_type":"code","source":"oil_tmp = oil.copy()\noil['date'] = pd.to_datetime(oil['date'])\n# Resample\noil = oil.set_index(\"date\").dcoilwtico.resample(\"D\").sum().reset_index()\n# Interpolate\noil[\"dcoilwtico\"] = np.where(oil[\"dcoilwtico\"] == 0, np.nan, oil[\"dcoilwtico\"])\noil[\"dcoilwtico_interpolated\"] =oil.dcoilwtico.interpolate()\noil.drop(columns = ['dcoilwtico'], inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:16.163507Z","iopub.execute_input":"2022-07-29T07:12:16.164280Z","iopub.status.idle":"2022-07-29T07:12:16.189535Z","shell.execute_reply.started":"2022-07-29T07:12:16.164229Z","shell.execute_reply":"2022-07-29T07:12:16.188435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(oil_tmp, oil_tmp['date'], oil_tmp['dcoilwtico'], title='Oil Prices')\nfig.show()\nfig = px.line(oil, oil['date'], oil['dcoilwtico_interpolated'], title='Oil Prices with Interpolation')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:16.191277Z","iopub.execute_input":"2022-07-29T07:12:16.191710Z","iopub.status.idle":"2022-07-29T07:12:17.509483Z","shell.execute_reply.started":"2022-07-29T07:12:16.191667Z","shell.execute_reply":"2022-07-29T07:12:17.508214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 石油価格と売上の関係グラフ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(30,8))\nplt.plot(oil['date'],oil['dcoilwtico_interpolated']*10000, color='orangered',linewidth='2.5', linestyle='-',label='dcoilwtico_interpolated')\ntrain.groupby(['date'])['sales'].sum().plot(color='dodgerblue',alpha=0.8)\nplt.title(f'Oil Price & Sales(2013-01-01 ~ 2017-08-15)',fontsize=30)\nplt.xlabel('date',fontsize=30)\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:17.514395Z","iopub.execute_input":"2022-07-29T07:12:17.515281Z","iopub.status.idle":"2022-07-29T07:12:17.952119Z","shell.execute_reply.started":"2022-07-29T07:12:17.515239Z","shell.execute_reply":"2022-07-29T07:12:17.950948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 祝日・イベントと売上の関係","metadata":{}},{"cell_type":"code","source":"holidays[('2013-01-01'<= holidays['date']) & (holidays['date'] <='2017-08-31')].groupby(['date'])['locale'].count()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:17.953423Z","iopub.execute_input":"2022-07-29T07:12:17.953738Z","iopub.status.idle":"2022-07-29T07:12:17.967292Z","shell.execute_reply.started":"2022-07-29T07:12:17.953709Z","shell.execute_reply":"2022-07-29T07:12:17.966224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,8))\ntrain.groupby(['date'])['sales'].sum().plot(color='dodgerblue',alpha=0.8)\nholidays[('2013-01-01'<= holidays['date']) & (holidays['date'] <='2017-08-15')].groupby(['date'])['locale'].count().plot(secondary_y=True, color='red',alpha=0.9, linestyle='-')\nplt.ylim([1,1.1])\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:17.969102Z","iopub.execute_input":"2022-07-29T07:12:17.969470Z","iopub.status.idle":"2022-07-29T07:12:18.477207Z","shell.execute_reply.started":"2022-07-29T07:12:17.969425Z","shell.execute_reply":"2022-07-29T07:12:18.476132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## family毎の関係","metadata":{}},{"cell_type":"code","source":"family_df = train.groupby(['date','family'])['sales'].sum().reset_index()\nplt.figure(figsize=(30,50))\ni = 0\nfor family_name in family_df['family'].unique():\n    plt.subplot(11,3,i+1)\n    family_df1 = family_df[family_df['family']==family_name]\n    plt.plot(family_df1['date'],family_df1['sales'],label=family_name,color='dodgerblue')\n    plt.title(f'Family:{family_name} sales')\n    plt.legend()\n    i += 1","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:18.478525Z","iopub.execute_input":"2022-07-29T07:12:18.478866Z","iopub.status.idle":"2022-07-29T07:12:24.694388Z","shell.execute_reply.started":"2022-07-29T07:12:18.478836Z","shell.execute_reply":"2022-07-29T07:12:24.693189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 店舗毎の売上","metadata":{}},{"cell_type":"code","source":"store_df = pd.merge(train, stores, how='left', left_on='store_nbr', right_on='store_nbr')\nplt.figure(figsize=(30,81))\nfor i in store_df['store_nbr'].unique():\n    plt.subplot(18,3,i)\n    store_df1= store_df[store_df['store_nbr']==i].groupby(['date','store_nbr'])['sales'].sum().reset_index()\n    plt.plot(store_df1['date'],store_df1['sales'],label=family_name,color='dodgerblue')\n    plt.title(f'Store:{i} sales')\n    plt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:24.695832Z","iopub.execute_input":"2022-07-29T07:12:24.696193Z","iopub.status.idle":"2022-07-29T07:12:35.498692Z","shell.execute_reply.started":"2022-07-29T07:12:24.696160Z","shell.execute_reply":"2022-07-29T07:12:35.497583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 予測表示用","metadata":{}},{"cell_type":"markdown","source":"# **<u>Predict</u>**","metadata":{}},{"cell_type":"markdown","source":"## 再度データを読み込み","metadata":{}},{"cell_type":"code","source":"oil = pd.read_csv('../input/store-sales-time-series-forecasting/oil.csv')\nsubmission = pd.read_csv('../input/store-sales-time-series-forecasting/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:35.500212Z","iopub.execute_input":"2022-07-29T07:12:35.500578Z","iopub.status.idle":"2022-07-29T07:12:35.520706Z","shell.execute_reply.started":"2022-07-29T07:12:35.500546Z","shell.execute_reply":"2022-07-29T07:12:35.519448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trainデータ・Testデータ  ","metadata":{}},{"cell_type":"markdown","source":"## trainデータから目的変数である\"sales\"を分割  ","metadata":{}},{"cell_type":"code","source":"train_plot = train.copy() #最後の予測結果表示用\ntrain_y = train['sales']\ntrain.drop(columns = ['sales'] , inplace = True , axis = 1)\n\ntrain_x = train\ntest_x = test\n\ntrain_x['date'] = pd.to_datetime(train_x['date'])\ntrain_plot['date'] = pd.to_datetime(train_plot['date'])\ntest_x['date'] = pd.to_datetime(test_x['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:35.522055Z","iopub.execute_input":"2022-07-29T07:12:35.522385Z","iopub.status.idle":"2022-07-29T07:12:36.130112Z","shell.execute_reply.started":"2022-07-29T07:12:35.522355Z","shell.execute_reply":"2022-07-29T07:12:36.129045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train・testデータの\"family\"と\"sotre_nbr\"変数をダミー化","metadata":{}},{"cell_type":"code","source":"train_x = pd.get_dummies(train_x,columns=['store_nbr','family'])\ntest_x = pd.get_dummies(test_x,columns=['store_nbr','family'])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:36.131421Z","iopub.execute_input":"2022-07-29T07:12:36.131748Z","iopub.status.idle":"2022-07-29T07:12:38.195696Z","shell.execute_reply.started":"2022-07-29T07:12:36.131718Z","shell.execute_reply":"2022-07-29T07:12:38.194673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:38.197557Z","iopub.execute_input":"2022-07-29T07:12:38.198427Z","iopub.status.idle":"2022-07-29T07:12:38.491170Z","shell.execute_reply.started":"2022-07-29T07:12:38.198375Z","shell.execute_reply":"2022-07-29T07:12:38.489966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:38.492710Z","iopub.execute_input":"2022-07-29T07:12:38.493963Z","iopub.status.idle":"2022-07-29T07:12:38.521905Z","shell.execute_reply.started":"2022-07-29T07:12:38.493912Z","shell.execute_reply":"2022-07-29T07:12:38.521001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Oilデータ","metadata":{}},{"cell_type":"code","source":"oil","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:38.523345Z","iopub.execute_input":"2022-07-29T07:12:38.523787Z","iopub.status.idle":"2022-07-29T07:12:38.538430Z","shell.execute_reply.started":"2022-07-29T07:12:38.523755Z","shell.execute_reply":"2022-07-29T07:12:38.537089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 欠損値の処理","metadata":{}},{"cell_type":"code","source":"oil['date'] = pd.to_datetime(oil['date'])\n# Resample\noil = oil.set_index(\"date\").dcoilwtico.resample(\"D\").sum().reset_index()\n# Interpolate\noil[\"dcoilwtico\"] = np.where(oil[\"dcoilwtico\"] == 0, np.nan, oil[\"dcoilwtico\"])\noil[\"dcoilwtico_interpolated\"] =oil.dcoilwtico.interpolate()\noil.drop(columns = ['dcoilwtico'], inplace = True , axis = 1)\noil_plot = oil.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:38.540279Z","iopub.execute_input":"2022-07-29T07:12:38.541003Z","iopub.status.idle":"2022-07-29T07:12:38.558052Z","shell.execute_reply.started":"2022-07-29T07:12:38.540940Z","shell.execute_reply":"2022-07-29T07:12:38.557211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## oilデータを分割","metadata":{}},{"cell_type":"code","source":"oil_train = oil[oil['date'] <= datetime(2017,8,15)]\noil_test = oil[oil['date'] >= datetime(2017,8,16)]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:38.559284Z","iopub.execute_input":"2022-07-29T07:12:38.560514Z","iopub.status.idle":"2022-07-29T07:12:38.570073Z","shell.execute_reply.started":"2022-07-29T07:12:38.560469Z","shell.execute_reply":"2022-07-29T07:12:38.568911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## oilデータを説明変数に追加","metadata":{}},{"cell_type":"code","source":"train_x = pd.merge(train_x,oil_train,how='inner',on='date')\ntest_x = pd.merge(test_x,oil_test,how='inner',on='date')\ntrain_x.interpolate(method='bfill',inplace=True)\ntest_x.interpolate(method='bfill',inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:38.571558Z","iopub.execute_input":"2022-07-29T07:12:38.572603Z","iopub.status.idle":"2022-07-29T07:12:40.575456Z","shell.execute_reply.started":"2022-07-29T07:12:38.572553Z","shell.execute_reply":"2022-07-29T07:12:40.574287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# holidays_eventsデータ","metadata":{}},{"cell_type":"markdown","source":"## 前処理","metadata":{}},{"cell_type":"code","source":"holidays['date'] = pd.to_datetime(holidays['date'])\nholidays = holidays[holidays['date'] >= datetime(2013,1,1)]\nholidays_plot = holidays.copy()\nholidays.drop(columns = ['locale_name','description','transferred'], inplace = True , axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:40.576886Z","iopub.execute_input":"2022-07-29T07:12:40.577259Z","iopub.status.idle":"2022-07-29T07:12:40.587802Z","shell.execute_reply.started":"2022-07-29T07:12:40.577224Z","shell.execute_reply":"2022-07-29T07:12:40.586516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_df = pd.DataFrame()\ndate_df['date'] = oil['date']\ndate_df['type'] = 'NaN'\ndate_df['locale'] = 'NaN'","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:40.596241Z","iopub.execute_input":"2022-07-29T07:12:40.597135Z","iopub.status.idle":"2022-07-29T07:12:40.605101Z","shell.execute_reply.started":"2022-07-29T07:12:40.597092Z","shell.execute_reply":"2022-07-29T07:12:40.604051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(date_df)):\n    for j in range(len(holidays)):\n        if date_df['date'].iloc[i] == holidays['date'].iloc[j]:\n            date_df['type'].iloc[i] = holidays['type'].iloc[j]\n            date_df['locale'].iloc[i] = holidays['locale'].iloc[j]\n            break","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:40.607341Z","iopub.execute_input":"2022-07-29T07:12:40.607949Z","iopub.status.idle":"2022-07-29T07:12:56.388247Z","shell.execute_reply.started":"2022-07-29T07:12:40.607892Z","shell.execute_reply":"2022-07-29T07:12:56.387049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holidays = date_df","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:56.389797Z","iopub.execute_input":"2022-07-29T07:12:56.390193Z","iopub.status.idle":"2022-07-29T07:12:56.396508Z","shell.execute_reply.started":"2022-07-29T07:12:56.390159Z","shell.execute_reply":"2022-07-29T07:12:56.395001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holidays","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:56.398087Z","iopub.execute_input":"2022-07-29T07:12:56.398592Z","iopub.status.idle":"2022-07-29T07:12:56.418991Z","shell.execute_reply.started":"2022-07-29T07:12:56.398556Z","shell.execute_reply":"2022-07-29T07:12:56.417783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ダミー化","metadata":{}},{"cell_type":"code","source":"holidays = pd.get_dummies(holidays,columns=['type','locale'])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:56.420355Z","iopub.execute_input":"2022-07-29T07:12:56.420701Z","iopub.status.idle":"2022-07-29T07:12:56.432082Z","shell.execute_reply.started":"2022-07-29T07:12:56.420663Z","shell.execute_reply":"2022-07-29T07:12:56.430831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train用とtest用に分割","metadata":{}},{"cell_type":"code","source":"holidays_train = holidays[(holidays['date'] <= datetime(2017,8,15))]\nholidays_test = holidays[(holidays['date'] >= datetime(2017,8,16))]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:56.433604Z","iopub.execute_input":"2022-07-29T07:12:56.434088Z","iopub.status.idle":"2022-07-29T07:12:56.444487Z","shell.execute_reply.started":"2022-07-29T07:12:56.434043Z","shell.execute_reply":"2022-07-29T07:12:56.443490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## holidaysデータをtrainデータ・testデータと結合","metadata":{}},{"cell_type":"code","source":"train_x = pd.merge(train_x,holidays_train,how='inner',on='date')\ntest_x = pd.merge(test_x,holidays_test,how='inner',on='date')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:56.445949Z","iopub.execute_input":"2022-07-29T07:12:56.446353Z","iopub.status.idle":"2022-07-29T07:12:58.688875Z","shell.execute_reply.started":"2022-07-29T07:12:56.446318Z","shell.execute_reply":"2022-07-29T07:12:58.687617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 日付を'day','month','year'に分割","metadata":{}},{"cell_type":"code","source":"train_x['day']  = pd.to_datetime(train_x['date']).dt.day\ntrain_x['month']  = pd.to_datetime(train_x['date']).dt.month\ntrain_x['year']  = pd.to_datetime(train_x['date']).dt.year\ntest_x['day']  = pd.to_datetime(test_x['date']).dt.day\ntest_x['month']  = pd.to_datetime(test_x['date']).dt.month\ntest_x['year']  = pd.to_datetime(test_x['date']).dt.year\ntrain_x.drop('date',axis=1,inplace=True)\ntest_x.drop('date',axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:12:58.690304Z","iopub.execute_input":"2022-07-29T07:12:58.690668Z","iopub.status.idle":"2022-07-29T07:13:00.615159Z","shell.execute_reply.started":"2022-07-29T07:12:58.690636Z","shell.execute_reply":"2022-07-29T07:13:00.613831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### onpromotionを標準化","metadata":{}},{"cell_type":"code","source":"# train\ntrain_scaler = StandardScaler()\ntrain_tmp = train_x.copy()\ntrain_scaler.fit(train_tmp[['onpromotion']])\ntrain_ss = pd.DataFrame(train_scaler.transform(train_tmp[['onpromotion']]))\ntrain_x['onpromotion'] = train_ss","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:00.616426Z","iopub.execute_input":"2022-07-29T07:13:00.616749Z","iopub.status.idle":"2022-07-29T07:13:00.864693Z","shell.execute_reply.started":"2022-07-29T07:13:00.616720Z","shell.execute_reply":"2022-07-29T07:13:00.863536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test\ntest_scaler = StandardScaler()\ntest_tmp = test_x.copy()\ntest_scaler.fit(test_tmp[['onpromotion']])\ntest_ss = pd.DataFrame(test_scaler.transform(test_tmp[['onpromotion']]))\ntest_x['onpromotion'] = test_ss","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:00.866533Z","iopub.execute_input":"2022-07-29T07:13:00.866955Z","iopub.status.idle":"2022-07-29T07:13:00.880681Z","shell.execute_reply.started":"2022-07-29T07:13:00.866919Z","shell.execute_reply":"2022-07-29T07:13:00.879549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:00.883924Z","iopub.execute_input":"2022-07-29T07:13:00.884826Z","iopub.status.idle":"2022-07-29T07:13:00.912250Z","shell.execute_reply.started":"2022-07-29T07:13:00.884782Z","shell.execute_reply":"2022-07-29T07:13:00.911053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデル学習","metadata":{}},{"cell_type":"markdown","source":"## 線形回帰\n目的変数と説明変数の線形関係を表すモデル．   \n重回帰  \n$$ y = \\omega_{0} + \\omega_{1}x_{1} + \\cdots + \\omega_{n}x_{n} = \\omega_{0} + \\sum_{i=1}^{n} \\omega_{i}x_{i} $$  \n---\n目的変数(y)は，2017年8月16日～2017年8月31日の売上(sales)  \n\n説明変数(x)は，  \n* date\n* onpromotion\n* store_nbr\n* family\n* dcoilwtico_interpolated(oilデータ)\n* type(hgolidaysデータ)\n* locale(holidaysデータ) \n\nを使用する．  \n\n---\n※モデルの学習では，2013年1月１日～2017年8月15日のデータ(目的変数と説明変数)を使用して，重み(ω)を決めています．","metadata":{}},{"cell_type":"code","source":"model = LinearRegression()\nmodel.fit(train_x , train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:00.913832Z","iopub.execute_input":"2022-07-29T07:13:00.914509Z","iopub.status.idle":"2022-07-29T07:13:24.063933Z","shell.execute_reply.started":"2022-07-29T07:13:00.914464Z","shell.execute_reply":"2022-07-29T07:13:24.062581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.coef_","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.070511Z","iopub.execute_input":"2022-07-29T07:13:24.074407Z","iopub.status.idle":"2022-07-29T07:13:24.093466Z","shell.execute_reply.started":"2022-07-29T07:13:24.074333Z","shell.execute_reply":"2022-07-29T07:13:24.091876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.intercept_","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.100645Z","iopub.execute_input":"2022-07-29T07:13:24.104663Z","iopub.status.idle":"2022-07-29T07:13:24.120961Z","shell.execute_reply.started":"2022-07-29T07:13:24.104592Z","shell.execute_reply":"2022-07-29T07:13:24.119340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習済みモデルを用いて売上予測","metadata":{}},{"cell_type":"code","source":"pred = model.predict(test_x)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.125322Z","iopub.execute_input":"2022-07-29T07:13:24.126467Z","iopub.status.idle":"2022-07-29T07:13:24.158554Z","shell.execute_reply.started":"2022-07-29T07:13:24.126407Z","shell.execute_reply":"2022-07-29T07:13:24.157029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.160473Z","iopub.execute_input":"2022-07-29T07:13:24.160994Z","iopub.status.idle":"2022-07-29T07:13:24.178350Z","shell.execute_reply.started":"2022-07-29T07:13:24.160932Z","shell.execute_reply":"2022-07-29T07:13:24.176909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x['sales'] = pred\ntest['sales'] = pred","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.180550Z","iopub.execute_input":"2022-07-29T07:13:24.181453Z","iopub.status.idle":"2022-07-29T07:13:24.193151Z","shell.execute_reply.started":"2022-07-29T07:13:24.181402Z","shell.execute_reply":"2022-07-29T07:13:24.191895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_x","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.199860Z","iopub.execute_input":"2022-07-29T07:13:24.203441Z","iopub.status.idle":"2022-07-29T07:13:24.255105Z","shell.execute_reply.started":"2022-07-29T07:13:24.203366Z","shell.execute_reply":"2022-07-29T07:13:24.253808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_2017_08 = oil_plot[(oil_plot['date']>='2017-08-16') & (oil_plot['date']<='2017-08-31')]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.261274Z","iopub.execute_input":"2022-07-29T07:13:24.264505Z","iopub.status.idle":"2022-07-29T07:13:24.275450Z","shell.execute_reply.started":"2022-07-29T07:13:24.264440Z","shell.execute_reply":"2022-07-29T07:13:24.274198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_2017_08","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.277102Z","iopub.execute_input":"2022-07-29T07:13:24.278027Z","iopub.status.idle":"2022-07-29T07:13:24.291148Z","shell.execute_reply.started":"2022-07-29T07:13:24.277959Z","shell.execute_reply":"2022-07-29T07:13:24.290050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.plot(oil_2017_08['date'],oil_2017_08['dcoilwtico_interpolated']*10000, color='red',linewidth='2.5', linestyle='-',label='dcoilwtico_interpolated')\ntest.groupby(['date'])['sales'].sum().plot(color='green',marker='o',markerfacecolor='green')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.293955Z","iopub.execute_input":"2022-07-29T07:13:24.294370Z","iopub.status.idle":"2022-07-29T07:13:24.642441Z","shell.execute_reply.started":"2022-07-29T07:13:24.294335Z","shell.execute_reply":"2022-07-29T07:13:24.641302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,8))\nplt.plot(oil['date'],oil['dcoilwtico_interpolated']*10000, color='orange',linewidth='2.5', linestyle='-',label='dcoilwtico_interpolated')\ntrain_plot.groupby(['date'])['sales'].sum().plot(color='dodgerblue')\ntest.groupby(['date'])['sales'].sum().plot(color='red')\n#plt.axvline(x='2017-08-15',color='black',linewidth='2.5')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:24.644108Z","iopub.execute_input":"2022-07-29T07:13:24.644807Z","iopub.status.idle":"2022-07-29T07:13:25.086553Z","shell.execute_reply.started":"2022-07-29T07:13:24.644761Z","shell.execute_reply":"2022-07-29T07:13:25.085262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2017_16_31 = train_plot[train_plot['date']>='2017-08-16']\ntrain_2017_16_31 = train_2017_16_31.append(test)\ntrain_2016_16_31 = train_plot[(train_plot['date']>='2016-08-16') & (train_plot['date']<='2016-08-31')]\ntrain_2015_16_31 = train_plot[(train_plot['date']>='2015-08-16') & (train_plot['date']<='2015-08-31')]\ntrain_2014_16_31 = train_plot[(train_plot['date']>='2014-08-16') & (train_plot['date']<='2014-08-31')]\ntrain_2013_16_31 = train_plot[(train_plot['date']>='2013-08-16') & (train_plot['date']<='2013-08-31')]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:25.088132Z","iopub.execute_input":"2022-07-29T07:13:25.088578Z","iopub.status.idle":"2022-07-29T07:13:25.203745Z","shell.execute_reply.started":"2022-07-29T07:13:25.088534Z","shell.execute_reply":"2022-07-29T07:13:25.202773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2017 = train_plot[train_plot['date']>='2017-01-01']\ntrain_2017 = train_2017.append(test)\ntrain_2016 = train_plot[(train_plot['date']>='2016-01-01') & (train_plot['date']<='2016-12-31')]\ntrain_2015 = train_plot[(train_plot['date']>='2015-01-01') & (train_plot['date']<='2015-12-31')]\ntrain_2014 = train_plot[(train_plot['date']>='2014-01-01') & (train_plot['date']<='2014-12-31')]\ntrain_2013 = train_plot[(train_plot['date']>='2013-01-01') & (train_plot['date']<='2013-12-31')]\nall_sales = train_plot.append(test)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:25.204877Z","iopub.execute_input":"2022-07-29T07:13:25.205670Z","iopub.status.idle":"2022-07-29T07:13:25.567729Z","shell.execute_reply.started":"2022-07-29T07:13:25.205636Z","shell.execute_reply":"2022-07-29T07:13:25.566590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,8))\nall_sales.groupby(['date'])['sales'].sum().plot(color='grey')\ntrain_2017_16_31.groupby(['date'])['sales'].sum().plot(color='blue')\ntrain_2016_16_31.groupby(['date'])['sales'].sum().plot(color='blue')\ntrain_2015_16_31.groupby(['date'])['sales'].sum().plot(color='blue')\ntrain_2014_16_31.groupby(['date'])['sales'].sum().plot(color='blue')\ntrain_2013_16_31.groupby(['date'])['sales'].sum().plot(color='blue')\nholidays_plot[('2013-01-01'<= holidays_plot['date']) & (holidays_plot['date'] <='2017-08-15')].groupby(['date'])['locale'].count().plot(secondary_y=True, color='red',alpha=0.9, linestyle='-')\nplt.ylim([1,1.1])\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:25.569151Z","iopub.execute_input":"2022-07-29T07:13:25.569498Z","iopub.status.idle":"2022-07-29T07:13:26.154367Z","shell.execute_reply.started":"2022-07-29T07:13:25.569465Z","shell.execute_reply":"2022-07-29T07:13:26.152969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = [train_2013,train_2014,train_2015,train_2016,train_2017]\ndate_list = [\n    ['2013-01-01','2013-04-01','2013-07-01','2013-08-16','2013-08-31','2013-10-01','2013-12-01'],\n    ['2014-01-01','2014-04-01','2014-07-01','2014-08-16','2014-08-31','2014-10-01','2014-12-01'],\n    ['2015-01-01','2015-04-01','2015-07-01','2015-08-16','2015-08-31','2015-10-01','2015-12-01'],\n    ['2016-01-01','2016-04-01','2016-07-01','2016-08-16','2016-08-31','2016-10-01','2016-12-01'],\n    ['2017-01-01','2017-04-01','2017-07-01','2017-08-16','2017-08-31']\n]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:26.156091Z","iopub.execute_input":"2022-07-29T07:13:26.156642Z","iopub.status.idle":"2022-07-29T07:13:26.163514Z","shell.execute_reply.started":"2022-07-29T07:13:26.156601Z","shell.execute_reply":"2022-07-29T07:13:26.162401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,8))\ntrain_2017.groupby(['date'])['sales'].sum().plot(color='dodgerblue',marker='o',markerfacecolor='dodgerblue')\nplt.xticks(['2017-01-01','2017-04-01','2017-07-01','2017-08-16','2017-08-31'])\nplt.axvline(x='2017-08-16',color='red',linewidth='2.5')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:26.164996Z","iopub.execute_input":"2022-07-29T07:13:26.165437Z","iopub.status.idle":"2022-07-29T07:13:26.558964Z","shell.execute_reply.started":"2022-07-29T07:13:26.165394Z","shell.execute_reply":"2022-07-29T07:13:26.558187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,40))\ncnt = 1\nfor df in train_list:\n    plt.subplot(5,1,cnt)\n    plt.subplots_adjust(hspace=1)\n    df.groupby(['date'])['sales'].sum().plot(color='dodgerblue',marker='o',markerfacecolor='dodgerblue')\n    plt.xticks(date_list[cnt-1])\n    plt.axvline(x=date_list[cnt-1][3],color='red',linewidth='2.5')\n    plt.axvline(x=date_list[cnt-1][4],color='red',linewidth='2.5')\n    plt.title(f'{2013+cnt-1} Year\\'s sales')\n    plt.legend()\n    cnt += 1","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:26.560182Z","iopub.execute_input":"2022-07-29T07:13:26.560656Z","iopub.status.idle":"2022-07-29T07:13:28.237714Z","shell.execute_reply.started":"2022-07-29T07:13:26.560625Z","shell.execute_reply":"2022-07-29T07:13:28.236262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 提出csv作成","metadata":{}},{"cell_type":"code","source":"submission['sales']= pred\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:28.239127Z","iopub.execute_input":"2022-07-29T07:13:28.239536Z","iopub.status.idle":"2022-07-29T07:13:28.311863Z","shell.execute_reply.started":"2022-07-29T07:13:28.239500Z","shell.execute_reply":"2022-07-29T07:13:28.311037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:13:28.313100Z","iopub.execute_input":"2022-07-29T07:13:28.313626Z","iopub.status.idle":"2022-07-29T07:13:28.326179Z","shell.execute_reply.started":"2022-07-29T07:13:28.313592Z","shell.execute_reply":"2022-07-29T07:13:28.325052Z"},"trusted":true},"execution_count":null,"outputs":[]}]}