{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-25T15:29:06.634696Z","iopub.status.idle":"2022-07-25T15:29:06.635259Z","shell.execute_reply.started":"2022-07-25T15:29:06.634979Z","shell.execute_reply":"2022-07-25T15:29:06.635003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport collections\nimport csv\nimport time\nimport datetime\nimport collections\n        \nfrom scipy.stats import skew,norm,zscore\nfrom scipy.signal import periodogram\n\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nfrom statsmodels.tsa.deterministic import DeterministicProcess, CalendarFourier\n\nfrom sklearn.model_selection import train_test_split, cross_val_score, TimeSeriesSplit, GridSearchCV, cross_validate\nfrom sklearn.metrics import mean_squared_error, make_scorer, mean_squared_log_error, mean_absolute_error, mean_absolute_percentage_error\n\nfrom sklearn.ensemble import RandomForestRegressor as RF\nfrom sklearn.linear_model import LinearRegression as LR\n\nsns.set_theme()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.636649Z","iopub.status.idle":"2022-07-25T15:29:06.637187Z","shell.execute_reply.started":"2022-07-25T15:29:06.636910Z","shell.execute_reply":"2022-07-25T15:29:06.636934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **探索的データ解析(*EDA*)**","metadata":{}},{"cell_type":"code","source":"# HD : holidays_events\n# OL : oil\n# SS : sample_submission\n# ST : stores\n# TR : transactions\nHD = pd.read_csv('../input/store-sales-time-series-forecasting/holidays_events.csv', parse_dates = ['date'])\nOL = pd.read_csv('../input/store-sales-time-series-forecasting/oil.csv', parse_dates = ['date'])\nSS_ans_format = pd.read_csv('../input/store-sales-time-series-forecasting/sample_submission.csv')\nST = pd.read_csv('../input/store-sales-time-series-forecasting/stores.csv')\nTR = pd.read_csv('../input/store-sales-time-series-forecasting/transactions.csv', parse_dates = ['date'])\ntest = pd.read_csv('../input/store-sales-time-series-forecasting/test.csv', parse_dates = ['date'])\ntrain = pd.read_csv('../input/store-sales-time-series-forecasting/train.csv', parse_dates = ['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.639818Z","iopub.status.idle":"2022-07-25T15:29:06.640884Z","shell.execute_reply.started":"2022-07-25T15:29:06.640564Z","shell.execute_reply":"2022-07-25T15:29:06.640592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **原油価格のデータ整形**","metadata":{}},{"cell_type":"code","source":"sns.lineplot(y=OL.dcoilwtico, x=OL.date)\nOL","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.642290Z","iopub.status.idle":"2022-07-25T15:29:06.643433Z","shell.execute_reply.started":"2022-07-25T15:29:06.643139Z","shell.execute_reply":"2022-07-25T15:29:06.643166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">  **oil価格の欠損と欠落を2013/1/1~2017/8/31まで全てに前後の平均で補完**","metadata":{}},{"cell_type":"code","source":"OL_token = pd.date_range(start='1/1/2013', end='31/8/2017', freq='D')\nOL_token = pd.DataFrame(OL_token, columns=['date'])\nOL_token\nOL_dropna = pd.merge(OL_token, OL, on = \"date\", how = \"left\")\nOL_dropna['date'] = OL_dropna['date'].astype(str)\nOL_dropna = OL_dropna.interpolate(limit_direction='both')\nOL_dropna['date'] = pd.to_datetime(OL_dropna['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.644737Z","iopub.status.idle":"2022-07-25T15:29:06.645687Z","shell.execute_reply.started":"2022-07-25T15:29:06.645395Z","shell.execute_reply":"2022-07-25T15:29:06.645423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#　train と test データの両方にoil価格を日付けでマージする\ntrain_oil = pd.merge(train, OL_dropna, on = \"date\", how = \"left\")\ntest_oil = pd.merge(test, OL_dropna, on = \"date\", how = \"left\")\nprint(train_oil.isnull().sum(), test_oil.isnull().sum(), sep=\"\\n\\n\")\ntrain_oil","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.647289Z","iopub.status.idle":"2022-07-25T15:29:06.648304Z","shell.execute_reply.started":"2022-07-25T15:29:06.648016Z","shell.execute_reply":"2022-07-25T15:29:06.648042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **店舗情報のデータ整形**","metadata":{}},{"cell_type":"code","source":"ST","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.649707Z","iopub.status.idle":"2022-07-25T15:29:06.650633Z","shell.execute_reply.started":"2022-07-25T15:29:06.650437Z","shell.execute_reply":"2022-07-25T15:29:06.650456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **店舗ごとの売り上げ推移**","metadata":{}},{"cell_type":"code","source":"# store_1 の合計売上の推移\ntrain_oil[train.store_nbr==1].groupby(by = ['date'])['sales'].sum().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.651635Z","iopub.status.idle":"2022-07-25T15:29:06.652364Z","shell.execute_reply.started":"2022-07-25T15:29:06.652134Z","shell.execute_reply":"2022-07-25T15:29:06.652156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **各店舗の推移**","metadata":{}},{"cell_type":"code","source":"n = 1\nfig, axes = plt.subplots(sharex=True, sharey=True, figsize=(18,4), tight_layout=True)\nsns.lineplot(x = 'date', \n             y = 'sales', \n             data = train_oil[train.store_nbr==(n+1)].groupby(by = ['date'])['sales'].sum().reset_index(),\n             ax = axes)\naxes.set_title(f'store_{n}')","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.653364Z","iopub.status.idle":"2022-07-25T15:29:06.653694Z","shell.execute_reply.started":"2022-07-25T15:29:06.653528Z","shell.execute_reply":"2022-07-25T15:29:06.653542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **全店舗の推移**","metadata":{}},{"cell_type":"code","source":"n = 54\nfig, axes = plt.subplots(n,1,sharex=True, sharey=True, figsize=(18, n*4), tight_layout=True)\nfor i in range(n):\n    sns.lineplot(x = 'date', \n                 y = 'sales', \n                 data = train_oil[train.store_nbr==(i+1)].groupby(by = ['date'])['sales'].sum().reset_index(),\n                 ax = axes[i])\n    axes[i].set_title(f'store_{i+1}')","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.655550Z","iopub.status.idle":"2022-07-25T15:29:06.655912Z","shell.execute_reply.started":"2022-07-25T15:29:06.655727Z","shell.execute_reply":"2022-07-25T15:29:06.655742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train と test データの両方に店舗情報を store_nbr でマージする\ntrain_oil_ST = pd.merge(train_oil, ST, on = \"store_nbr\", how = \"left\")\ntest_oil_ST = pd.merge(test_oil, ST, on = \"store_nbr\", how = \"left\")\nprint(train_oil.isnull().sum(), test_oil.isnull().sum(), sep=\"\\n\\n\")\ntrain_oil_ST","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.657240Z","iopub.status.idle":"2022-07-25T15:29:06.657567Z","shell.execute_reply.started":"2022-07-25T15:29:06.657409Z","shell.execute_reply":"2022-07-25T15:29:06.657425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **transactions の検討**","metadata":{}},{"cell_type":"code","source":"TR","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.659019Z","iopub.status.idle":"2022-07-25T15:29:06.659374Z","shell.execute_reply.started":"2022-07-25T15:29:06.659208Z","shell.execute_reply":"2022-07-25T15:29:06.659224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 各店舗の sales と transactions の推移\nn = 10\nfig, axes = plt.subplots(sharex=True, sharey=True, figsize=(18,4), tight_layout=True)\nsns.lineplot(x = 'date', \n             y = 'transactions', \n             data = TR[TR.store_nbr==(n)],\n             ax = axes)\naxes.set_title(f'transactions_{n}')\n\nfig, axes = plt.subplots(sharex=True, sharey=True, figsize=(18,4), tight_layout=True)\nsns.lineplot(x = 'date', \n             y = 'sales', \n             data = train_oil[train.store_nbr==(n+1)].groupby(by = ['date'])['sales'].sum().reset_index(),\n             ax = axes)\naxes.set_title(f'store_{n}')","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.661069Z","iopub.status.idle":"2022-07-25T15:29:06.661773Z","shell.execute_reply.started":"2022-07-25T15:29:06.661477Z","shell.execute_reply":"2022-07-25T15:29:06.661505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train と test データの両方に店舗情報を store_nbr でマージする\ntrain_oil_ST_TR = pd.merge(train_oil_ST, TR, on = [\"store_nbr\", \"date\"], how = \"left\")\ntest_oil_ST_TR = pd.merge(test_oil_ST, TR, on = [\"store_nbr\", \"date\"], how = \"left\")\nprint(train_oil.isnull().sum(), test_oil.isnull().sum(), sep=\"\\n\\n\")\ntrain_oil_ST_TR = train_oil_ST_TR.drop(columns=['id'])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.663238Z","iopub.status.idle":"2022-07-25T15:29:06.663916Z","shell.execute_reply.started":"2022-07-25T15:29:06.663608Z","shell.execute_reply":"2022-07-25T15:29:06.663634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_oil_ST_TR","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.665233Z","iopub.status.idle":"2022-07-25T15:29:06.665735Z","shell.execute_reply.started":"2022-07-25T15:29:06.665473Z","shell.execute_reply":"2022-07-25T15:29:06.665496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_oil_ST_TR.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.667125Z","iopub.status.idle":"2022-07-25T15:29:06.667665Z","shell.execute_reply.started":"2022-07-25T15:29:06.667390Z","shell.execute_reply":"2022-07-25T15:29:06.667414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train_oil_ST_TR.corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.669013Z","iopub.status.idle":"2022-07-25T15:29:06.670003Z","shell.execute_reply.started":"2022-07-25T15:29:06.669687Z","shell.execute_reply":"2022-07-25T15:29:06.669713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sales = train.groupby('date').agg({\"sales\" : \"sum\"}).reset_index()\ndf_sales['sales_ma'] = df_sales['sales'].rolling(7).mean()\ndf_view = train_oil_ST_TR.groupby('date').agg({\"transactions\" : \"sum\"}).reset_index()\ndf_view['transactions_mean'] = df_view['transactions'].rolling(7).mean()\nprint(df_view)\n\nfig = make_subplots(rows=3, cols=1,\n                    subplot_titles=[\"Sales\", \"Transactions\", \"Sales / Transactions\"],\n                    vertical_spacing=.1)\nfig.add_scatter(x=df_sales['date'], y=df_sales['sales'],\n                mode='lines', marker=dict(color='blue'),\n                name='Sales', row=1, col=1)\nfig.add_scatter(x=df_sales['date'], y=df_sales['sales_ma'],\n                mode='lines', marker=dict(color='red'),\n                name='7d moving avearge', row=1, col=1)\nfig.add_scatter(x=df_sales['date'],\n                y=df_view['transactions'],\n                mode='lines',\n                marker=dict(color='blue'),\n                name='Transactions', row=2, col=1)\nfig.add_scatter(x=df_sales['date'],\n                y=df_view['transactions_mean'],\n                mode='lines',\n                marker=dict(color='red'),\n                name='7d moving avearge', row=2, col=1)\nfig.add_scatter(x=df_sales['sales'],\n                y=df_view['transactions'],\n                mode='markers',\n                marker=dict(color='blue', size=2),\n                name='Sales/Transactions', row=3, col=1)\n\n# style\nfig.update_xaxes(title='Sales', row=3, col=1)\nfig.update_yaxes(title='Transactions', row=3, col=1)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.671667Z","iopub.status.idle":"2022-07-25T15:29:06.672218Z","shell.execute_reply.started":"2022-07-25T15:29:06.671939Z","shell.execute_reply":"2022-07-25T15:29:06.671962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_view_trans = pd.merge(df_sales, df_view, on='date', how='left')\nsns.heatmap(corr_view_trans.corr())\nprint(corr_view_trans.corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.673872Z","iopub.status.idle":"2022-07-25T15:29:06.674394Z","shell.execute_reply.started":"2022-07-25T15:29:06.674129Z","shell.execute_reply":"2022-07-25T15:29:06.674154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **店舗情報を含めると相関は下がる**","metadata":{}},{"cell_type":"markdown","source":"# **休日情報の検討**","metadata":{}},{"cell_type":"code","source":"HD.head(50)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.676047Z","iopub.status.idle":"2022-07-25T15:29:06.676546Z","shell.execute_reply.started":"2022-07-25T15:29:06.676286Z","shell.execute_reply":"2022-07-25T15:29:06.676309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ST.sample(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.679698Z","iopub.status.idle":"2022-07-25T15:29:06.680410Z","shell.execute_reply.started":"2022-07-25T15:29:06.680122Z","shell.execute_reply":"2022-07-25T15:29:06.680149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 共通要素\nset(ST['city'].unique()) & set(HD['locale_name'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.682085Z","iopub.status.idle":"2022-07-25T15:29:06.682755Z","shell.execute_reply.started":"2022-07-25T15:29:06.682471Z","shell.execute_reply":"2022-07-25T15:29:06.682497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_oil_ST_TR.rename(columns={'type': 'store_type'}, inplace=True)\ntest_oil_ST_TR.rename(columns={'type': 'store_type'}, inplace=True)\nHD_new = HD[HD[['date', 'locale_name']].duplicated()].index.values\nprint(HD_new.dtype)\ntrain_oil_ST_TR","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.684454Z","iopub.status.idle":"2022-07-25T15:29:06.685329Z","shell.execute_reply.started":"2022-07-25T15:29:06.685046Z","shell.execute_reply":"2022-07-25T15:29:06.685072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HD = HD.drop(HD_new)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.687116Z","iopub.status.idle":"2022-07-25T15:29:06.687647Z","shell.execute_reply.started":"2022-07-25T15:29:06.687385Z","shell.execute_reply":"2022-07-25T15:29:06.687409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#　休日を全てdfにマージ\ndf = train_oil_ST_TR\ndf_test = test_oil_ST_TR\n\nnat_df = HD.query(\"locale=='National'\")\nloc_df = HD.query(\"locale=='Local'\")\nreg_df = HD.query(\"locale=='Regional'\")\n\ndf = pd.merge(train_oil_ST_TR, nat_df, on=['date'], how='left')\ndf = pd.merge(df, loc_df, left_on=['date', 'city'], right_on=['date', 'locale_name'], how='left')\ndf = pd.merge(df, reg_df, left_on=['date', 'state'], right_on=['date', 'locale_name'], how='left')\n# test\ndf_test = pd.merge(test_oil_ST_TR, nat_df, on=['date'], how='left')\ndf_test = pd.merge(df_test, loc_df, left_on=['date', 'city'], right_on=['date', 'locale_name'], how='left')\ndf_test = pd.merge(df_test, reg_df, left_on=['date', 'state'], right_on=['date', 'locale_name'], how='left')\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.689049Z","iopub.status.idle":"2022-07-25T15:29:06.689548Z","shell.execute_reply.started":"2022-07-25T15:29:06.689288Z","shell.execute_reply":"2022-07-25T15:29:06.689312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.690949Z","iopub.status.idle":"2022-07-25T15:29:06.691997Z","shell.execute_reply.started":"2022-07-25T15:29:06.691817Z","shell.execute_reply":"2022-07-25T15:29:06.691837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.isnull().sum())\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.692880Z","iopub.status.idle":"2022-07-25T15:29:06.693512Z","shell.execute_reply.started":"2022-07-25T15:29:06.693289Z","shell.execute_reply":"2022-07-25T15:29:06.693316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 地名、休日名など不要なカラムを落とす\ndf = df.drop(columns=['locale_x',\n                      'locale_name_x',\n                      'description_x',\n                      'transferred_x',\n                      'locale_y',\n                      'locale_name_y',\n                      'description_y',\n                      'transferred_y',\n                      'locale',\n                      'locale_name',\n                      'description',\n                      'transferred'])\ndf_test = df_test.drop(columns=['locale_x',\n                                'locale_name_x',\n                                'description_x',\n                                'transferred_x',\n                                'locale_y',\n                                'locale_name_y',\n                                'description_y',\n                                'transferred_y',\n                                'locale',\n                                'locale_name',\n                                'description',\n                                'transferred'])\ndf.rename(columns={'type': 'type_National',\n                   'type_x': 'type_Local',\n                   'type_y': 'type_Regional'}, inplace=True)\ndf_test.rename(columns={'type': 'type_National',\n                        'type_x': 'type_Local',\n                        'type_y': 'type_Regional'}, inplace=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.695032Z","iopub.status.idle":"2022-07-25T15:29:06.695404Z","shell.execute_reply.started":"2022-07-25T15:29:06.695234Z","shell.execute_reply":"2022-07-25T15:29:06.695251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# type の　Null　は全て　Work Day　に置換\ndf = df.fillna({'type_National': 'Work Day', 'type_Local': 'Work Day', 'type_Regional': 'Work Day'})\ndf_test = df_test.fillna({'type_National': 'Work Day', 'type_Local': 'Work Day', 'type_Regional': 'Work Day'})\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.696673Z","iopub.status.idle":"2022-07-25T15:29:06.697039Z","shell.execute_reply.started":"2022-07-25T15:29:06.696867Z","shell.execute_reply":"2022-07-25T15:29:06.696884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **曜日や季節情報の取り込み**","metadata":{}},{"cell_type":"code","source":"df['year'] = df['date'].dt.year\ndf['month'] = df['date'].dt.month\ndf['week'] = df['date'].dt.isocalendar().week\ndf['quarter'] = df['date'].dt.quarter\ndf['weekday'] = df['date'].dt.dayofweek\ndf['day_name'] = df['date'].dt.day_name()\n\ndf_test['year'] = df_test['date'].dt.year\ndf_test['month'] = df_test['date'].dt.month\ndf_test['week'] = df_test['date'].dt.isocalendar().week\ndf_test['quarter'] = df_test['date'].dt.quarter\ndf_test['weekday'] = df_test['date'].dt.dayofweek\ndf_test['day_name'] = df_test['date'].dt.day_name()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.698043Z","iopub.status.idle":"2022-07-25T15:29:06.698400Z","shell.execute_reply.started":"2022-07-25T15:29:06.698229Z","shell.execute_reply":"2022-07-25T15:29:06.698244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.info())\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.700108Z","iopub.status.idle":"2022-07-25T15:29:06.700900Z","shell.execute_reply.started":"2022-07-25T15:29:06.700672Z","shell.execute_reply":"2022-07-25T15:29:06.700692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_test.info())\ndf_test","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.702243Z","iopub.status.idle":"2022-07-25T15:29:06.703008Z","shell.execute_reply.started":"2022-07-25T15:29:06.702814Z","shell.execute_reply":"2022-07-25T15:29:06.702839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(collections.Counter(df[['transactions']]))","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.704257Z","iopub.status.idle":"2022-07-25T15:29:06.704598Z","shell.execute_reply.started":"2022-07-25T15:29:06.704433Z","shell.execute_reply":"2022-07-25T15:29:06.704449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **一連の処理では振替休日？を考慮していない**(処理が煩雑すぎる)","metadata":{}},{"cell_type":"markdown","source":"# **商品の種別検討**","metadata":{}},{"cell_type":"code","source":"df_a = train.groupby(['date', 'family']).agg({\"sales\" : \"sum\"})\ndf_a = df_a.unstack(level=0).T.droplevel(level=0, axis=0).rolling(7).mean()\nfig = go.Figure()\nfor col in df_a.columns:\n    fig.add_trace(go.Scatter(x=df_a.index, y=df_a[col], name=col, mode='lines'))\nfig.update_layout(height=850,\n                  title_text=\"各品種による売り上げの7日間移動平均\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.705834Z","iopub.status.idle":"2022-07-25T15:29:06.706180Z","shell.execute_reply.started":"2022-07-25T15:29:06.706009Z","shell.execute_reply":"2022-07-25T15:29:06.706025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Modeling**","metadata":{}},{"cell_type":"code","source":"import warnings\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.metrics import make_scorer, r2_score, mean_squared_error\nfrom sklearn.linear_model import Ridge, Lasso, LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom xgboost import XGBRegressor\nfrom joblib import Parallel, delayed\nfrom sklearn.metrics import mean_squared_log_error as RMSLE","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.707854Z","iopub.status.idle":"2022-07-25T15:29:06.708194Z","shell.execute_reply.started":"2022-07-25T15:29:06.708025Z","shell.execute_reply":"2022-07-25T15:29:06.708041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_data(orig_df, X, y, end_date, test_size):\n    \n    # Splitting train and test\n    idx_train, idx_test = train_test_split(orig_df.index, test_size=test_size, shuffle=False)\n    X_train, X_test = X.loc[idx_train, :], X.loc[idx_test, :]\n    y_train, y_test = y.loc[idx_train], y.loc[idx_test]\n    \n    return X_train, y_train, X_test, y_test","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.709331Z","iopub.status.idle":"2022-07-25T15:29:06.709970Z","shell.execute_reply.started":"2022-07-25T15:29:06.709753Z","shell.execute_reply":"2022-07-25T15:29:06.709772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.info())\nprint(df.keys())","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.711125Z","iopub.status.idle":"2022-07-25T15:29:06.711745Z","shell.execute_reply.started":"2022-07-25T15:29:06.711555Z","shell.execute_reply":"2022-07-25T15:29:06.711574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train data\ny = df['sales']\n# 説明変数にdateを含めていない\nset_X_train = df[['store_nbr', 'family', 'onpromotion', 'dcoilwtico',\n       'city', 'state', 'store_type', 'cluster', 'transactions', 'type_Local',\n       'type_Regional', 'type_National', 'year', 'month', 'week', 'quarter',\n       'weekday', 'day_name']]\nset_X_train = pd.get_dummies(set_X_train)\n\n#test data\nset_X_test = df[['store_nbr', 'family', 'onpromotion', 'dcoilwtico',\n       'city', 'state', 'store_type', 'cluster', 'transactions', 'type_Local',\n       'type_Regional', 'type_National', 'year', 'month', 'week', 'quarter',\n       'weekday', 'day_name']]\nset_X_test = pd.get_dummies(set_X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.713077Z","iopub.status.idle":"2022-07-25T15:29:06.713411Z","shell.execute_reply.started":"2022-07-25T15:29:06.713248Z","shell.execute_reply":"2022-07-25T15:29:06.713263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train, X_test, y_test = train_test_split(set_X_train, y, random_state = 3)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.714482Z","iopub.status.idle":"2022-07-25T15:29:06.714826Z","shell.execute_reply.started":"2022-07-25T15:29:06.714645Z","shell.execute_reply":"2022-07-25T15:29:06.714660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **LinearRegression**","metadata":{}},{"cell_type":"code","source":"#pridict\nmodel_LR = LR()\nmodel_LR.fit(X_train, y_train)\ny_pred_train = model_LR.pridict(X_train)\ny_pred_test  = model_LR.pridict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.715872Z","iopub.status.idle":"2022-07-25T15:29:06.716199Z","shell.execute_reply.started":"2022-07-25T15:29:06.716034Z","shell.execute_reply":"2022-07-25T15:29:06.716049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#evaluation RMSLE\nRMSLE_train = RMSLE(y_train, y_pred_train)\nRMSLE_test  = RMSLE(y_test, y_pred_test)\nprint(RMSLE_train)\nprint(RMSLE_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T15:29:06.717211Z","iopub.status.idle":"2022-07-25T15:29:06.717527Z","shell.execute_reply.started":"2022-07-25T15:29:06.717369Z","shell.execute_reply":"2022-07-25T15:29:06.717384Z"},"trusted":true},"execution_count":null,"outputs":[]}]}