{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression, Lasso, Ridge","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:41:25.872851Z","iopub.execute_input":"2022-07-22T06:41:25.873268Z","iopub.status.idle":"2022-07-22T06:41:26.538873Z","shell.execute_reply.started":"2022-07-22T06:41:25.873232Z","shell.execute_reply":"2022-07-22T06:41:26.537401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/store-sales-time-series-forecasting/train.csv\")\ntest = pd.read_csv(\"../input/store-sales-time-series-forecasting/test.csv\")\noil = pd.read_csv(\"../input/store-sales-time-series-forecasting/oil.csv\")\nholidays = pd.read_csv(\"../input/store-sales-time-series-forecasting/holidays_events.csv\")\ntransactions = pd.read_csv(\"../input/store-sales-time-series-forecasting/transactions.csv\")\nstores = pd.read_csv(\"../input/store-sales-time-series-forecasting/stores.csv\")\nsubmission = pd.read_csv(\"../input/store-sales-time-series-forecasting/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:41:28.424667Z","iopub.execute_input":"2022-07-22T06:41:28.425098Z","iopub.status.idle":"2022-07-22T06:41:31.697798Z","shell.execute_reply.started":"2022-07-22T06:41:28.425064Z","shell.execute_reply":"2022-07-22T06:41:31.696498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dateカラムが現状文字列データ(オブジェクト)なので，timestamp型に変更\ntrain['date'] = pd.to_datetime(train['date'])\ntest['date'] = pd.to_datetime(test['date'])\noil['date'] = pd.to_datetime(oil['date'])\nholidays['date'] = pd.to_datetime(holidays['date'])\ntransactions['date'] = pd.to_datetime(transactions['date'])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:41:34.936556Z","iopub.execute_input":"2022-07-22T06:41:34.936989Z","iopub.status.idle":"2022-07-22T06:41:35.499067Z","shell.execute_reply.started":"2022-07-22T06:41:34.936954Z","shell.execute_reply":"2022-07-22T06:41:35.498052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データ可視化","metadata":{}},{"cell_type":"code","source":"# oilグラフ\noil.plot(x='date', y='dcoilwtico', figsize=(30,10))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:41:38.480613Z","iopub.execute_input":"2022-07-22T06:41:38.481594Z","iopub.status.idle":"2022-07-22T06:41:38.833368Z","shell.execute_reply.started":"2022-07-22T06:41:38.481552Z","shell.execute_reply":"2022-07-22T06:41:38.831982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# holidaysグラフ\n# demo = oil.copy().drop('dcoilwtico', axis=1)\n# demo['one-hot'] = 0\n# demo.loc[demo['date'].isin(holidays['date']), 'one-hot'] = 1\n# demo.set_index('date', drop=True, inplace=True)\n# demo.plot(kind='line', style='-o', figsize=(30,10))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# familyごとの売り上げ\nfamily_gp = train.groupby(['family', 'date'])['sales'].sum().reset_index().groupby('family')\nfor family, data in family_gp:\n    print(family)\n    plt.figure(figsize=(30,10))\n    plt.plot(data.date, data.sales)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:44:39.280178Z","iopub.execute_input":"2022-07-22T06:44:39.280580Z","iopub.status.idle":"2022-07-22T06:44:50.363696Z","shell.execute_reply.started":"2022-07-22T06:44:39.280550Z","shell.execute_reply":"2022-07-22T06:44:50.362895Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# familyごとの売り上げ\nstore_nbr_gp = train.groupby(['store_nbr', 'date'])['sales'].sum().reset_index().groupby('store_nbr')\nfor store_nbr, data in store_nbr_gp:\n    print(store_nbr)\n    plt.figure(figsize=(30,10))\n    plt.plot(data.date, data.sales)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:46:10.680237Z","iopub.execute_input":"2022-07-22T06:46:10.680845Z","iopub.status.idle":"2022-07-22T06:46:27.631257Z","shell.execute_reply.started":"2022-07-22T06:46:10.680780Z","shell.execute_reply":"2022-07-22T06:46:27.630088Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1日の合計売り上げ\ntrain.groupby('date')['sales'].sum().plot(kind='line', style='-o', figsize=(30,10))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T06:43:35.518255Z","iopub.execute_input":"2022-07-22T06:43:35.518687Z","iopub.status.idle":"2022-07-22T06:43:35.941411Z","shell.execute_reply.started":"2022-07-22T06:43:35.518655Z","shell.execute_reply":"2022-07-22T06:43:35.940146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stem\n# total_daily_sales = train.groupby('date')['sales'].sum().reset_index()\n# plt.figure(figsize=(30,10))\n# plt.stem(total_daily_sales.date, total_daily_sales.sales)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 前処理","metadata":{}},{"cell_type":"code","source":"train_org = train.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:09:57.401422Z","iopub.execute_input":"2022-07-22T04:09:57.402596Z","iopub.status.idle":"2022-07-22T04:09:57.453559Z","shell.execute_reply.started":"2022-07-22T04:09:57.402554Z","shell.execute_reply":"2022-07-22T04:09:57.452268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 店舗の売り上げが0である日はその店舗が休みであると判断し，除去\ntmp = train_org.groupby(['date', 'store_nbr'])['sales'].sum().reset_index()\ntmp['delete1'] = 0\ntmp.loc[tmp['sales'] == 0, 'delete1'] = 1\ntmp.drop('sales', axis=1, inplace=True)\ntrain = pd.merge(train, tmp, on=['date', 'store_nbr'])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:01.401469Z","iopub.execute_input":"2022-07-22T04:10:01.401884Z","iopub.status.idle":"2022-07-22T04:10:02.185038Z","shell.execute_reply.started":"2022-07-22T04:10:01.401841Z","shell.execute_reply":"2022-07-22T04:10:02.183839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 全店舗で売り上げが0であるfamilyは，そのそもその期間に販売していないと判断し，除去\ntmp = train_org.groupby(['date', 'family'])['sales'].sum().reset_index()\ntmp['delete2'] = 0\ntmp.loc[tmp['sales'] == 0, 'delete2'] = 1\ntmp.drop('sales', axis=1, inplace=True)\ntrain = pd.merge(train, tmp, on=['date', 'family']) # なぜか順序が変わる","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:04.913913Z","iopub.execute_input":"2022-07-22T04:10:04.914431Z","iopub.status.idle":"2022-07-22T04:10:06.318002Z","shell.execute_reply.started":"2022-07-22T04:10:04.914383Z","shell.execute_reply":"2022-07-22T04:10:06.316984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# oilの欠損を線形補完\noil_new = pd.DataFrame(pd.date_range(start=oil['date'].min(), end=oil['date'].max(), freq='D'), columns=['date'])\noil_new = pd.merge(oil_new, oil, how='left', on='date')\noil_new['dcoilwtico'] = oil_new.drop('date', axis=1).interpolate(limit_direction='both')['dcoilwtico']\n\n# trainとtestに分ける\noil_train = oil_new[(train['date'].min() <= oil_new['date']) & (oil_new['date'] <= train['date'].max())]\noil_test = oil_new[(test['date'].min() <= oil_new['date']) & (oil_new['date'] <= test['date'].max())]\n\n# oilを結合\ntrain = pd.merge(train, oil_train, how='left', on='date')\ntest = pd.merge(test, oil_test, how='left', on='date')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:08.505931Z","iopub.execute_input":"2022-07-22T04:10:08.50672Z","iopub.status.idle":"2022-07-22T04:10:09.016237Z","shell.execute_reply.started":"2022-07-22T04:10:08.506672Z","shell.execute_reply":"2022-07-22T04:10:09.014579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# holidaysを結合\nholidays_new = pd.DataFrame(pd.date_range(start=holidays['date'].min(), end=holidays['date'].max(), freq='D'), columns=['date'])\nholidays_new = pd.merge(holidays_new, holidays, how='left', on='date')\nholidays_train = holidays_new[(train['date'].min() <= holidays_new['date']) & (holidays_new['date'] <= train['date'].max())]\nholidays_test = holidays_new[(test['date'].min() <= holidays_new['date']) & (holidays_new['date'] <= test['date'].max())]\n\ntrain = pd.merge(train, holidays_train, how='left', on='date')\ntest = pd.merge(test, holidays_test, how='left', on='date')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:11.70744Z","iopub.execute_input":"2022-07-22T04:10:11.708131Z","iopub.status.idle":"2022-07-22T04:10:12.579354Z","shell.execute_reply.started":"2022-07-22T04:10:11.708085Z","shell.execute_reply":"2022-07-22T04:10:12.578199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# storesを結合\ntrain = pd.merge(train, stores, how='left', on='store_nbr')\ntest = pd.merge(test, stores, how='left', on='store_nbr')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:15.058629Z","iopub.execute_input":"2022-07-22T04:10:15.059044Z","iopub.status.idle":"2022-07-22T04:10:16.051476Z","shell.execute_reply.started":"2022-07-22T04:10:15.05901Z","shell.execute_reply":"2022-07-22T04:10:16.050652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# deleteフラグがたっている行を削除\ntrain = train[(train['delete1'] == 0) & (train['delete2'] == 0)]\n\n# delete1，delete2カラムを削除\ntrain.drop('delete1', axis=1, inplace=True)\ntrain.drop('delete2', axis=1, inplace=True)\n\n# idカラムを削除\ntrain.drop('id', axis=1, inplace=True)\ntest.drop('id', axis=1, inplace=True)\n\n# store_nbrカラムを削除\ntrain.drop('store_nbr', axis=1, inplace=True)\ntest.drop('store_nbr', axis=1, inplace=True)\n\n# dateカラムを変更\ntrain['day']  = pd.to_datetime(train['date']).dt.day\ntrain['month']  = pd.to_datetime(train['date']).dt.month\ntrain['year']  = pd.to_datetime(train['date']).dt.year\ntest['day']  = pd.to_datetime(test['date']).dt.day\ntest['month']  = pd.to_datetime(test['date']).dt.month\ntest['year']  = pd.to_datetime(test['date']).dt.year\ntrain.drop('date', axis=1, inplace=True)\ntest.drop('date', axis=1, inplace=True)\n\n# salesを分離\ntrain_y = train['sales']\ntrain.drop('sales', inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:20.465813Z","iopub.execute_input":"2022-07-22T04:10:20.466253Z","iopub.status.idle":"2022-07-22T04:10:24.614024Z","shell.execute_reply.started":"2022-07-22T04:10:20.466217Z","shell.execute_reply":"2022-07-22T04:10:24.613192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaling_columns = ['onpromotion', 'dcoilwtico']\nscaler = StandardScaler()\nscaler.fit(train[scaling_columns])\nscaled_train = pd.DataFrame(scaler.transform(train[scaling_columns]), columns=scaling_columns, index=train.index)\ntrain.update(scaled_train)\nscaled_test = pd.DataFrame(scaler.transform(test[scaling_columns]), columns=scaling_columns, index=test.index)\ntest.update(scaled_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:29.44267Z","iopub.execute_input":"2022-07-22T04:10:29.443066Z","iopub.status.idle":"2022-07-22T04:10:29.745382Z","shell.execute_reply.started":"2022-07-22T04:10:29.443036Z","shell.execute_reply":"2022-07-22T04:10:29.743909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummies_columns = ['family', 'type_x', 'locale', 'locale_name', 'description', 'transferred', 'city', 'state', 'type_y', 'cluster']\n\ntrain_objs_num = len(train)\ndataset = pd.concat(objs=[train, test], axis=0)\ndataset_preprocessed = pd.get_dummies(dataset, columns=dummies_columns, drop_first=True, dummy_na=True)\ntrain = dataset_preprocessed[:train_objs_num]\ntest = dataset_preprocessed[train_objs_num:]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:10:43.849641Z","iopub.execute_input":"2022-07-22T04:10:43.850073Z","iopub.status.idle":"2022-07-22T04:10:50.824829Z","shell.execute_reply.started":"2022-07-22T04:10:43.850041Z","shell.execute_reply":"2022-07-22T04:10:50.823527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習","metadata":{}},{"cell_type":"code","source":"model = LinearRegression()\nmodel.fit(train, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:11:21.9982Z","iopub.execute_input":"2022-07-22T04:11:21.998571Z","iopub.status.idle":"2022-07-22T04:12:07.43539Z","shell.execute_reply.started":"2022-07-22T04:11:21.998541Z","shell.execute_reply":"2022-07-22T04:12:07.434089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(test)\nsubmission['sales'] = pred","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:14:29.300902Z","iopub.execute_input":"2022-07-22T04:14:29.301704Z","iopub.status.idle":"2022-07-22T04:14:29.333732Z","shell.execute_reply.started":"2022-07-22T04:14:29.301665Z","shell.execute_reply":"2022-07-22T04:14:29.332243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.loc[submission['sales'] < 0, 'sales'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:14:32.380123Z","iopub.execute_input":"2022-07-22T04:14:32.380527Z","iopub.status.idle":"2022-07-22T04:14:32.38734Z","shell.execute_reply.started":"2022-07-22T04:14:32.380494Z","shell.execute_reply":"2022-07-22T04:14:32.386207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T04:14:35.463328Z","iopub.execute_input":"2022-07-22T04:14:35.463703Z","iopub.status.idle":"2022-07-22T04:14:35.555978Z","shell.execute_reply.started":"2022-07-22T04:14:35.463673Z","shell.execute_reply":"2022-07-22T04:14:35.554883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. データの内容を確認\n## trainの各column\n#### id\n\n#### date\n- 2013-01-01 ~ 2017-08-15\n\n#### store_nbr\n- 店の識別番号．storesに識別番号ごとの詳細情報あり．\n- 1 ~ 54\n\n#### family\n- 商品のカテゴリ\n- 33種類\n\n#### sales\n- 応答変数\n\n#### onpromotion\n- 商品カテゴリごとの宣伝された商品の個数\n- 0 ~ 741","metadata":{}},{"cell_type":"markdown","source":"## データの確認：test  \n- id\n- date  \n    2017-08-16 ~ 2017-08-31  \n    testデータ後15日分\n- store_nbr\n- family\n- onpromotion","metadata":{}},{"cell_type":"markdown","source":"## データの確認：oil  \n- date  \n    2013-01-01 ~ 2017-08-31  \n    テストデータ分も含まれている\n- dcoilwtico  \n欠損あり","metadata":{}},{"cell_type":"markdown","source":"## データの確認：holidays\n- date  \n    2012-03-02 ~ 2017-12-26  \n    テストデータ分も含まれている\n- type  \n    6種類\n- locale\n- locale_name\n- description\n- transferred","metadata":{}},{"cell_type":"markdown","source":"## データの確認：transactions\nテストデータ分が含まれていない  \n使えない？","metadata":{}},{"cell_type":"markdown","source":"## データの確認：stores\ncityはholidaysと内容が被る","metadata":{}}]}