{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":29781,"databundleVersionId":2887556,"sourceType":"competition"}],"dockerImageVersionId":30458,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom statsmodels.graphics.tsaplots import plot_acf\nfrom statsmodels.graphics.tsaplots import plot_pacf","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#paths\nTrain_path =  \"/kaggle/input/store-sales-time-series-forecasting/train.csv\"\nTest_path = \"/kaggle/input/store-sales-time-series-forecasting/test.csv\"\nTransaction_path = \"/kaggle/input/store-sales-time-series-forecasting/transactions.csv\"\nStores_path = \"/kaggle/input/store-sales-time-series-forecasting/stores.csv\"\nOil_path = \"/kaggle/input/store-sales-time-series-forecasting/oil.csv\"\nHoliday_path = \"/kaggle/input/store-sales-time-series-forecasting/holidays_events.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:24.388647Z","iopub.execute_input":"2023-06-01T05:32:24.390039Z","iopub.status.idle":"2023-06-01T05:32:24.397401Z","shell.execute_reply.started":"2023-06-01T05:32:24.389971Z","shell.execute_reply":"2023-06-01T05:32:24.395989Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(Train_path)\ntest_df = pd.read_csv(Test_path)\ntransaction_df = pd.read_csv(Transaction_path)\nstores_df = pd.read_csv(Stores_path)\noil_df = pd.read_csv(Oil_path)\nholiday_df = pd.read_csv(Holiday_path)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:24.399285Z","iopub.execute_input":"2023-06-01T05:32:24.400001Z","iopub.status.idle":"2023-06-01T05:32:28.527690Z","shell.execute_reply.started":"2023-06-01T05:32:24.399951Z","shell.execute_reply":"2023-06-01T05:32:28.525935Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nそれぞれの日での店ごとの商品の種類と売り上げ\nおよびその日に宣伝していた各種類での商品数\n\"\"\"\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.531469Z","iopub.execute_input":"2023-06-01T05:32:28.532697Z","iopub.status.idle":"2023-06-01T05:32:28.580576Z","shell.execute_reply.started":"2023-06-01T05:32:28.532625Z","shell.execute_reply":"2023-06-01T05:32:28.578989Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.582830Z","iopub.execute_input":"2023-06-01T05:32:28.583396Z","iopub.status.idle":"2023-06-01T05:32:28.605630Z","shell.execute_reply.started":"2023-06-01T05:32:28.583341Z","shell.execute_reply":"2023-06-01T05:32:28.604161Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nエクアドルはオイルに依存しているのでオイルの価格が変動すると\n商品の値段が変動する可能性が高い。\n\"\"\"\noil_df\n","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.607670Z","iopub.execute_input":"2023-06-01T05:32:28.608870Z","iopub.status.idle":"2023-06-01T05:32:28.629349Z","shell.execute_reply.started":"2023-06-01T05:32:28.608827Z","shell.execute_reply":"2023-06-01T05:32:28.627591Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\n店の種類、似ているものはクラスターが同じ\n\"\"\"\nstores_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.631942Z","iopub.execute_input":"2023-06-01T05:32:28.632535Z","iopub.status.idle":"2023-06-01T05:32:28.653871Z","shell.execute_reply.started":"2023-06-01T05:32:28.632463Z","shell.execute_reply":"2023-06-01T05:32:28.651842Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\n祝日タイプholidayは一時的な祝日、additionalは毎年ある\n日にちが変わっているものもあるので注意\n\"\"\"\n#typeがstores_dfのtypeと被るのでrename\ndisplay(holiday_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.657232Z","iopub.execute_input":"2023-06-01T05:32:28.658830Z","iopub.status.idle":"2023-06-01T05:32:28.682430Z","shell.execute_reply.started":"2023-06-01T05:32:28.658757Z","shell.execute_reply":"2023-06-01T05:32:28.680538Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transaction_df","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.684010Z","iopub.execute_input":"2023-06-01T05:32:28.684997Z","iopub.status.idle":"2023-06-01T05:32:28.709400Z","shell.execute_reply.started":"2023-06-01T05:32:28.684949Z","shell.execute_reply":"2023-06-01T05:32:28.707797Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train_dfにstores_df, oil_df, holiday_df結合 必要なdfをまとめることが大事\nholiday_df = holiday_df.rename(columns={\"type\":\"holiday_type\"})\ntrain_df = train_df.merge(stores_df, on=\"store_nbr\")\ntrain_df = train_df.merge(oil_df, on=\"date\", how=\"left\")\ntrain_df = train_df.merge(holiday_df, on=\"date\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:28.715794Z","iopub.execute_input":"2023-06-01T05:32:28.717249Z","iopub.status.idle":"2023-06-01T05:32:31.872278Z","shell.execute_reply.started":"2023-06-01T05:32:28.717179Z","shell.execute_reply":"2023-06-01T05:32:31.870472Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:31.874039Z","iopub.execute_input":"2023-06-01T05:32:31.874542Z","iopub.status.idle":"2023-06-01T05:32:31.898957Z","shell.execute_reply.started":"2023-06-01T05:32:31.874490Z","shell.execute_reply":"2023-06-01T05:32:31.897125Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#各データの型を見る\ntrain_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:31.900876Z","iopub.execute_input":"2023-06-01T05:32:31.901421Z","iopub.status.idle":"2023-06-01T05:32:31.924062Z","shell.execute_reply.started":"2023-06-01T05:32:31.901369Z","shell.execute_reply":"2023-06-01T05:32:31.922887Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#欠損値を処理する　\"パーセンテージ表記\" #母数がdfの長さとわかりきってるので\ntrain_df.isnull().sum() / len(train_df) * 100\n#holiday_dfで欠損値多いのは当然である。祝日以外NaN","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:31.926039Z","iopub.execute_input":"2023-06-01T05:32:31.926490Z","iopub.status.idle":"2023-06-01T05:32:34.608891Z","shell.execute_reply.started":"2023-06-01T05:32:31.926449Z","shell.execute_reply":"2023-06-01T05:32:34.607594Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#重複を削除する\nprint(train_df.duplicated().any())","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:34.610554Z","iopub.execute_input":"2023-06-01T05:32:34.611099Z","iopub.status.idle":"2023-06-01T05:32:39.764939Z","shell.execute_reply.started":"2023-06-01T05:32:34.611057Z","shell.execute_reply":"2023-06-01T05:32:39.763130Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.duplicated().any()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:39.766599Z","iopub.execute_input":"2023-06-01T05:32:39.767143Z","iopub.status.idle":"2023-06-01T05:32:39.786651Z","shell.execute_reply.started":"2023-06-01T05:32:39.767105Z","shell.execute_reply":"2023-06-01T05:32:39.785281Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#次にEDAをしていく\n#groupbyしてから可視化する。","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:39.789882Z","iopub.execute_input":"2023-06-01T05:32:39.790987Z","iopub.status.idle":"2023-06-01T05:32:39.799971Z","shell.execute_reply.started":"2023-06-01T05:32:39.790919Z","shell.execute_reply":"2023-06-01T05:32:39.798656Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#まず店ごとに取り扱っている商品に違いがあるか調べる\nid_df = train_df[[\"store_nbr\",\"id\"]]\nid_df[\"id\"].describe()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:39.801867Z","iopub.execute_input":"2023-06-01T05:32:39.802664Z","iopub.status.idle":"2023-06-01T05:32:40.984622Z","shell.execute_reply.started":"2023-06-01T05:32:39.802605Z","shell.execute_reply":"2023-06-01T05:32:40.982919Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# まず店ごとの売り上げをみて店ごとの人気具合を把握する\npopular = train_df.groupby(\"store_nbr\")[\"sales\"].sum().reset_index()\npopular = popular.sort_values(\"sales\", ascending=False)\ndisplay(popular)\n#可視化\nplt.figure(figsize=(12,6))\nsns.barplot(data=popular, x=\"store_nbr\", y=\"sales\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:40.988013Z","iopub.execute_input":"2023-06-01T05:32:40.988740Z","iopub.status.idle":"2023-06-01T05:32:41.992447Z","shell.execute_reply.started":"2023-06-01T05:32:40.988671Z","shell.execute_reply":"2023-06-01T05:32:41.991012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#datetime型を作る + 年、月を取り出す\ntrain_df[\"date\"] = pd.to_datetime(train_df[\"date\"])\ntrain_df[\"year\"] = train_df[\"date\"].dt.year\ntrain_df[\"month\"] = train_df[\"date\"].dt.month","metadata":{"execution":{"iopub.status.busy":"2023-06-01T12:58:44.244420Z","iopub.execute_input":"2023-06-01T12:58:44.244953Z","iopub.status.idle":"2023-06-01T12:58:45.370086Z","shell.execute_reply.started":"2023-06-01T12:58:44.244910Z","shell.execute_reply":"2023-06-01T12:58:45.368911Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#各年での月ごとの売り上げを可視化する。\nyears = train_df[\"year\"].unique()\nmonths = train_df[\"month\"].unique()\ntime_sales = train_df.groupby([\"year\", \"month\"])[\"sales\"].sum().reset_index()\nplt.figure(figsize=(11,6))\nfor y in years:\n    time_sales_obj = time_sales.loc[time_sales[\"year\"]==y]\n    plt.plot(time_sales_obj[\"month\"], time_sales_obj[\"sales\"], label=str(y), marker=\"o\")\nplt.xlabel(\"Month\")\nplt.ylabel(\"sales\")\nplt.xticks(months)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-01T13:21:38.794537Z","iopub.execute_input":"2023-06-01T13:21:38.795061Z","iopub.status.idle":"2023-06-01T13:21:39.343323Z","shell.execute_reply.started":"2023-06-01T13:21:38.795019Z","shell.execute_reply":"2023-06-01T13:21:39.339530Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#特売の売り上げを調べる\n#自己相関(Autocorrelation)を見る。さらに自己変編相関計数も調べる。\npromo_sales = train_df.groupby(\"date\")[\"sales\", \"onpromotion\"].sum().reset_index()\npromo_sales.info()\nACFvalues = promo_sales[\"sales\"].autocorr()\nprint(\"Autocorrelation: \", ACFvalues)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#店ごとに取り扱っている商品を調べる。また商品間の相関関係を調べる。\nfamily_df= train_df[[\"store_nbr\", \"family\",\"sales\"]]\nstore_N = len(family_df[\"store_nbr\"].unique())\nfor i in range (0, 5):\n    fam_list = family_df.loc[family_df[\"store_nbr\"]==i]\n    print(\"store_nbr:\" + str(i))\n    df_pivot = pd.pivot_table(fam_list, index=\"store_nbr\", columns=\"family\", values=\"sales\", aggfunc=\"sum\")\n    display(df_pivot)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:41.994279Z","iopub.execute_input":"2023-06-01T05:32:41.995587Z","iopub.status.idle":"2023-06-01T05:32:42.282414Z","shell.execute_reply.started":"2023-06-01T05:32:41.995532Z","shell.execute_reply":"2023-06-01T05:32:42.280747Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#すべての店で取り扱っている商品は同じ種類33種類\n#各店ごとに回帰分析をおこなってみる。回帰式をたてる前に特徴量の相関関係を調べる。\n#商品ごとの相関関係を調べる。\n#fam_corr = family_df\n#fam_corr = family_df.groupby(\"store\")\n#display(sns.heatmap(fam_corr, vmax=1, vmin=-1, center=0))\nfam_corr_df = pd.DataFrame()\nfor i in range (store_N):\n    fam_list = family_df.loc[family_df[\"store_nbr\"]==i]\n    fam_uni = fam_list[\"family\"].unique()\n    fam_list = fam_list.groupby(\"family\").sum()[\"sales\"]\n    for j in range(len(fam_uni)):\n        fam_corr_df[str(i)] = fam_list\n    # for j in range (len(fam_uni)):\n       # fam_corr_df[fam_uni[j]] = fam_list[\"sales\"].loc[fam_list[\"family\"]==fam_uni[j]]\ndisplay(fam_corr_df)","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:42.284227Z","iopub.execute_input":"2023-06-01T05:32:42.284637Z","iopub.status.idle":"2023-06-01T05:32:43.794278Z","shell.execute_reply.started":"2023-06-01T05:32:42.284599Z","shell.execute_reply":"2023-06-01T05:32:43.792755Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#相関関係を調べる\nfam_corr = fam_corr_df.corr()\nsns.heatmap(fam_corr, vmax=1, vmin=-1)\n#全体的に相関係数高い店だけ選ぶ。2,6,18,41,45,47を選ぶ。","metadata":{"execution":{"iopub.status.busy":"2023-06-01T05:32:43.796503Z","iopub.execute_input":"2023-06-01T05:32:43.797454Z","iopub.status.idle":"2023-06-01T05:32:44.543179Z","shell.execute_reply.started":"2023-06-01T05:32:43.797393Z","shell.execute_reply":"2023-06-01T05:32:44.541597Z"},"trusted":true},"outputs":[],"execution_count":null}]}