{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !kaggle competitions download -c store-sales-time-series-forecasting","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from zipfile import ZipFile","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with ZipFile('store-sales-time-series-forecasting.zip', 'r') as zipObj:\n#     zipObj.extractall()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/store-sales-time-series-forecasting/train.csv')\ntrain_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['date'] = pd.to_datetime(train_df['date'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"train_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(np.log1p(train_df['sales']), kde=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store_nbr_sales_per_month = train_df.groupby(['date', 'store_nbr']).aggregate({'sales': 'sum'}).unstack().resample('1M').sum()\nstore_nbr_sales_per_month.plot(figsize=(21,5))\nplt.legend(bbox_to_anchor=(0.9,-.2), ncol=9)\nplt.title(\"Sales per month by store_nbr\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"family_sales_per_month = train_df.groupby(['date', 'family']).aggregate({'sales': 'sum'}).unstack().resample('1M').sum()\nfamily_sales_per_month.plot(figsize=(21,5))\nplt.legend(bbox_to_anchor=(1, -.2), ncol=6)\nplt.title(\"Sales per month by family\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=train_df, x='onpromotion')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['sales'] = np.log1p(train_df['sales'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stores_df = pd.read_csv('../input/store-sales-time-series-forecasting/stores.csv')\nstores_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stores_df.describe(include=object)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stores_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df = pd.read_csv('../input/store-sales-time-series-forecasting/transactions.csv')\ntransactions_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df['date'] = pd.to_datetime(transactions_df['date'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"store_nbr_transactions_per_month = transactions_df.groupby(['date', 'store_nbr']).aggregate({'transactions': 'sum'}).unstack().resample('1M').sum()\nstore_nbr_transactions_per_month.plot(figsize=(21,5))\nplt.legend(bbox_to_anchor=(0.9,-0.2), ncol=7)\nplt.title('Transactions per month by store_nbr')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/store-sales-time-series-forecasting/test.csv')\ntest_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_id = test_df['id']\ntest_df['date'] = pd.to_datetime(test_df['date'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holidays_df = pd.read_csv('../input/store-sales-time-series-forecasting/holidays_events.csv')\nholidays_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holidays_df['date'] = pd.to_datetime(holidays_df['date'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"holidays_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_df = pd.read_csv('../input/store-sales-time-series-forecasting/oil.csv')\noil_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_df['dcoilwtico'].fillna(method='backfill', inplace=True)\noil_df['date'] = pd.to_datetime(oil_df['date'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oil_df.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=oil_df, x='date', y='dcoilwtico')\nplt.title('Oil price')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lag_features(df, lags):\n    for lag in lags:\n        df[f\"sales_t-{lag}\"] = df.groupby([\"store_nbr\", \"family\"])[\"sales\"].transform(\n            lambda x: x.shift(lag))\n    return df\n\ndef fill_na(df):\n    df['holiday_type'] = df['holiday_type'].fillna('Common')\n    df['locale'] = df['locale'].fillna('Common')\n    # df['locale_name'] = df['locale_name'].fillna('Unknown')\n    df['description'] = df['description'].fillna('Unknown')\n    df['transferred'] = df['transferred'].fillna(False)\n    df['dcoilwtico'] = df['dcoilwtico'].fillna(method='backfill')\n    # df['transactions'] = df['transactions'].fillna(0)\n    return df\n\ndef merge_data(df):\n    df = df \\\n        .merge(stores_df, left_on=\"store_nbr\", right_on=\"store_nbr\", how=\"left\") \\\n        .rename(columns={\"type\": \"store_type\"}) \\\n        .merge(transactions_df, left_on=[\"date\", \"store_nbr\"], right_on=[\"date\", \"store_nbr\"], how=\"left\") \\\n        .merge(holidays_df, left_on=\"date\", right_on=\"date\", how=\"left\") \\\n        .drop_duplicates(subset=\"id\") \\\n        .rename(columns={\"type\": \"holiday_type\"}) \\\n        .merge(oil_df, left_on=\"date\", right_on=\"date\", how=\"left\") \n    return df\n\ndef create_date_features(df):\n    df['month'] = df.date.dt.month\n    df['day_of_month'] = df.date.dt.day\n    df['day_of_year'] = df.date.dt.dayofyear\n    df['week_of_year'] = df.date.dt.weekofyear\n    df['day_of_week'] = df.date.dt.dayofweek\n    df['year'] = df.date.dt.year\n    return df\n\n# Random Noise\ndef random_noise(dataframe):\n    return np.random.normal(scale=2.0, size=(len(dataframe),))\n\ndef roll_mean_features(dataframe, windows):\n    for window in windows:\n        dataframe['sales_roll_mean_' + str(window)] = dataframe.groupby([\"store_nbr\", \"family\"])['sales']. \\\n                                                          transform(\n            lambda x: x.shift(16).rolling(window=window, min_periods=7, win_type=\"triang\").mean()) + random_noise(\n            dataframe)\n    return dataframe\n\ndef ewm_features(dataframe, alphas, lags):\n    for alpha in alphas:\n        for lag in lags:\n            dataframe['sales_ewm_alpha_' + str(alpha).replace(\".\", \"\") + \"_lag_\" + str(lag)] = \\\n                dataframe.groupby([\"store_nbr\", \"family\"])['sales'].transform(lambda x: x.shift(lag).ewm(alpha=alpha).mean())\n    return dataframe","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = [\n    'date',\n    'store_nbr', \n    'family', \n    'sales', \n    'onpromotion', \n    'cluster', \n    'holiday_type', \n    'locale', \n    'description', \n    'transferred',\n    'dcoilwtico'\n]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df = pd.concat([train_df, test_df], axis=0)\nall_df = merge_data(all_df)\nall_df = all_df[col]\nall_df = fill_na(all_df)\nall_df = create_date_features(all_df)\nall_df = lag_features(all_df, lags = [*range(1, 16), 16,17,18,19,20,21,22,30,31,90,180,364])\nall_df = roll_mean_features(all_df,[16,17,18,30])\n\nalphas = [0.95, 0.9, 0.8, 0.5]\nlags =[1, 7,30]\nall_df = ewm_features(all_df, alphas, lags)\n\nall_df['store_nbr'] = all_df['store_nbr'].astype('category')\nall_df['family'] = all_df['family'].astype('category')\nall_df['store_nbr'] = all_df['store_nbr'].astype('category')\nall_df['cluster'] = all_df['cluster'].astype('category')\nall_df['family'] = all_df['family'].astype('category')\nall_df['holiday_type'] = all_df['holiday_type'].astype('category')\nall_df['locale'] = all_df['locale'].astype('category')\nall_df['description'] = all_df['description'].astype('category')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df = all_df.query(\"date > '2013-12-31'\")\nX = all_df.query(\"date <= '2017-08-15'\")\nX_test = all_df.query(\"date > '2017-08-15'\")\nX.drop(['date'], inplace=True, axis=1)\nX_test.drop(['date', 'year'], inplace=True, axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = X.corr()\ncorr['sales'].sort_values(ascending=False)","metadata":{"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y = X['sales']\nX = X.drop(['sales', 'year'], axis=1)\nX_test = X_test.drop(['sales'], axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, Y.shape, X_test.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install lightgbm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install catboost","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import catboost as cb\nfrom catboost import CatBoostRegressor\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error, mean_squared_log_error\nfrom sklearn.model_selection import train_test_split","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.2, random_state=42)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.dtypes","metadata":{"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recursive_predict(model):\n    global X_test\n    output = np.array([])\n    for day in range(16, 32):\n        pred = model.predict(X_test.query(f\"day_of_month == {day}\"))\n        pred[pred < 0] = 0\n        output = np.concatenate([output, pred], axis=0)\n        for k in range(day+1, 32):\n            X_test.loc[X_test[X_test[\"day_of_month\"] == k].index, f\"sales_t-{k-day}\"] = pred\n    return output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {\n    'metric': 'mse',\n    'boosting_type' : 'gbdt',\n    'num_leaves': 8,\n    'learning_rate': 0.2,\n    'max_depth': 7,\n    'verbose': 0,\n    'num_boost_round': 5000,\n    'early_stopping_rounds': 200,\n    'nthread': -1,\n    'force_col_wise': True,\n}\n\ntrain_dataset = lgb.Dataset(data=X_train, label=Y_train, feature_name='auto')\nval_dataset = lgb.Dataset(data=X_val, label=Y_val, reference=train_dataset, feature_name='auto')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = lgb.train(\n    params=lgb_params, \n    train_set=train_dataset,\n    valid_sets=[train_dataset, val_dataset],\n    verbose_eval=100,\n)","metadata":{"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_val_pred = model.predict(X_val, num_iteration=model.best_iteration)\nY_val_pred[Y_val_pred < 0] = 0\nprint(np.sqrt(mean_squared_error(Y_val, Y_val_pred)))\nprint(mean_squared_log_error(np.expm1(Y_val), np.expm1(Y_val_pred)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rec_pred_lgb = recursive_predict(model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = all_df.query(\"date > '2017-08-15'\")\nX_test.drop(['date', 'year', 'sales'], inplace=True, axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_indices = np.where(X_train.dtypes == 'category')[0]\ntrain_dataset_cb = cb.Pool(X_train, Y_train, cat_features=cat_indices)\nval_dataset_cb = cb.Pool(X_val, Y_val, cat_features=cat_indices)\n\nmodel_cb = CatBoostRegressor(\n    loss_function='RMSE',\n    cat_features=cat_indices,\n    max_depth=7,\n    n_estimators=100\n)\nmodel_cb.fit(train_dataset_cb, eval_set=val_dataset_cb)","metadata":{"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_val_pred_cb = model_cb.predict(X_val)\nY_val_pred_cb[Y_val_pred_cb < 0] = 0\nprint(np.sqrt(mean_squared_error(Y_val, Y_val_pred_cb)))\nprint(mean_squared_log_error(np.expm1(Y_val), np.expm1(Y_val_pred_cb)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rec_pred_cb = recursive_predict(model_cb)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = all_df.query(\"date > '2017-08-15'\")\nX_test.drop(['date', 'year', 'sales'], inplace=True, axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution = pd.DataFrame({'id': test_id, 'sales': np.expm1(rec_pred_lgb)})\nsolution.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}