{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport random\nimport math\nfrom decimal import Decimal as dec\nimport datetime\nimport time\nimport gc\nimport lightgbm as lgb\nimport pickle\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-08T08:39:01.723995Z","iopub.execute_input":"2022-08-08T08:39:01.724538Z","iopub.status.idle":"2022-08-08T08:39:03.598385Z","shell.execute_reply.started":"2022-08-08T08:39:01.724433Z","shell.execute_reply":"2022-08-08T08:39:03.597095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ndef load_data():\n    train_df = reduce_mem_usage(pd.read_csv(\"../input/m5-forecasting-accuracy/sales_train_evaluation.csv\"))\n    prices_df = reduce_mem_usage(pd.read_csv(\"../input/m5-forecasting-accuracy/sell_prices.csv\"))\n    calendar_df = reduce_mem_usage(pd.read_csv(\"../input/m5-forecasting-accuracy/calendar.csv\"))\n    submission_df = reduce_mem_usage(pd.read_csv(\"../input/m5-forecasting-accuracy/sample_submission.csv\"))\n    return train_df, prices_df, calendar_df, submission_df\n\ntrain_df, prices_df, calendar_df, submission_df = load_data()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:39:03.601502Z","iopub.execute_input":"2022-08-08T08:39:03.602019Z","iopub.status.idle":"2022-08-08T08:42:52.155104Z","shell.execute_reply.started":"2022-08-08T08:39:03.601965Z","shell.execute_reply":"2022-08-08T08:42:52.153863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_base_grid(train_df, end_train_day_x, predict_horizon):\n    index_columns = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']\n\n    grid_df = pd.melt(train_df, id_vars=index_columns, var_name='d', value_name='sales')\n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n\n    grid_df['d_org'] = grid_df['d']\n    grid_df['d'] = grid_df['d'].apply(lambda x: x[2:]).astype(np.int16)\n\n    time_mask = (grid_df['d'] > end_train_day_x) &  (grid_df['d'] <= end_train_day_x + predict_horizon)\n    holdout_df = grid_df.loc[time_mask, [\"id\", \"d\", \"sales\"]].reset_index(drop=True)\n    holdout_df.to_feather(f\"holdout_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    del(holdout_df)\n    gc.collect()\n\n    grid_df = grid_df[grid_df['d'] <= end_train_day_x]\n    grid_df['d'] = grid_df['d_org']\n    grid_df = grid_df.drop('d_org', axis=1)\n\n    add_grid = pd.DataFrame()\n    for i in range(predict_horizon):\n        temp_df = train_df[index_columns]\n        temp_df = temp_df.drop_duplicates()\n        temp_df['d'] = 'd_' + str(end_train_day_x + i + 1)\n        temp_df['sales'] = np.nan\n        add_grid = pd.concat([add_grid, temp_df])\n    \n    grid_df = pd.concat([grid_df, add_grid])\n    grid_df = grid_df.reset_index(drop=True)\n    \n    for col in index_columns:\n        grid_df[col] = grid_df[col].astype('category')\n    \n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    grid_df.to_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    del(grid_df)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.157112Z","iopub.execute_input":"2022-08-08T08:42:52.157594Z","iopub.status.idle":"2022-08-08T08:42:52.170712Z","shell.execute_reply.started":"2022-08-08T08:42:52.157548Z","shell.execute_reply":"2022-08-08T08:42:52.169487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_by_concat(df1, df2, merge_on):\n    merged_gf = df1[merge_on]\n    merged_gf = merged_gf.merge(df2, on=merge_on, how='left')\n    new_columns = [col for col in list(merged_gf) if col not in merge_on]\n    df1 = pd.concat([df1, merged_gf[new_columns]], axis=1)\n    return df1\n    \ndef calc_release_week(prices_df, end_train_day_x, predict_horizon):\n    index_columns = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']\n    \n    grid_df = pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    \n    release_df = prices_df.groupby(['store_id', 'item_id'])['wm_yr_wk'].agg(['min']).reset_index()\n    release_df.columns = ['store_id', 'item_id', 'release']\n    \n    grid_df = merge_by_concat(grid_df, release_df, ['store_id', 'item_id'])\n    \n    del release_df\n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    gc.collect()\n    \n    grid_df = merge_by_concat(grid_df, calendar_df[['wm_yr_wk', 'd']], ['d'])\n    grid_df = grid_df.reset_index(drop=True)\n\n    grid_df['release'] = grid_df['release'] - grid_df['release'].min()\n    grid_df['release'] = grid_df['release'].astype(np.int16)\n    \n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    grid_df.to_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    del(grid_df)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.175014Z","iopub.execute_input":"2022-08-08T08:42:52.175495Z","iopub.status.idle":"2022-08-08T08:42:52.189007Z","shell.execute_reply.started":"2022-08-08T08:42:52.175437Z","shell.execute_reply":"2022-08-08T08:42:52.187592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_grid_price(prices_df, calendar_df, end_train_day_x, predict_horizon):\n\n    grid_df = pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n\n    prices_df['price_max'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('max')\n    prices_df['price_min'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('min')\n    prices_df['price_std'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('std')\n    prices_df['price_mean'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('mean')\n    prices_df['price_norm'] = prices_df['sell_price'] / prices_df['price_max']\n    prices_df['price_nunique'] = prices_df.groupby(['store_id', 'item_id'])['sell_price'].transform('nunique')\n    prices_df['item_nunique'] = prices_df.groupby(['store_id', 'sell_price'])['item_id'].transform('nunique')\n\n    calendar_prices = calendar_df[['wm_yr_wk', 'month', 'year']]\n    calendar_prices = calendar_prices.drop_duplicates(subset=['wm_yr_wk'])\n    prices_df = prices_df.merge(calendar_prices[['wm_yr_wk', 'month', 'year']], on=['wm_yr_wk'], how='left')\n    \n    del calendar_prices\n    gc.collect()\n    \n    prices_df['price_momentum'] = prices_df['sell_price'] / prices_df.groupby(['store_id', 'item_id'])[\n        'sell_price'].transform(lambda x: x.shift(1))\n    prices_df['price_momentum_m'] = prices_df['sell_price'] / prices_df.groupby(['store_id', 'item_id', 'month'])[\n        'sell_price'].transform('mean')\n    prices_df['price_momentum_y'] = prices_df['sell_price'] / prices_df.groupby(['store_id', 'item_id', 'year'])[\n        'sell_price'].transform('mean')\n\n    prices_df['sell_price_cent'] = [math.modf(p)[0] for p in prices_df['sell_price']]\n    prices_df['price_max_cent'] = [math.modf(p)[0] for p in prices_df['price_max']]\n    prices_df['price_min_cent'] = [math.modf(p)[0] for p in prices_df['price_min']]\n\n    del prices_df['month'], prices_df['year']\n    prices_df = reduce_mem_usage(prices_df, verbose=False)\n    gc.collect()\n    \n    original_columns = list(grid_df)\n    grid_df = grid_df.merge(prices_df, on=['store_id', 'item_id', 'wm_yr_wk'], how='left')\n    del(prices_df)\n    gc.collect()\n    \n    keep_columns = [col for col in list(grid_df) if col not in original_columns]\n    grid_df = grid_df[['id', 'd'] + keep_columns]\n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n\n    grid_df.to_feather(f\"grid_price_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    del(grid_df)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.191096Z","iopub.execute_input":"2022-08-08T08:42:52.191709Z","iopub.status.idle":"2022-08-08T08:42:52.210099Z","shell.execute_reply.started":"2022-08-08T08:42:52.191666Z","shell.execute_reply":"2022-08-08T08:42:52.208949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_moon_phase(d):  # 0=new, 4=full; 4 days/phase\n    diff = datetime.datetime.strptime(d, '%Y-%m-%d') - datetime.datetime(2001, 1, 1)\n    days = dec(diff.days) + (dec(diff.seconds) / dec(86400))\n    lunations = dec(\"0.20439731\") + (days * dec(\"0.03386319269\"))\n    phase_index = math.floor((lunations % dec(1) * dec(8)) + dec('0.5'))\n    return int(phase_index) & 7\n    \ndef generate_grid_calendar(calendar_df, end_train_day_x, predict_horizon):\n    \n    grid_df = pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    grid_df = grid_df[['id', 'd']]\n    gc.collect()\n\n    calendar_df['moon'] = calendar_df.date.apply(get_moon_phase)\n\n    # Merge calendar partly\n    icols = ['date',\n             'd',\n             'event_name_1',\n             'event_type_1',\n             'event_name_2',\n             'event_type_2',\n             'snap_CA',\n             'snap_TX',\n             'snap_WI',\n             'moon',\n             ]\n\n    grid_df = grid_df.merge(calendar_df[icols], on=['d'], how='left')\n\n    icols = ['event_name_1',\n             'event_type_1',\n             'event_name_2',\n             'event_type_2',\n             'snap_CA',\n             'snap_TX',\n             'snap_WI']\n    \n    for col in icols:\n        grid_df[col] = grid_df[col].astype('category')\n\n    grid_df['date'] = pd.to_datetime(grid_df['date'])\n\n    grid_df['tm_d'] = grid_df['date'].dt.day.astype(np.int8)\n    grid_df['tm_w'] = grid_df['date'].dt.isocalendar().week.astype(np.int8)\n    grid_df['tm_m'] = grid_df['date'].dt.month.astype(np.int8)\n    grid_df['tm_y'] = grid_df['date'].dt.year\n    grid_df['tm_y'] = (grid_df['tm_y'] - grid_df['tm_y'].min()).astype(np.int8)\n    grid_df['tm_wm'] = grid_df['tm_d'].apply(lambda x: math.ceil(x / 7)).astype(np.int8)\n\n    grid_df['tm_dw'] = grid_df['date'].dt.dayofweek.astype(np.int8)\n    grid_df['tm_w_end'] = (grid_df['tm_dw'] >= 5).astype(np.int8)\n                                                         \n    del(grid_df['date'])\n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    grid_df.to_feather(f\"grid_calendar_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n                                                         \n    del(grid_df)\n    del(calendar_df)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.212244Z","iopub.execute_input":"2022-08-08T08:42:52.213135Z","iopub.status.idle":"2022-08-08T08:42:52.230296Z","shell.execute_reply.started":"2022-08-08T08:42:52.21309Z","shell.execute_reply":"2022-08-08T08:42:52.229342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def modify_grid_base(end_train_day_x, predict_horizon):\n    grid_df = pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    grid_df['d'] = grid_df['d'].apply(lambda x: x[2:]).astype(np.int16)\n    del grid_df['wm_yr_wk']\n    \n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    grid_df.to_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    \n    del(grid_df)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.232074Z","iopub.execute_input":"2022-08-08T08:42:52.232937Z","iopub.status.idle":"2022-08-08T08:42:52.246502Z","shell.execute_reply.started":"2022-08-08T08:42:52.232866Z","shell.execute_reply":"2022-08-08T08:42:52.245282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_lag_feature(end_train_day_x, predict_horizon):\n    grid_df = pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    grid_df = grid_df[['id', 'd', 'sales']]\n    \n    num_lag_day_list = []\n    num_lag_day = 15\n    for col in range(predict_horizon, predict_horizon + num_lag_day):\n        num_lag_day_list.append(col)\n    \n    num_rolling_day_list = [7, 14, 30, 60, 180]\n    num_shift_rolling_day_list = []\n    for num_shift_day in [1, 7, 14]:\n        for num_rolling_day in [7, 14, 30, 60]:\n            num_shift_rolling_day_list.append([num_shift_day, num_rolling_day])\n   \n    grid_df = grid_df.assign(**{\n        '{}_lag_{}'.format(col, l): grid_df.groupby(['id'])['sales'].transform(lambda x: x.shift(l))\n        for l in num_lag_day_list\n    })\n\n    for col in list(grid_df):\n        if 'lag' in col:\n            grid_df[col] = grid_df[col].astype(np.float16)\n\n    for num_rolling_day in num_rolling_day_list:\n        grid_df['rolling_mean_' + str(num_rolling_day)] = grid_df.groupby(['id'])['sales'].transform(\n            lambda x: x.shift(predict_horizon).rolling(num_rolling_day).mean()).astype(np.float16)\n        grid_df['rolling_std_' + str(num_rolling_day)] = grid_df.groupby(['id'])['sales'].transform(\n            lambda x: x.shift(predict_horizon).rolling(num_rolling_day).std()).astype(np.float16)\n\n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    grid_df.to_feather(f\"lag_feature_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    \n    del(grid_df)\n    gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.248622Z","iopub.execute_input":"2022-08-08T08:42:52.249192Z","iopub.status.idle":"2022-08-08T08:42:52.262645Z","shell.execute_reply.started":"2022-08-08T08:42:52.249149Z","shell.execute_reply":"2022-08-08T08:42:52.261263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_target_encoding_feature(end_train_day_x, predict_horizon):\n\n    grid_df = pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    \n    grid_df.loc[grid_df['d'] > (end_train_day_x - predict_horizon), 'sales'] = np.nan\n    base_cols = list(grid_df)\n\n    icols = [\n        ['state_id'],\n        ['store_id'],\n        ['cat_id'],\n        ['dept_id'],\n        ['state_id', 'cat_id'],\n        ['state_id', 'dept_id'],\n        ['store_id', 'cat_id'],\n        ['store_id', 'dept_id'],\n        ['item_id'],\n        ['item_id', 'state_id'],\n        ['item_id', 'store_id']\n    ]\n\n    for col in icols:\n        col_name = '_' + '_'.join(col) + '_'\n        grid_df['enc' + col_name + 'mean'] = grid_df.groupby(col)['sales'].transform('mean').astype(\n            np.float16)\n        grid_df['enc' + col_name + 'std'] = grid_df.groupby(col)['sales'].transform('std').astype(\n            np.float16)\n\n    keep_cols = [col for col in list(grid_df) if col not in base_cols]\n    grid_df = grid_df[['id', 'd'] + keep_cols]\n\n    grid_df = reduce_mem_usage(grid_df, verbose=False)\n    grid_df.to_feather(f\"target_encoding_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n    \n    del(grid_df)\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.264185Z","iopub.execute_input":"2022-08-08T08:42:52.264543Z","iopub.status.idle":"2022-08-08T08:42:52.278965Z","shell.execute_reply.started":"2022-08-08T08:42:52.26451Z","shell.execute_reply":"2022-08-08T08:42:52.277843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def assemble_grid_by_store(train_df, end_train_day_x, predict_horizon):\n    grid_df = pd.concat([pd.read_feather(f\"grid_df_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\"),\n                     pd.read_feather(f\"grid_price_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\").iloc[:, 2:],\n                     pd.read_feather(f\"grid_calendar_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\").iloc[:, 2:]],\n                     axis=1)\n    gc.collect()\n    store_id_set_list = list(train_df['store_id'].unique())\n\n    index_store = dict()\n    for store_id in store_id_set_list:\n        extract = grid_df[grid_df['store_id'] == store_id]\n        index_store[store_id] = extract.index.to_numpy()\n        extract = extract.reset_index(drop=True)\n        extract.to_feather(f\"grid_full_store_{store_id}_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n\n    del(grid_df)\n    gc.collect()\n    \n    mean_features = [\n        'enc_cat_id_mean', 'enc_cat_id_std',\n        'enc_dept_id_mean', 'enc_dept_id_std',\n        'enc_item_id_mean', 'enc_item_id_std'\n        ]\n    df2 = pd.read_feather(f\"target_encoding_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")[mean_features]\n\n    for store_id in store_id_set_list:\n        df = pd.read_feather(f\"grid_full_store_{store_id}_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n        df = pd.concat([df, df2[df2.index.isin(index_store[store_id])].reset_index(drop=True)], axis=1)\n        df.to_feather(f\"grid_full_store_{store_id}_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n\n    del(df2)\n    gc.collect()\n    \n    df3 = pd.read_feather(f\"lag_feature_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\").iloc[:, 3:]\n\n    for store_id in store_id_set_list:\n        df = pd.read_feather(f\"grid_full_store_{store_id}_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n        df = pd.concat([df, df3[df3.index.isin(index_store[store_id])].reset_index(drop=True)], axis=1)\n        df.to_feather(f\"grid_full_store_{store_id}_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n\n    del(df3)\n    del(store_id_set_list)\n    gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.281535Z","iopub.execute_input":"2022-08-08T08:42:52.282312Z","iopub.status.idle":"2022-08-08T08:42:52.295511Z","shell.execute_reply.started":"2022-08-08T08:42:52.28227Z","shell.execute_reply":"2022-08-08T08:42:52.294669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_grid_by_store(end_train_day_x, predict_horizon, store_id):\n    df = pd.read_feather(f\"grid_full_store_{store_id}_{end_train_day_x}_to_{end_train_day_x + predict_horizon}.feather\")\n                          \n    remove_features = ['id', 'state_id', 'store_id', 'date', 'wm_yr_wk', 'd', 'sales']\n    enable_features = [col for col in list(df) if col not in remove_features]\n    df = df[['id', 'd', 'sales'] + enable_features]\n    df = reduce_mem_usage(df, verbose=False)\n    gc.collect()\n                          \n    return df, enable_features","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.296817Z","iopub.execute_input":"2022-08-08T08:42:52.297328Z","iopub.status.idle":"2022-08-08T08:42:52.322495Z","shell.execute_reply.started":"2022-08-08T08:42:52.297297Z","shell.execute_reply":"2022-08-08T08:42:52.321265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(train_df, seed, end_train_day_x, predict_horizon):\n    \n    lgb_params = {\n            'boosting_type': 'goss',\n            'objective': 'tweedie',\n            'tweedie_variance_power': 1.1,\n            'metric': 'rmse',\n             #'subsample': 0.5,\n             #'subsample_freq': 1,\n            'learning_rate': 0.03,\n            'num_leaves': 2 ** 11 - 1,\n            'min_data_in_leaf': 2 ** 12 - 1,\n            'feature_fraction': 0.5,\n            'max_bin': 100,\n            'boost_from_average': False,\n            'num_boost_round': 1400,\n            'verbose': -1,\n            'num_threads': os.cpu_count(),\n            'force_row_wise': True,\n        }\n\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n    lgb_params['seed'] = seed\n\n    store_id_set_list = list(train_df['store_id'].unique())\n    print(f\"training stores: {store_id_set_list}\")\n    \n    feature_importance_all_df = pd.DataFrame()\n    for store_index, store_id in enumerate(store_id_set_list):\n        print(f'now training {store_id} store')\n\n        grid_df, enable_features = load_grid_by_store(end_train_day_x, predict_horizon, store_id)\n\n        train_mask = grid_df['d'] <= end_train_day_x\n        valid_mask = train_mask & (grid_df['d'] > (end_train_day_x - predict_horizon))\n        preds_mask = grid_df['d'] > (end_train_day_x - 100)\n\n        train_data = lgb.Dataset(grid_df[train_mask][enable_features],\n                                 label=grid_df[train_mask]['sales'])\n\n        valid_data = lgb.Dataset(grid_df[valid_mask][enable_features],\n                                 label=grid_df[valid_mask]['sales'])\n\n\n        # Saving part of the dataset for later predictions\n        # Removing features that we need to calculate recursively\n        grid_df = grid_df[preds_mask].reset_index(drop=True)\n        grid_df.to_feather(f'test_{store_id}_{predict_horizon}.feather')\n        del(grid_df)\n        gc.collect()\n        \n        estimator = lgb.train(lgb_params,\n                              train_data,\n                              valid_sets=[valid_data],\n                              callbacks=[lgb.log_evaluation(period=100, show_stdv=False)],\n                              )\n\n        model_name = str(f'lgb_model_{store_id}_{predict_horizon}.bin')\n        feature_importance_store_df = pd.DataFrame(sorted(zip(enable_features, estimator.feature_importance())),\n                                                   columns=['feature_name', 'importance'])\n        feature_importance_store_df = feature_importance_store_df.sort_values('importance', ascending=False)\n        feature_importance_store_df['store_id'] = store_id\n        feature_importance_store_df.to_csv(f'feature_importance_{store_id}_{predict_horizon}.csv', index=False)\n        feature_importance_all_df = pd.concat([feature_importance_all_df, feature_importance_store_df])\n        pickle.dump(estimator, open(model_name, 'wb'))\n\n        del([train_data, valid_data, estimator])\n        gc.collect()\n\n    feature_importance_all_df.to_csv(f'feature_importance_all_{predict_horizon}.csv', index=False)\n    feature_importance_agg_df = feature_importance_all_df.groupby(\n        'feature_name')['importance'].agg(['mean', 'std']).reset_index()\n    feature_importance_agg_df.columns = ['feature_name', 'importance_mean', 'importance_std']\n    feature_importance_agg_df = feature_importance_agg_df.sort_values('importance_mean', ascending=False)\n    feature_importance_agg_df.to_csv(f'feature_importance_agg_{predict_horizon}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.324109Z","iopub.execute_input":"2022-08-08T08:42:52.324463Z","iopub.status.idle":"2022-08-08T08:42:52.34146Z","shell.execute_reply.started":"2022-08-08T08:42:52.324434Z","shell.execute_reply":"2022-08-08T08:42:52.340318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_pipeline(train_df, prices_df, calendar_df, end_train_day_x_list, prediction_horizon_list):\n    \n    for end_train_day_x in end_train_day_x_list:\n        \n        for predict_horizon in prediction_horizon_list:\n            \n            print(f\"end training point day: {end_train_day_x} - prediction horizon: {predict_horizon} days\")\n\n            # Data preparation\n            generate_base_grid(train_df, end_train_day_x, predict_horizon)\n            calc_release_week(prices_df, end_train_day_x, predict_horizon)\n            generate_grid_price(prices_df, calendar_df, end_train_day_x, predict_horizon)\n            generate_grid_calendar(calendar_df, end_train_day_x, predict_horizon)\n            modify_grid_base(end_train_day_x, predict_horizon)\n            generate_lag_feature(end_train_day_x, predict_horizon)\n            generate_target_encoding_feature(end_train_day_x, predict_horizon)\n            assemble_grid_by_store(train_df, end_train_day_x, predict_horizon)\n\n            # Modelling\n            train(train_df, seed, end_train_day_x, predict_horizon)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.344333Z","iopub.execute_input":"2022-08-08T08:42:52.344813Z","iopub.status.idle":"2022-08-08T08:42:52.357727Z","shell.execute_reply.started":"2022-08-08T08:42:52.344751Z","shell.execute_reply":"2022-08-08T08:42:52.356258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_train_day_x_list = [1913] # [1941, 1913, 1885, 1857, 1829, 1577]\nprediction_horizon_list = [21] # [7, 14, 21, 28]\nseed = 42\n\ntrain_pipeline(train_df, prices_df, calendar_df, end_train_day_x_list, prediction_horizon_list)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T08:42:52.359338Z","iopub.execute_input":"2022-08-08T08:42:52.359842Z"},"trusted":true},"execution_count":null,"outputs":[]}]}