{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook shows a part of Nth solution.\n\nPlease see [Part of 22nd solution - single LGBM (Private:0.03038)](https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/324152) for detail. \n\nPlease see [22nd-place-lgbm-model-single-train](https://www.kaggle.com/code/iwatatakuya/22nd-place-lgbm-model-single-train) for train part.\n\nLocal(9/16~22):0.03559\nPublic:0.03022\nPrivate:0.03038","metadata":{}},{"cell_type":"code","source":"import cudf\nfrom cuml import ForestInference","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-06T23:17:05.604544Z","iopub.execute_input":"2022-05-06T23:17:05.604802Z","iopub.status.idle":"2022-05-06T23:17:09.95638Z","shell.execute_reply.started":"2022-05-06T23:17:05.604731Z","shell.execute_reply":"2022-05-06T23:17:09.955591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/h-and-m-personalized-fashion-recommendations/\"","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:17:09.959006Z","iopub.execute_input":"2022-05-06T23:17:09.959246Z","iopub.status.idle":"2022-05-06T23:17:09.963726Z","shell.execute_reply.started":"2022-05-06T23:17:09.959212Z","shell.execute_reply":"2022-05-06T23:17:09.963036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport gc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nimport datetime\nimport itertools\nimport os\nfrom contextlib import redirect_stdout\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:17:09.968561Z","iopub.execute_input":"2022-05-06T23:17:09.969282Z","iopub.status.idle":"2022-05-06T23:17:10.804988Z","shell.execute_reply.started":"2022-05-06T23:17:09.969245Z","shell.execute_reply":"2022-05-06T23:17:10.804229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand = 64\nlgb_params = {}\nlgb_params[\"class\"] = {\n    \"objective\": \"binary\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"subsample\": 0.8,\n    \"subsample_freq\": 1,\n    \"bagging_seed\": rand,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 100,\n    \"lambda_l1\": 0,\n    \"lambda_l2\": 0,\n    \"random_state\": rand,\n    \"metric\": \"auc\",#\"binary_logloss\",\n    \"verbose\": -1\n}\n\nlgb_params[\"rank\"] = {\n    \"objective\": \"lambdarank\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"subsample\": 0.8,\n    \"subsample_freq\": 1,\n    \"bagging_seed\": rand,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 100,\n    \"lambda_l1\": 0,\n    \"lambda_l2\": 0,\n    \"random_state\": rand,\n    \"metric\": \"map\",\n    \"eval_at\": 12,\n    \"verbose\": -1\n}","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:17:10.806334Z","iopub.execute_input":"2022-05-06T23:17:10.806631Z","iopub.status.idle":"2022-05-06T23:17:10.813248Z","shell.execute_reply.started":"2022-05-06T23:17:10.806594Z","shell.execute_reply":"2022-05-06T23:17:10.812575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tran_dtypes = {\"t_dat\":\"str\",\n               \"customer_id\":\"str\",\n               \"article_id\":\"int\",\n               \"product_code\":\"int\",\n               \"price\":\"float\",\n               \"sales_channel_id\":\"int\"}\nart_dtypes = {\"article_id\":\"int\",\n              \"product_code\":\"int\",\n              \"product_type_no\":\"int\",\n              \"graphical_appearance_no\":\"int\",\n              \"colour_group_code\":\"int\",\n              \"department_no\":\"int\",\n              \"index_code\":\"str\",\n              \"index_group_no\":\"int\",\n              \"section_no\":\"int\",\n              \"garment_group_no\":\"int\"}\ncust_dtypes = {\"customer_id\":\"str\"}\n\nobj = \"class\" # \"class\" or \"rank\"\nN = 15000\nN_div = 20\nn_iter = 2 # num of iteration\nidx_file = \"exp05\"\nlen_hist = 366\nn_round = 4000\nn_splits = 1\ntmp_top = 200\ntr_set = [1,8,15,22] # set of train date\nlen_tr = 7 # length of validation period\nnobuy = 20 # num of negative samples","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:17:10.814604Z","iopub.execute_input":"2022-05-06T23:17:10.814993Z","iopub.status.idle":"2022-05-06T23:17:10.825873Z","shell.execute_reply.started":"2022-05-06T23:17:10.814957Z","shell.execute_reply":"2022-05-06T23:17:10.8251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_trans = pd.read_csv(path+\"transactions_train.csv\",dtype=tran_dtypes)\ndf_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"],format=\"%Y-%m-%d\")\ndf_trans = df_trans.drop_duplicates([\"customer_id\",\"article_id\",\"t_dat\"])\n\nday_end_valtmp = df_trans[\"t_dat\"].max()\nday_start_valtmp = day_end_valtmp - datetime.timedelta(days=6)  \ndf_trans_val_1 = df_trans.query(\"(t_dat >= @day_start_valtmp) and (t_dat <= @day_end_valtmp)\").copy()\ndf_trans_val_1[\"article_id\"] = df_trans_val_1[\"article_id\"].astype(str).str.zfill(10)\ndf_agg_val_1 = df_trans_val_1.groupby(\"customer_id\")[\"article_id\"].apply(list).reset_index()\ndf_agg_val_1 = df_agg_val_1[df_agg_val_1[\"article_id\"].apply(len) != 0]\n\ndel df_trans, df_trans_val_1\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:17:10.827299Z","iopub.execute_input":"2022-05-06T23:17:10.82773Z","iopub.status.idle":"2022-05-06T23:18:33.410863Z","shell.execute_reply.started":"2022-05-06T23:17:10.827695Z","shell.execute_reply":"2022-05-06T23:18:33.410192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(day_oldest):\n    df_art = pd.read_csv(path+\"articles.csv\",dtype=art_dtypes)\n    le = LabelEncoder()\n    le.fit(df_art[\"index_code\"].unique())\n    df_art[\"index_code\"] = le.transform(df_art[\"index_code\"])\n#     display(df_art[\"index_code\"].unique())\n\n    df_cust = pd.read_csv(path+\"customers.csv\",dtype=cust_dtypes)\n    df_cust[\"age\"] = df_cust[\"age\"].fillna(df_cust[\"age\"].mean())\n    df_cust[[\"FN\",\"Active\"]] = df_cust[[\"FN\",\"Active\"]].fillna(0)\n    df_cust[\"club_member_status\"] = df_cust[\"club_member_status\"].apply(lambda x:1 if x == \"ACTIVE\" else 0)\n    df_cust[\"fashion_news_frequency\"] = df_cust[\"fashion_news_frequency\"].apply(lambda x:0 if x == \"NONE\" else 1)\n\n    df_trans = pd.read_csv(path+\"transactions_train.csv\",dtype=tran_dtypes)\n    df_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"],format=\"%Y-%m-%d\")\n    df_trans = df_trans.query(f\"t_dat >= '{day_oldest}'\").copy()\n    df_trans = df_trans.drop_duplicates([\"customer_id\",\"article_id\",\"t_dat\"])\n    df_trans = df_trans.merge(df_art[[\"article_id\",\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",on=\"article_id\")\n    df_trans = df_trans.merge(df_cust[[\"customer_id\",\"age\"]],how=\"left\",on=\"customer_id\")\n    \n    dict_vec = {}\n    vec_art = np.load(\"../input/h-m-rapids-article2vec/articles.npy\")\n    df_vec = pd.concat([df_art[\"article_id\"],pd.DataFrame(vec_art)],axis=1)\n    for i in range(len(vec_art)):\n        dict_vec[df_art[\"article_id\"][i]] = vec_art[i]\n    del vec_art,df_vec\n\n    df_art = df_art.set_index(\"article_id\")\n    df_cust = df_cust.set_index(\"customer_id\")\n\n    return df_trans,cudf.from_pandas(df_art),cudf.from_pandas(df_cust),dict_vec","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:33.412247Z","iopub.execute_input":"2022-05-06T23:18:33.412518Z","iopub.status.idle":"2022-05-06T23:18:33.425093Z","shell.execute_reply.started":"2022-05-06T23:18:33.412483Z","shell.execute_reply":"2022-05-06T23:18:33.424157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_store(df_trans,l_cust,ds,de,dsr,der,dsh,deh):\n    feat ={}\n\n    df_trans_yesterday = df_trans.query(\"(t_dat == @der)\")\n    df_trans_recent = df_trans.query(\"(t_dat >= @dsr) and (t_dat <= @der)\")\n    df_trans_hist = df_trans.query(\"(t_dat >= @dsh) and (t_dat <= @deh)\")\n\n    feat[\"art_buy_hist\"] = df_trans_hist.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_hist=\"count\")\n    feat[\"art_buy_recent\"] = df_trans_recent.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_recent=\"count\")\n    feat[\"art_buy_yesterday\"] = df_trans_yesterday.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_yesterday=\"count\")\n    df_buy1 = df_trans_hist.groupby(\"article_id\")[\"customer_id\"].nunique().reset_index().rename(columns={\"customer_id\":\"cnt_buy1\"})\n    df_buy2 = df_trans_hist[df_trans_hist.duplicated([\"customer_id\",\"article_id\"])].copy()\n    df_buy2 = df_buy2.drop_duplicates([\"customer_id\",\"article_id\"])\n    df_buy2 = df_buy2.groupby(\"article_id\")[\"article_id\"].agg(cnt_buy2='count').reset_index()\n    df_buy = pd.merge(df_buy1,df_buy2,how=\"left\",on=\"article_id\").fillna(0)\n    df_buy[\"rebuy_rate\"] = df_buy[\"cnt_buy2\"]/df_buy[\"cnt_buy1\"]\n    feat[\"rebuy_rate\"] = df_buy[[\"article_id\",\"rebuy_rate\"]]\n\n    df_trans_yesterday = df_trans_yesterday.query(\"(customer_id in @l_cust)\")\n    df_trans_recent = df_trans_recent.query(\"(customer_id in @l_cust)\")\n    df_trans_hist = df_trans_hist.query(\"(customer_id in @l_cust)\")\n    feat[\"rate_sales_channel_hist\"] = df_trans_hist.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_hist=\"mean\")\n    feat[\"rate_sales_channel_recent\"] = df_trans_recent.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_recent=\"mean\")\n    feat[\"n_buy_hist\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_hist=\"count\")\n    feat[\"n_buy_recent\"] = df_trans_recent.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_recent=\"count\")\n    feat[\"days_after_buy\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(days_after_buy=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_hist_all=\"count\")\n    feat[\"n_buy_recent_all\"] = df_trans_recent.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_recent_all=\"count\")\n    feat[\"days_after_buy_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(days_after_buy_all=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_hist_prod=\"count\")\n    feat[\"n_buy_recent_prod\"] = df_trans_recent.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_recent_prod=\"count\")\n    feat[\"days_after_buy_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(days_after_buy_prod=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_hist_ptype=\"count\")\n    feat[\"n_buy_recent_ptype\"] = df_trans_recent.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_recent_ptype=\"count\")\n    feat[\"days_after_buy_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(days_after_buy_ptype=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_hist_graph=\"count\")\n    feat[\"n_buy_recent_graph\"] = df_trans_recent.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_recent_graph=\"count\")\n    feat[\"days_after_buy_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(days_after_buy_graph=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_hist_col=\"count\")\n    feat[\"n_buy_recent_col\"] = df_trans_recent.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_recent_col=\"count\")\n    feat[\"days_after_buy_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(days_after_buy_col=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_hist_dep=\"count\")\n    feat[\"n_buy_recent_dep\"] = df_trans_recent.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_recent_dep=\"count\")\n    feat[\"days_after_buy_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(days_after_buy_dep=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_hist_idx=\"count\")\n    feat[\"n_buy_recent_idx\"] = df_trans_recent.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_recent_idx=\"count\")\n    feat[\"days_after_buy_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(days_after_buy_idx=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_hist_idxg=\"count\")\n    feat[\"n_buy_recent_idxg\"] = df_trans_recent.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_recent_idxg=\"count\")\n    feat[\"days_after_buy_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(days_after_buy_idxg=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_hist_sec=\"count\")\n    feat[\"n_buy_recent_sec\"] = df_trans_recent.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_recent_sec=\"count\")\n    feat[\"days_after_buy_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(days_after_buy_sec=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_hist_garm=\"count\")\n    feat[\"n_buy_recent_garm\"] = df_trans_recent.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_recent_garm=\"count\")\n    feat[\"days_after_buy_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(days_after_buy_garm=lambda x:(ds - max(x)).days)\n    feat[\"art_id_recent\"] = df_trans_recent.groupby(\"customer_id\")[\"article_id\"].apply(list).rename(\"art_id_recent\")\n\n    for k in feat.keys():\n        feat[k] = cudf.from_pandas(feat[k])\n\n    del df_trans_yesterday, df_trans_recent, df_trans_hist, df_buy1, df_buy2, df_buy\n    gc.collect()\n\n    return feat","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:33.426868Z","iopub.execute_input":"2022-05-06T23:18:33.427189Z","iopub.status.idle":"2022-05-06T23:18:33.485069Z","shell.execute_reply.started":"2022-05-06T23:18:33.427151Z","shell.execute_reply":"2022-05-06T23:18:33.484452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_feat(df,ds,de,dsr,der,dsh,deh,feat,dict_vec):\n  # rate_sales_channel_hist\n  df = df.merge(feat[\"rate_sales_channel_hist\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # rate_sales_channel_recent\n  df = df.merge(feat[\"rate_sales_channel_recent\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)  \n  # art_buy_hist\n  df = df.merge(feat[\"art_buy_hist\"],how=\"left\",left_on=[\"article_id\"], right_index=True)\n  # art_buy_recent\n  df = df.merge(feat[\"art_buy_recent\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # art_buy_yesterday\n  df = df.merge(feat[\"art_buy_yesterday\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # n_buy_hist\n  df = df.merge(feat[\"n_buy_hist\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_recent\n  df = df.merge(feat[\"n_buy_recent\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # days_after_buy\n  df = df.merge(feat[\"days_after_buy\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_hist_all\n  df = df.merge(feat[\"n_buy_hist_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_recent_all\n  df = df.merge(feat[\"n_buy_recent_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # days_after_buy_all\n  df = df.merge(feat[\"days_after_buy_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_hist_prod\n  df = df.merge(feat[\"n_buy_hist_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_recent_prod\n  df = df.merge(feat[\"n_buy_recent_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # days_after_buy_prod\n  df = df.merge(feat[\"days_after_buy_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_hist_ptype\n  df = df.merge(feat[\"n_buy_hist_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_recent_ptype\n  df = df.merge(feat[\"n_buy_recent_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # days_after_buy_ptype\n  df = df.merge(feat[\"days_after_buy_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_hist_graph\n  df = df.merge(feat[\"n_buy_hist_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_recent_graph\n  df = df.merge(feat[\"n_buy_recent_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # days_after_buy_graph\n  df = df.merge(feat[\"days_after_buy_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_hist_col\n  df = df.merge(feat[\"n_buy_hist_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_recent_col\n  df = df.merge(feat[\"n_buy_recent_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # days_after_buy_col\n  df = df.merge(feat[\"days_after_buy_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_hist_dep\n  df = df.merge(feat[\"n_buy_hist_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_recent_dep\n  df = df.merge(feat[\"n_buy_recent_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # days_after_buy_dep\n  df = df.merge(feat[\"days_after_buy_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_hist_idx\n  df = df.merge(feat[\"n_buy_hist_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_recent_idx\n  df = df.merge(feat[\"n_buy_recent_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # days_after_buy_idx\n  df = df.merge(feat[\"days_after_buy_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_hist_idxg\n  df = df.merge(feat[\"n_buy_hist_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_recent_idxg\n  df = df.merge(feat[\"n_buy_recent_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # days_after_buy_idxg\n  df = df.merge(feat[\"days_after_buy_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_hist_sec\n  df = df.merge(feat[\"n_buy_hist_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_recent_sec\n  df = df.merge(feat[\"n_buy_recent_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # days_after_buy_sec\n  df = df.merge(feat[\"days_after_buy_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_hist_garm\n  df = df.merge(feat[\"n_buy_hist_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # n_buy_recent_garm\n  df = df.merge(feat[\"n_buy_recent_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # days_after_buy_garm\n  df = df.merge(feat[\"days_after_buy_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # rebuy_rate\n  df = df.merge(feat[\"rebuy_rate\"],how=\"left\",on=\"article_id\")\n  # sim_article\n  df = df.merge(feat[\"art_id_recent\"],how=\"left\",left_on=\"customer_id\", right_index = True)\n  sim_max,sim_sum,sim_mean = [],[],[]\n  # display(df[[\"article_id\",\"art_id_recent\"]].head())\n  tmp = df[[\"article_id\",\"art_id_recent\"]].to_pandas().values\n  for i in range(len(df)):\n      if not isinstance(tmp[i][1],list):\n        sim_max.append(0);sim_sum.append(0);sim_mean.append(0)\n      else:\n        list_sim = [cos_sim(dict_vec[tmp[i][0]],dict_vec[x]) for x in tmp[i][1]]\n        sim_max.append(max(list_sim))\n        sim_sum.append(sum(list_sim))\n        sim_mean.append(np.mean(list_sim))\n  df[\"sim_max\"] = sim_max\n  df[\"sim_sum\"] = sim_sum\n  df[\"sim_mean\"] = sim_mean\n  df = df.drop([\"art_id_recent\"], axis = 1)\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]] =\\\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]].fillna(0)\n\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]] = \\\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]].fillna(10+len_hist)\n\n  df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]] = df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]].fillna(1.5)\n  \n  return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:33.488285Z","iopub.execute_input":"2022-05-06T23:18:33.488572Z","iopub.status.idle":"2022-05-06T23:18:33.540967Z","shell.execute_reply.started":"2022-05-06T23:18:33.48853Z","shell.execute_reply":"2022-05-06T23:18:33.540236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend_pred(series_cust,day_start_val,submit=False,sub_no = 0):\n    day_start = [day_start_val - datetime.timedelta(days=i-1+len_tr) for i in tr_set]\n    day_end = [day_start_val - datetime.timedelta(days=i) for i in tr_set]\n    day_start_rec = [x - datetime.timedelta(days=7) for x in day_start]\n    day_end_rec = [x - datetime.timedelta(days=1) for x in day_start]\n    day_start_hist = [x - datetime.timedelta(days=len_hist) for x in day_start]\n    day_end_hist = [x - datetime.timedelta(days=1) for x in day_start]\n    day_start_rec_test = day_start_val - datetime.timedelta(days=7)\n    day_end_rec_test = day_start_val - datetime.timedelta(days=1)\n    day_start_hist_test = day_start_val - datetime.timedelta(days=1+len_hist)\n    day_end_hist_test = day_start_val - datetime.timedelta(days=1)\n\n    day_end_val = day_start_val + datetime.timedelta(days=6)\n\n    df_trans, df_art, df_cust,  dict_vec = read_data(day_oldest = day_start_hist[-1])\n    \n    q_date = \"\"\n    for i in range(len(day_start)):\n      if i == 0: q_date = f\"((t_dat >= '{day_start[0]}') and (t_dat <= '{day_end[0]}'))\"\n      else: q_date = q_date + f\" or ((t_dat >= '{day_start[i]}') and (t_dat <= '{day_end[i]}'))\"\n    top_art_all = df_trans.query(q_date).groupby(\"article_id\")[\"t_dat\"].count().sort_values(ascending = False).index[:N].tolist()\n\n    list_sl = list(range(0,N,N_div))\n    if list_sl[-1] != N:list_sl.append(N)\n    df_ans = pd.DataFrame()\n\n    feat = feat_store(df_trans,series_cust.tolist(),day_start_val,day_end_val,day_start_rec_test,day_end_rec_test,day_start_hist_test,day_end_hist_test)\n    del df_trans\n\n    for iter_art in tqdm(range(len(list_sl)-1)):\n      top_art = top_art_all[list_sl[iter_art]:list_sl[iter_art+1]]\n      df_test = cudf.from_pandas(pd.DataFrame(itertools.product(series_cust.tolist(),top_art),columns=[\"customer_id\",\"article_id\"]))\n      df_test = df_test.merge(df_art[[\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",left_on=\"article_id\",right_index=True)\n      df_test = df_test.merge(df_cust[[\"age\",\"FN\",\"Active\",\"club_member_status\",\"fashion_news_frequency\"]],how=\"left\",left_on=\"customer_id\",right_index=True)\n\n      df_test = add_feat(df_test,day_start_val,day_end_val,day_start_rec_test,day_end_rec_test,day_start_hist_test,day_end_hist_test,feat,dict_vec)\n\n      df_pred = df_test[[\"customer_id\",\"article_id\"]].copy()\n      df_test = df_test.drop([\"customer_id\",\"product_code\",\"product_type_no\",\"department_no\"],axis=1)\n      pred = np.zeros(len(df_pred))\n      for iter_train in range(n_iter):\n        list_model = pd.read_pickle(f\"../input/hmmodel/{iter_train}_models_{idx_file}_{day_start_val.date()}.pkl\")\n        for i in range(max(1,n_splits)):\n          list_model[i].save_model(f\"lgbm_{idx_file}.model\")\n          if obj == \"rank\":\n            with redirect_stdout(open(os.devnull, 'w')):\n              fm = ForestInference.load(filename=f\"lgbm_{idx_file}.model\",model_type='lightgbm')\n            pred += fm.predict(df_test) / (max(1,n_splits) * n_iter)\n          else:\n            with redirect_stdout(open(os.devnull, 'w')):\n              fm = ForestInference.load(filename=f\"lgbm_{idx_file}.model\",output_class=True,model_type='lightgbm')\n            pred += fm.predict_proba(df_test)[:,1] / (max(1,n_splits) * n_iter)      \n      df_pred[\"pred\"] = pred\n\n      df_ans = df_ans.append(df_pred)\n      df_ans = df_ans.sort_values([\"customer_id\",\"pred\"],ascending = False)\n      df_ans = df_ans.groupby(\"customer_id\").head(tmp_top)\n      del list_model,df_test,df_pred,pred\n      gc.collect()\n        \n    df_ans[\"article_id\"] = df_ans[\"article_id\"].astype(str).str.zfill(10)\n    if submit: df_ans.groupby(\"customer_id\").head(tmp_top).to_csv(f\"oof_{idx_file}_{day_start_val.date()}_{sub_no}.csv\",index = False)\n    else: df_ans.groupby(\"customer_id\").head(tmp_top).to_csv(path+f\"output/oof_{idx_file}_{day_start_val.date()}.csv\",index = False)\n    df_ans = df_ans.groupby(\"customer_id\").head(12)\n    if not submit:\n      mapk_val = mapk(df_agg_val_1[\"article_id\"].tolist(),df_ans.groupby(\"customer_id\")[\"article_id\"].apply(list).tolist())\n      print(f\"mapk:{mapk_val} \")\n    df_ans = df_ans.groupby(\"customer_id\")[\"article_id\"].apply(list).reset_index()\n    df_ans = df_ans.rename({'article_id':'pred'},axis=1)\n    gc.collect()\n    return df_ans","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:33.542442Z","iopub.execute_input":"2022-05-06T23:18:33.543022Z","iopub.status.idle":"2022-05-06T23:18:33.569047Z","shell.execute_reply.started":"2022-05-06T23:18:33.542986Z","shell.execute_reply":"2022-05-06T23:18:33.568274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(path+\"sample_submission.csv\")\ndf_sub.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:18:56.294893Z","iopub.execute_input":"2022-05-06T23:18:56.295378Z","iopub.status.idle":"2022-05-06T23:19:00.466612Z","shell.execute_reply.started":"2022-05-06T23:18:56.295324Z","shell.execute_reply":"2022-05-06T23:19:00.465905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nsize_block = 50000\nlist_slice = list(range(0,len(df_sub),size_block))\nif list_slice[-1] != len(df_sub):list_slice.append(len(df_sub))\n# os.makedirs(path+f\"output/div{idx_file}\",exist_ok = True)\nfor i in tqdm(range(len(list_slice)-1)):\n  time.sleep(1)\n  if not os.path.exists(f\"submission_{i}.csv\"):\n    df_sub_0 = df_sub[list_slice[i]:list_slice[i+1]].copy()\n    df_ans = recommend_pred(df_sub_0[\"customer_id\"],day_end_valtmp+datetime.timedelta(days=1),submit=True,sub_no = i)\n    df_ans[\"prediction\"] = df_ans[\"pred\"].apply(lambda x:' '.join(x))\n    df_ans[[\"customer_id\",\"prediction\"]].to_csv(f\"submission_{i}.csv\",index = False)\n    del df_sub_0,df_ans\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-06T23:19:10.0732Z","iopub.execute_input":"2022-05-06T23:19:10.073779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}