{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook shows a part of Nth solution.\n\nPlease see [Part of 22nd solution - single LGBM (Private:0.03038)](https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/324152) for detail. \n\nPlease see [22nd-place-lgbm-model-single-infer](https://www.kaggle.com/code/iwatatakuya/22nd-place-lgbm-model-single-infer) for inference part.\n\nLocal(9/16~22):0.03559\nPublic:0.03022\nPrivate:0.03038","metadata":{}},{"cell_type":"code","source":"path = \"../input/h-and-m-personalized-fashion-recommendations/\"","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.402553Z","iopub.execute_input":"2022-05-10T09:06:32.40288Z","iopub.status.idle":"2022-05-10T09:06:32.408178Z","shell.execute_reply.started":"2022-05-10T09:06:32.402847Z","shell.execute_reply":"2022-05-10T09:06:32.406951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport gc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nimport datetime\nimport itertools\nimport os\nfrom contextlib import redirect_stdout\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.437746Z","iopub.execute_input":"2022-05-10T09:06:32.438228Z","iopub.status.idle":"2022-05-10T09:06:32.444597Z","shell.execute_reply.started":"2022-05-10T09:06:32.438195Z","shell.execute_reply":"2022-05-10T09:06:32.443878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand = 64\nlgb_params = {\n    \"objective\": \"binary\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"subsample\": 0.8,\n    \"subsample_freq\": 1,\n    \"bagging_seed\": rand,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 100,\n    \"lambda_l1\": 0,\n    \"lambda_l2\": 0,\n    \"random_state\": rand,\n    \"metric\": \"auc\",#\"binary_logloss\",\n    \"verbose\": -1\n}\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.469946Z","iopub.execute_input":"2022-05-10T09:06:32.470275Z","iopub.status.idle":"2022-05-10T09:06:32.477434Z","shell.execute_reply.started":"2022-05-10T09:06:32.470243Z","shell.execute_reply":"2022-05-10T09:06:32.476104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tran_dtypes = {\"t_dat\":\"str\",\n               \"customer_id\":\"str\",\n               \"article_id\":\"int\",\n               \"product_code\":\"int\",\n               \"price\":\"float\",\n               \"sales_channel_id\":\"int\"}\nart_dtypes = {\"article_id\":\"int\",\n              \"product_code\":\"int\",\n              \"product_type_no\":\"int\",\n              \"graphical_appearance_no\":\"int\",\n              \"colour_group_code\":\"int\",\n              \"department_no\":\"int\",\n              \"index_code\":\"str\",\n              \"index_group_no\":\"int\",\n              \"section_no\":\"int\",\n              \"garment_group_no\":\"int\"}\ncust_dtypes = {\"customer_id\":\"str\"}\n\nobj = \"class\" # \"class\" or \"rank\"\nN = 15000\nn_iter = 2 # num of iteration\nidx_file = \"exp05\"\nlen_hist = 366\nn_round = 2000\nn_splits = 1\ntr_set = [1,8,15,22] # set of train date\nlen_tr = 7 # length of validation period\nnobuy = 20 # num of negative samples","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.52506Z","iopub.execute_input":"2022-05-10T09:06:32.525537Z","iopub.status.idle":"2022-05-10T09:06:32.535225Z","shell.execute_reply.started":"2022-05-10T09:06:32.525487Z","shell.execute_reply":"2022-05-10T09:06:32.534214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cos_sim(v1, v2):\n    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.572463Z","iopub.execute_input":"2022-05-10T09:06:32.573052Z","iopub.status.idle":"2022-05-10T09:06:32.57865Z","shell.execute_reply.started":"2022-05-10T09:06:32.57299Z","shell.execute_reply":"2022-05-10T09:06:32.578055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(day_oldest):\n    df_art = pd.read_csv(path+\"articles.csv\",dtype=art_dtypes)\n    le = LabelEncoder()\n    le.fit(df_art[\"index_code\"].unique())\n    df_art[\"index_code\"] = le.transform(df_art[\"index_code\"])\n\n    df_trans = pd.read_csv(path+\"transactions_train.csv\",dtype=tran_dtypes)\n    df_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"],format=\"%Y-%m-%d\")\n\n    df_trans = df_trans.query(f\"t_dat >= '{day_oldest}'\").copy()\n    df_trans = df_trans.drop_duplicates([\"customer_id\",\"article_id\",\"t_dat\"])\n    df_trans = df_trans.merge(df_art[[\"article_id\",\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",on=\"article_id\")\n\n    df_cust = pd.read_csv(path+\"customers.csv\",dtype=cust_dtypes)\n    df_cust[\"age\"] = df_cust[\"age\"].fillna(df_cust[\"age\"].mean())\n    df_cust[[\"FN\",\"Active\"]] = df_cust[[\"FN\",\"Active\"]].fillna(0)\n    df_cust[\"club_member_status\"] = df_cust[\"club_member_status\"].apply(lambda x:1 if x == \"ACTIVE\" else 0)\n    df_cust[\"fashion_news_frequency\"] = df_cust[\"fashion_news_frequency\"].apply(lambda x:0 if x == \"NONE\" else 1)\n\n    dict_vec = {}\n    vec_art = np.load(\"../input/h-m-rapids-article2vec/articles.npy\")\n    df_vec = pd.concat([df_art[\"article_id\"],pd.DataFrame(vec_art)],axis=1)\n    for i in range(len(vec_art)):\n        dict_vec[df_art[\"article_id\"][i]] = vec_art[i]\n    del vec_art,df_vec\n\n\n    return df_trans,df_art,df_cust, dict_vec","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.619032Z","iopub.execute_input":"2022-05-10T09:06:32.6196Z","iopub.status.idle":"2022-05-10T09:06:32.634709Z","shell.execute_reply.started":"2022-05-10T09:06:32.619566Z","shell.execute_reply":"2022-05-10T09:06:32.633868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_store(df_trans,l_cust,ds,de,dsr,der,dsh,deh):\n    feat ={}\n\n    df_trans_yesterday = df_trans.query(\"(t_dat == @der)\")\n    df_trans_recent = df_trans.query(\"(t_dat >= @dsr) and (t_dat <= @der)\")\n    df_trans_hist = df_trans.query(\"(t_dat >= @dsh) and (t_dat <= @deh)\")\n\n    feat[\"art_buy_hist\"] = df_trans_hist.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_hist=\"count\")\n    feat[\"art_buy_recent\"] = df_trans_recent.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_recent=\"count\")\n    feat[\"art_buy_yesterday\"] = df_trans_yesterday.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_yesterday=\"count\")\n    df_buy1 = df_trans_hist.groupby(\"article_id\")[\"customer_id\"].nunique().reset_index().rename(columns={\"customer_id\":\"cnt_buy1\"})\n    df_buy2 = df_trans_hist[df_trans_hist.duplicated([\"customer_id\",\"article_id\"])].copy()\n    df_buy2 = df_buy2.drop_duplicates([\"customer_id\",\"article_id\"])\n    df_buy2 = df_buy2.groupby(\"article_id\")[\"article_id\"].agg(cnt_buy2='count').reset_index()\n    df_buy = pd.merge(df_buy1,df_buy2,how=\"left\",on=\"article_id\").fillna(0)\n    df_buy[\"rebuy_rate\"] = df_buy[\"cnt_buy2\"]/df_buy[\"cnt_buy1\"]\n    feat[\"rebuy_rate\"] = df_buy[[\"article_id\",\"rebuy_rate\"]]\n\n    df_trans_yesterday = df_trans_yesterday.query(\"(customer_id in @l_cust)\")\n    df_trans_recent = df_trans_recent.query(\"(customer_id in @l_cust)\")\n    df_trans_hist = df_trans_hist.query(\"(customer_id in @l_cust)\")\n    feat[\"rate_sales_channel_hist\"] = df_trans_hist.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_hist=\"mean\")\n    feat[\"rate_sales_channel_recent\"] = df_trans_recent.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_recent=\"mean\")\n    feat[\"n_buy_hist\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_hist=\"count\")\n    feat[\"n_buy_recent\"] = df_trans_recent.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_recent=\"count\")\n    feat[\"days_after_buy\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(days_after_buy=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_hist_all=\"count\")\n    feat[\"n_buy_recent_all\"] = df_trans_recent.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_recent_all=\"count\")\n    feat[\"days_after_buy_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(days_after_buy_all=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_hist_prod=\"count\")\n    feat[\"n_buy_recent_prod\"] = df_trans_recent.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_recent_prod=\"count\")\n    feat[\"days_after_buy_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(days_after_buy_prod=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_hist_ptype=\"count\")\n    feat[\"n_buy_recent_ptype\"] = df_trans_recent.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_recent_ptype=\"count\")\n    feat[\"days_after_buy_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(days_after_buy_ptype=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_hist_graph=\"count\")\n    feat[\"n_buy_recent_graph\"] = df_trans_recent.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_recent_graph=\"count\")\n    feat[\"days_after_buy_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(days_after_buy_graph=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_hist_col=\"count\")\n    feat[\"n_buy_recent_col\"] = df_trans_recent.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_recent_col=\"count\")\n    feat[\"days_after_buy_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(days_after_buy_col=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_hist_dep=\"count\")\n    feat[\"n_buy_recent_dep\"] = df_trans_recent.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_recent_dep=\"count\")\n    feat[\"days_after_buy_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(days_after_buy_dep=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_hist_idx=\"count\")\n    feat[\"n_buy_recent_idx\"] = df_trans_recent.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_recent_idx=\"count\")\n    feat[\"days_after_buy_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(days_after_buy_idx=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_hist_idxg=\"count\")\n    feat[\"n_buy_recent_idxg\"] = df_trans_recent.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_recent_idxg=\"count\")\n    feat[\"days_after_buy_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(days_after_buy_idxg=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_hist_sec=\"count\")\n    feat[\"n_buy_recent_sec\"] = df_trans_recent.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_recent_sec=\"count\")\n    feat[\"days_after_buy_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(days_after_buy_sec=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_hist_garm=\"count\")\n    feat[\"n_buy_recent_garm\"] = df_trans_recent.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_recent_garm=\"count\")\n    feat[\"days_after_buy_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(days_after_buy_garm=lambda x:(ds - max(x)).days)\n    feat[\"art_id_recent\"] = df_trans_recent.groupby(\"customer_id\")[\"article_id\"].apply(list).rename(\"art_id_recent\")\n\n    del df_trans_yesterday, df_trans_recent, df_trans_hist, df_buy1, df_buy2, df_buy\n    gc.collect()\n\n    return feat","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.685906Z","iopub.execute_input":"2022-05-10T09:06:32.686471Z","iopub.status.idle":"2022-05-10T09:06:32.725624Z","shell.execute_reply.started":"2022-05-10T09:06:32.686436Z","shell.execute_reply":"2022-05-10T09:06:32.724906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_feat(df,ds,de,dsr,der,dsh,deh,feat,dict_vec):\n  # rate_sales_channel_hist\n  df = df.merge(feat[\"rate_sales_channel_hist\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # rate_sales_channel_recent\n  df = df.merge(feat[\"rate_sales_channel_recent\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)  \n  # art_buy_hist\n  df = df.merge(feat[\"art_buy_hist\"],how=\"left\",left_on=[\"article_id\"], right_index=True)\n  # art_buy_recent\n  df = df.merge(feat[\"art_buy_recent\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # art_buy_yesterday\n  df = df.merge(feat[\"art_buy_yesterday\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # n_buy_hist\n  df = df.merge(feat[\"n_buy_hist\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_recent\n  df = df.merge(feat[\"n_buy_recent\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # days_after_buy\n  df = df.merge(feat[\"days_after_buy\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_hist_all\n  df = df.merge(feat[\"n_buy_hist_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_recent_all\n  df = df.merge(feat[\"n_buy_recent_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # days_after_buy_all\n  df = df.merge(feat[\"days_after_buy_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_hist_prod\n  df = df.merge(feat[\"n_buy_hist_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_recent_prod\n  df = df.merge(feat[\"n_buy_recent_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # days_after_buy_prod\n  df = df.merge(feat[\"days_after_buy_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_hist_ptype\n  df = df.merge(feat[\"n_buy_hist_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_recent_ptype\n  df = df.merge(feat[\"n_buy_recent_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # days_after_buy_ptype\n  df = df.merge(feat[\"days_after_buy_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_hist_graph\n  df = df.merge(feat[\"n_buy_hist_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_recent_graph\n  df = df.merge(feat[\"n_buy_recent_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # days_after_buy_graph\n  df = df.merge(feat[\"days_after_buy_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_hist_col\n  df = df.merge(feat[\"n_buy_hist_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_recent_col\n  df = df.merge(feat[\"n_buy_recent_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # days_after_buy_col\n  df = df.merge(feat[\"days_after_buy_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_hist_dep\n  df = df.merge(feat[\"n_buy_hist_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_recent_dep\n  df = df.merge(feat[\"n_buy_recent_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # days_after_buy_dep\n  df = df.merge(feat[\"days_after_buy_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_hist_idx\n  df = df.merge(feat[\"n_buy_hist_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_recent_idx\n  df = df.merge(feat[\"n_buy_recent_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # days_after_buy_idx\n  df = df.merge(feat[\"days_after_buy_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_hist_idxg\n  df = df.merge(feat[\"n_buy_hist_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_recent_idxg\n  df = df.merge(feat[\"n_buy_recent_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # days_after_buy_idxg\n  df = df.merge(feat[\"days_after_buy_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_hist_sec\n  df = df.merge(feat[\"n_buy_hist_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_recent_sec\n  df = df.merge(feat[\"n_buy_recent_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # days_after_buy_sec\n  df = df.merge(feat[\"days_after_buy_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_hist_garm\n  df = df.merge(feat[\"n_buy_hist_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # n_buy_recent_garm\n  df = df.merge(feat[\"n_buy_recent_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # days_after_buy_garm\n  df = df.merge(feat[\"days_after_buy_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # rebuy_rate\n  df = df.merge(feat[\"rebuy_rate\"],how=\"left\",on=\"article_id\")\n  # sim_article\n  df = df.merge(feat[\"art_id_recent\"],how=\"left\",left_on=\"customer_id\", right_index = True)\n  sim_max,sim_sum,sim_mean = [],[],[]\n  # display(df[[\"article_id\",\"art_id_recent\"]].head())\n  tmp = df[[\"article_id\",\"art_id_recent\"]].values\n  for i in range(len(df)):\n      if not isinstance(tmp[i][1],list):\n        sim_max.append(0);sim_sum.append(0);sim_mean.append(0)\n      else:\n        list_sim = [cos_sim(dict_vec[tmp[i][0]],dict_vec[x]) for x in tmp[i][1]]\n        sim_max.append(max(list_sim))\n        sim_sum.append(sum(list_sim))\n        sim_mean.append(np.mean(list_sim))\n  df[\"sim_max\"] = sim_max\n  df[\"sim_sum\"] = sim_sum\n  df[\"sim_mean\"] = sim_mean\n  df = df.drop([\"art_id_recent\"], axis = 1)\n  # fillna\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]] =\\\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]].fillna(0)\n\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]] = \\\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]].fillna(10+len_hist)\n\n  df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]] = df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]].fillna(1.5)\n  \n  return df","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.738456Z","iopub.execute_input":"2022-05-10T09:06:32.738761Z","iopub.status.idle":"2022-05-10T09:06:32.784309Z","shell.execute_reply.started":"2022-05-10T09:06:32.738728Z","shell.execute_reply":"2022-05-10T09:06:32.782739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend_train(day_start_val):\n    day_start = [day_start_val - datetime.timedelta(days=i-1+len_tr) for i in tr_set]\n    day_end = [day_start_val - datetime.timedelta(days=i) for i in tr_set]\n    day_start_rec = [x - datetime.timedelta(days=7) for x in day_start]\n    day_end_rec = [x - datetime.timedelta(days=1) for x in day_start]\n    day_start_hist = [x - datetime.timedelta(days=len_hist) for x in day_start]\n    day_end_hist = [x - datetime.timedelta(days=1) for x in day_start]\n    day_start_rec_test = day_start_val - datetime.timedelta(days=7)\n    day_end_rec_test = day_start_val - datetime.timedelta(days=1)\n    day_start_hist_test = day_start_val - datetime.timedelta(days=1+len_hist)\n    day_end_hist_test = day_start_val - datetime.timedelta(days=1)\n    day_end_val = day_start_val + datetime.timedelta(days=6)\n\n    df_trans, df_art, df_cust, dict_vec = read_data(day_oldest = day_start_hist[-1])\n\n    q_date = \"\"\n    for i in range(len(day_start)):\n        if i == 0: q_date = f\"((t_dat >= '{day_start[0]}') and (t_dat <= '{day_end[0]}'))\"\n        else: q_date = q_date + f\" or ((t_dat >= '{day_start[i]}') and (t_dat <= '{day_end[i]}'))\"\n    top_art_all = df_trans.query(q_date).groupby(\"article_id\")[\"t_dat\"].count().sort_values(ascending = False).index[:N].tolist()\n\n    list_df_buy = []\n    list_list_cust =[]\n    # make posivive samples\n    for i in range(len(day_start)):\n        list_df_buy.append(df_trans.query(f\"(t_dat >= '{day_start[i]}') and (t_dat <= '{day_end[i]}') and (article_id in @top_art_all)\").drop_duplicates([\"customer_id\",\"article_id\"])[[\"customer_id\",\"article_id\"]].copy())\n        list_df_buy[i][\"target\"] = 1\n        list_list_cust.append(list_df_buy[i][\"customer_id\"].unique().tolist()) \n    # make negative samples(random pick)\n    for iter_train in tqdm(range(n_iter)):\n        list_df_nobuy = []\n        list_train =[]\n        for i in range(len(day_start)):\n            list_df_nobuy.append(pd.concat([pd.DataFrame({\"customer_id\":x,\"article_id\":random.sample(top_art_all,nobuy)}) for x in list_list_cust[i]]))\n            list_df_nobuy[i][\"target\"] = 0\n            list_train.append(pd.concat([list_df_buy[i],list_df_nobuy[i]]).drop_duplicates([\"customer_id\",\"article_id\"]))\n        del list_df_nobuy\n        display(list_train[0][\"target\"].value_counts())\n\n        # add feature\n        df_train = pd.DataFrame()\n        for i in tqdm(range(len(day_start))):\n            feat = feat_store(df_trans,list_list_cust[i],day_start[i],day_end[i],day_start_rec[i],day_end_rec[i],day_start_hist[i],day_end_hist[i])\n            list_train[i] = list_train[i].merge(df_art[[\"article_id\",\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",on=\"article_id\")\n            list_train[i] = list_train[i].merge(df_cust[[\"customer_id\",\"age\",\"FN\",\"Active\",\"club_member_status\",\"fashion_news_frequency\"]],how=\"left\",on=\"customer_id\")\n            df_train = df_train.append(add_feat(list_train[i],day_start[i],day_end[i],day_start_rec[i],day_end_rec[i],day_start_hist[i],day_end_hist[i],feat,dict_vec))\n            del feat\n        del list_train\n        gc.collect()\n\n        # train lgbm\n        X_train = df_train.drop([\"customer_id\",\"product_code\",\"product_type_no\",\"department_no\",\"target\"],axis=1)\n        y_train = df_train[\"target\"]\n        del df_train\n\n        list_model = []\n        if n_splits == 1:\n            X_tr, X_va, y_tr, y_va = train_test_split(X_train,y_train,stratify = y_train)\n            d_tr = lgb.Dataset(X_tr, label=y_tr,  free_raw_data=False)\n            d_va = lgb.Dataset(X_va, label=y_va,  free_raw_data=False)\n            list_model.append(lgb.train(lgb_params, train_set=d_tr, num_boost_round=n_round, valid_sets=[d_tr,d_va], verbose_eval=500, early_stopping_rounds=100))\n        else:\n            folds = StratifiedKFold(n_splits = n_splits, shuffle = True, random_state = rand)\n            for tr_idx,va_idx in folds.split(X_train,y_train):\n                X_tr, X_va, y_tr, y_va = X_train.iloc[tr_idx], X_train.iloc[va_idx], y_train.iloc[tr_idx], y_train.iloc[va_idx] \n                d_tr = lgb.Dataset(X_tr, label=y_tr,  free_raw_data=False)\n                d_va = lgb.Dataset(X_va, label=y_va,  free_raw_data=False)\n                list_model.append(lgb.train(lgb_params, train_set=d_tr, num_boost_round=n_round, valid_sets=[d_tr,d_va], verbose_eval=500, early_stopping_rounds=100))\n        # save model\n        pd.to_pickle(list_model,f\"models_{iter_train}.pkl\")\n        del X_train, y_train, X_tr, X_va, y_tr, y_va, d_tr, d_va\n        gc.collect()\n    del df_trans, df_art, df_cust\n    gc.collect()\n    return 0","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.805186Z","iopub.execute_input":"2022-05-10T09:06:32.805694Z","iopub.status.idle":"2022-05-10T09:06:32.836106Z","shell.execute_reply.started":"2022-05-10T09:06:32.805642Z","shell.execute_reply":"2022-05-10T09:06:32.835174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommend_train(datetime.datetime(2020,9,23))","metadata":{"execution":{"iopub.status.busy":"2022-05-10T09:06:32.83789Z","iopub.execute_input":"2022-05-10T09:06:32.838152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}