{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"Path =  \"/kaggle/input/h-and-m-personalized-fashion-recommendations/\"\n\narticles_path = Path + \"articles.csv\"\ncustomers_path = Path + \"customers.csv\"\ntransactions_path = Path + \"transactions_train.csv\"\narticle2vec_path =\"/kaggle/input/h-m-rapids-article2vec/articles.npy\"","metadata":{"execution":{"iopub.status.busy":"2022-12-27T02:46:39.916321Z","iopub.execute_input":"2022-12-27T02:46:39.918269Z","iopub.status.idle":"2022-12-27T02:46:39.940331Z","shell.execute_reply.started":"2022-12-27T02:46:39.918121Z","shell.execute_reply":"2022-12-27T02:46:39.939458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport gc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nimport datetime\nimport itertools\nimport os\nfrom contextlib import redirect_stdout\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-12-27T02:46:41.968512Z","iopub.execute_input":"2022-12-27T02:46:41.969046Z","iopub.status.idle":"2022-12-27T02:46:44.779097Z","shell.execute_reply.started":"2022-12-27T02:46:41.968978Z","shell.execute_reply":"2022-12-27T02:46:44.778249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_article = pd.read_csv(articles_path,dtype={'article_id': str})\n\n# df_cus = pd.read_csv(customers_path, dtype={'customer_id': str})\n\n# df_trans = pd.read_csv(transactions_path, dtype={'article_id': str})\n# df_trans['t_dat'] = pd.to_datetime(df_trans['t_dat'])","metadata":{"execution":{"iopub.status.busy":"2022-12-27T02:46:50.469330Z","iopub.execute_input":"2022-12-27T02:46:50.469708Z","iopub.status.idle":"2022-12-27T02:46:50.973257Z","shell.execute_reply.started":"2022-12-27T02:46:50.469678Z","shell.execute_reply":"2022-12-27T02:46:50.972242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand = 64\nlgb_params = {\n    \"objective\": \"binary\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"subsample\": 0.8,\n    \"subsample_freq\": 1,\n    \"bagging_seed\": rand,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 100,\n    \"lambda_l1\": 0,\n    \"lambda_l2\": 0,\n    \"random_state\": rand,\n    \"metric\": \"auc\",#\"binary_logloss\",\n    \"verbose\": -1\n}","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.563421Z","iopub.execute_input":"2022-12-24T15:25:08.563899Z","iopub.status.idle":"2022-12-24T15:25:08.570856Z","shell.execute_reply.started":"2022-12-24T15:25:08.563865Z","shell.execute_reply":"2022-12-24T15:25:08.569894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tran_dtypes = {\"t_dat\":\"str\",\n               \"customer_id\":\"str\",\n               \"article_id\":\"int\",\n               \"product_code\":\"int\",\n               \"price\":\"float\",\n               \"sales_channel_id\":\"int\"}\nart_dtypes = {\"article_id\":\"int\",\n              \"product_code\":\"int\",\n              \"product_type_no\":\"int\",\n              \"graphical_appearance_no\":\"int\",\n              \"colour_group_code\":\"int\",\n              \"department_no\":\"int\",\n              \"index_code\":\"str\",\n              \"index_group_no\":\"int\",\n              \"section_no\":\"int\",\n              \"garment_group_no\":\"int\"}\ncust_dtypes = {\"customer_id\":\"str\"}\n\nobj = \"class\" # \"class\" or \"rank\"\nN = 15000\nn_iter = 2 # num of iteration\nidx_file = \"exp05\"\nlen_hist = 366\nn_round = 2000\nn_splits = 1\ntr_set = [1,8,15,22] # set of train date\nlen_tr = 7 # length of validation period\nnobuy = 20 # num of negative samples","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.572867Z","iopub.execute_input":"2022-12-24T15:25:08.573288Z","iopub.status.idle":"2022-12-24T15:25:08.580403Z","shell.execute_reply.started":"2022-12-24T15:25:08.573254Z","shell.execute_reply":"2022-12-24T15:25:08.579559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cos_sim(v1, v2):\n    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.583902Z","iopub.execute_input":"2022-12-24T15:25:08.584151Z","iopub.status.idle":"2022-12-24T15:25:08.592881Z","shell.execute_reply.started":"2022-12-24T15:25:08.584128Z","shell.execute_reply":"2022-12-24T15:25:08.591948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(day_oldest):\n    df_art = pd.read_csv(articles_path, dtype=art_dtypes)\n    le = LabelEncoder()\n    le.fit(df_art[\"index_code\"].unique())\n    df_art[\"index_code\"] = le.transform(df_art[\"index_code\"])\n\n    df_trans = pd.read_csv(transactions_path,dtype=tran_dtypes)\n    df_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"],format=\"%Y-%m-%d\")\n\n    df_trans = df_trans.query(f\"t_dat >= '{day_oldest}'\").copy()\n    df_trans = df_trans.drop_duplicates([\"customer_id\",\"article_id\",\"t_dat\"])\n    df_trans = df_trans.merge(df_art[[\"article_id\",\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",on=\"article_id\")\n\n    df_cust = pd.read_csv(customers_path,dtype=cust_dtypes)\n    df_cust[\"age\"] = df_cust[\"age\"].fillna(df_cust[\"age\"].mean())\n    df_cust[[\"FN\",\"Active\"]] = df_cust[[\"FN\",\"Active\"]].fillna(0)\n    df_cust[\"club_member_status\"] = df_cust[\"club_member_status\"].apply(lambda x:1 if x == \"ACTIVE\" else 0)\n    df_cust[\"fashion_news_frequency\"] = df_cust[\"fashion_news_frequency\"].apply(lambda x:0 if x == \"NONE\" else 1)\n\n    dict_vec = {}\n    vec_art = np.load(article2vec_path)\n    df_vec = pd.concat([df_art[\"article_id\"],pd.DataFrame(vec_art)],axis=1)\n    for i in range(len(vec_art)):\n        dict_vec[df_art[\"article_id\"][i]] = vec_art[i]\n    del vec_art,df_vec\n\n\n    return df_trans,df_art,df_cust, dict_vec","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.594865Z","iopub.execute_input":"2022-12-24T15:25:08.595487Z","iopub.status.idle":"2022-12-24T15:25:08.602480Z","shell.execute_reply.started":"2022-12-24T15:25:08.595452Z","shell.execute_reply":"2022-12-24T15:25:08.601430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_store(df_trans,l_cust,ds,de,dsr,der,dsh,deh):\n    feat ={}\n\n    df_trans_yesterday = df_trans.query(\"(t_dat == @der)\")\n    df_trans_recent = df_trans.query(\"(t_dat >= @dsr) and (t_dat <= @der)\")\n    df_trans_hist = df_trans.query(\"(t_dat >= @dsh) and (t_dat <= @deh)\")\n\n    feat[\"art_buy_hist\"] = df_trans_hist.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_hist=\"count\")\n    feat[\"art_buy_recent\"] = df_trans_recent.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_recent=\"count\")\n    feat[\"art_buy_yesterday\"] = df_trans_yesterday.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_yesterday=\"count\")\n    df_buy1 = df_trans_hist.groupby(\"article_id\")[\"customer_id\"].nunique().reset_index().rename(columns={\"customer_id\":\"cnt_buy1\"})\n    df_buy2 = df_trans_hist[df_trans_hist.duplicated([\"customer_id\",\"article_id\"])].copy()\n    df_buy2 = df_buy2.drop_duplicates([\"customer_id\",\"article_id\"])\n    df_buy2 = df_buy2.groupby(\"article_id\")[\"article_id\"].agg(cnt_buy2='count').reset_index()\n    df_buy = pd.merge(df_buy1,df_buy2,how=\"left\",on=\"article_id\").fillna(0)\n    df_buy[\"rebuy_rate\"] = df_buy[\"cnt_buy2\"]/df_buy[\"cnt_buy1\"]\n    feat[\"rebuy_rate\"] = df_buy[[\"article_id\",\"rebuy_rate\"]]\n\n    df_trans_yesterday = df_trans_yesterday.query(\"(customer_id in @l_cust)\")\n    df_trans_recent = df_trans_recent.query(\"(customer_id in @l_cust)\")\n    df_trans_hist = df_trans_hist.query(\"(customer_id in @l_cust)\")\n    feat[\"rate_sales_channel_hist\"] = df_trans_hist.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_hist=\"mean\")\n    feat[\"rate_sales_channel_recent\"] = df_trans_recent.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_recent=\"mean\")\n    feat[\"n_buy_hist\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_hist=\"count\")\n    feat[\"n_buy_recent\"] = df_trans_recent.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_recent=\"count\")\n    feat[\"days_after_buy\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(days_after_buy=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_hist_all=\"count\")\n    feat[\"n_buy_recent_all\"] = df_trans_recent.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_recent_all=\"count\")\n    feat[\"days_after_buy_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(days_after_buy_all=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_hist_prod=\"count\")\n    feat[\"n_buy_recent_prod\"] = df_trans_recent.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_recent_prod=\"count\")\n    feat[\"days_after_buy_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(days_after_buy_prod=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_hist_ptype=\"count\")\n    feat[\"n_buy_recent_ptype\"] = df_trans_recent.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_recent_ptype=\"count\")\n    feat[\"days_after_buy_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(days_after_buy_ptype=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_hist_graph=\"count\")\n    feat[\"n_buy_recent_graph\"] = df_trans_recent.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_recent_graph=\"count\")\n    feat[\"days_after_buy_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(days_after_buy_graph=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_hist_col=\"count\")\n    feat[\"n_buy_recent_col\"] = df_trans_recent.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_recent_col=\"count\")\n    feat[\"days_after_buy_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(days_after_buy_col=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_hist_dep=\"count\")\n    feat[\"n_buy_recent_dep\"] = df_trans_recent.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_recent_dep=\"count\")\n    feat[\"days_after_buy_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(days_after_buy_dep=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_hist_idx=\"count\")\n    feat[\"n_buy_recent_idx\"] = df_trans_recent.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_recent_idx=\"count\")\n    feat[\"days_after_buy_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(days_after_buy_idx=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_hist_idxg=\"count\")\n    feat[\"n_buy_recent_idxg\"] = df_trans_recent.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_recent_idxg=\"count\")\n    feat[\"days_after_buy_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(days_after_buy_idxg=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_hist_sec=\"count\")\n    feat[\"n_buy_recent_sec\"] = df_trans_recent.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_recent_sec=\"count\")\n    feat[\"days_after_buy_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(days_after_buy_sec=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_hist_garm=\"count\")\n    feat[\"n_buy_recent_garm\"] = df_trans_recent.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_recent_garm=\"count\")\n    feat[\"days_after_buy_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(days_after_buy_garm=lambda x:(ds - max(x)).days)\n    feat[\"art_id_recent\"] = df_trans_recent.groupby(\"customer_id\")[\"article_id\"].apply(list).rename(\"art_id_recent\")\n\n    del df_trans_yesterday, df_trans_recent, df_trans_hist, df_buy1, df_buy2, df_buy\n    gc.collect()\n\n    return feat","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.656196Z","iopub.execute_input":"2022-12-24T15:25:08.656483Z","iopub.status.idle":"2022-12-24T15:25:08.663705Z","shell.execute_reply.started":"2022-12-24T15:25:08.656457Z","shell.execute_reply":"2022-12-24T15:25:08.662621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_feat(df,ds,de,dsr,der,dsh,deh,feat,dict_vec):\n  # rate_sales_channel_hist\n  df = df.merge(feat[\"rate_sales_channel_hist\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # rate_sales_channel_recent\n  df = df.merge(feat[\"rate_sales_channel_recent\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)  \n  # art_buy_hist\n  df = df.merge(feat[\"art_buy_hist\"],how=\"left\",left_on=[\"article_id\"], right_index=True)\n  # art_buy_recent\n  df = df.merge(feat[\"art_buy_recent\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # art_buy_yesterday\n  df = df.merge(feat[\"art_buy_yesterday\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # n_buy_hist\n  df = df.merge(feat[\"n_buy_hist\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_recent\n  df = df.merge(feat[\"n_buy_recent\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # days_after_buy\n  df = df.merge(feat[\"days_after_buy\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_hist_all\n  df = df.merge(feat[\"n_buy_hist_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_recent_all\n  df = df.merge(feat[\"n_buy_recent_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # days_after_buy_all\n  df = df.merge(feat[\"days_after_buy_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_hist_prod\n  df = df.merge(feat[\"n_buy_hist_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_recent_prod\n  df = df.merge(feat[\"n_buy_recent_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # days_after_buy_prod\n  df = df.merge(feat[\"days_after_buy_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_hist_ptype\n  df = df.merge(feat[\"n_buy_hist_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_recent_ptype\n  df = df.merge(feat[\"n_buy_recent_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # days_after_buy_ptype\n  df = df.merge(feat[\"days_after_buy_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_hist_graph\n  df = df.merge(feat[\"n_buy_hist_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_recent_graph\n  df = df.merge(feat[\"n_buy_recent_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # days_after_buy_graph\n  df = df.merge(feat[\"days_after_buy_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_hist_col\n  df = df.merge(feat[\"n_buy_hist_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_recent_col\n  df = df.merge(feat[\"n_buy_recent_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # days_after_buy_col\n  df = df.merge(feat[\"days_after_buy_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_hist_dep\n  df = df.merge(feat[\"n_buy_hist_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_recent_dep\n  df = df.merge(feat[\"n_buy_recent_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # days_after_buy_dep\n  df = df.merge(feat[\"days_after_buy_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_hist_idx\n  df = df.merge(feat[\"n_buy_hist_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_recent_idx\n  df = df.merge(feat[\"n_buy_recent_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # days_after_buy_idx\n  df = df.merge(feat[\"days_after_buy_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_hist_idxg\n  df = df.merge(feat[\"n_buy_hist_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_recent_idxg\n  df = df.merge(feat[\"n_buy_recent_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # days_after_buy_idxg\n  df = df.merge(feat[\"days_after_buy_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_hist_sec\n  df = df.merge(feat[\"n_buy_hist_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_recent_sec\n  df = df.merge(feat[\"n_buy_recent_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # days_after_buy_sec\n  df = df.merge(feat[\"days_after_buy_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_hist_garm\n  df = df.merge(feat[\"n_buy_hist_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # n_buy_recent_garm\n  df = df.merge(feat[\"n_buy_recent_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # days_after_buy_garm\n  df = df.merge(feat[\"days_after_buy_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # rebuy_rate\n  df = df.merge(feat[\"rebuy_rate\"],how=\"left\",on=\"article_id\")\n  # sim_article\n  df = df.merge(feat[\"art_id_recent\"],how=\"left\",left_on=\"customer_id\", right_index = True)\n  sim_max,sim_sum,sim_mean = [],[],[]\n  # display(df[[\"article_id\",\"art_id_recent\"]].head())\n  tmp = df[[\"article_id\",\"art_id_recent\"]].values\n  for i in range(len(df)):\n      if not isinstance(tmp[i][1],list):\n        sim_max.append(0);sim_sum.append(0);sim_mean.append(0)\n      else:\n        list_sim = [cos_sim(dict_vec[tmp[i][0]],dict_vec[x]) for x in tmp[i][1]]\n        sim_max.append(max(list_sim))\n        sim_sum.append(sum(list_sim))\n        sim_mean.append(np.mean(list_sim))\n  df[\"sim_max\"] = sim_max\n  df[\"sim_sum\"] = sim_sum\n  df[\"sim_mean\"] = sim_mean\n  df = df.drop([\"art_id_recent\"], axis = 1)\n  # fillna\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]] =\\\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]].fillna(0)\n\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]] = \\\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]].fillna(10+len_hist)\n\n  df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]] = df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]].fillna(1.5)\n  \n  return df","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.685153Z","iopub.execute_input":"2022-12-24T15:25:08.685435Z","iopub.status.idle":"2022-12-24T15:25:08.694020Z","shell.execute_reply.started":"2022-12-24T15:25:08.685410Z","shell.execute_reply":"2022-12-24T15:25:08.692887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend_train(day_start_val):\n    day_start = [day_start_val - datetime.timedelta(days=i-1+len_tr) for i in tr_set]\n    day_end = [day_start_val - datetime.timedelta(days=i) for i in tr_set]\n    day_start_rec = [x - datetime.timedelta(days=7) for x in day_start]\n    day_end_rec = [x - datetime.timedelta(days=1) for x in day_start]\n    day_start_hist = [x - datetime.timedelta(days=len_hist) for x in day_start]\n    day_end_hist = [x - datetime.timedelta(days=1) for x in day_start]\n    day_start_rec_test = day_start_val - datetime.timedelta(days=7)\n    day_end_rec_test = day_start_val - datetime.timedelta(days=1)\n    day_start_hist_test = day_start_val - datetime.timedelta(days=1+len_hist)\n    day_end_hist_test = day_start_val - datetime.timedelta(days=1)\n    day_end_val = day_start_val + datetime.timedelta(days=6)\n\n    df_trans, df_art, df_cust, dict_vec = read_data(day_oldest = day_start_hist[-1])\n\n    q_date = \"\"\n    for i in range(len(day_start)):\n        if i == 0: q_date = f\"((t_dat >= '{day_start[0]}') and (t_dat <= '{day_end[0]}'))\"\n        else: q_date = q_date + f\" or ((t_dat >= '{day_start[i]}') and (t_dat <= '{day_end[i]}'))\"\n    top_art_all = df_trans.query(q_date).groupby(\"article_id\")[\"t_dat\"].count().sort_values(ascending = False).index[:N].tolist()\n\n    list_df_buy = []\n    list_list_cust = []\n    # make posivive samples\n    for i in range(len(day_start)):\n        list_df_buy.append(df_trans.query(f\"(t_dat >= '{day_start[i]}') and (t_dat <= '{day_end[i]}') and (article_id in @top_art_all)\").drop_duplicates([\"customer_id\",\"article_id\"])[[\"customer_id\",\"article_id\"]].copy())\n        list_df_buy[i][\"target\"] = 1\n        list_list_cust.append(list_df_buy[i][\"customer_id\"].unique().tolist()) \n    # make negative samples(random pick)\n    for iter_train in tqdm(range(n_iter)):\n        list_df_nobuy = []\n        list_train =[]\n        for i in range(len(day_start)):\n            list_df_nobuy.append(pd.concat([pd.DataFrame({\"customer_id\":x,\"article_id\":random.sample(top_art_all,nobuy)}) for x in list_list_cust[i]]))\n            list_df_nobuy[i][\"target\"] = 0\n            list_train.append(pd.concat([list_df_buy[i],list_df_nobuy[i]]).drop_duplicates([\"customer_id\",\"article_id\"]))\n        del list_df_nobuy\n        display(list_train[0][\"target\"].value_counts())\n\n        # add feature\n        df_train = pd.DataFrame()\n        for i in tqdm(range(len(day_start))):\n            feat = feat_store(df_trans,list_list_cust[i],day_start[i],day_end[i],day_start_rec[i],day_end_rec[i],day_start_hist[i],day_end_hist[i])\n            list_train[i] = list_train[i].merge(df_art[[\"article_id\",\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",on=\"article_id\")\n            list_train[i] = list_train[i].merge(df_cust[[\"customer_id\",\"age\",\"FN\",\"Active\",\"club_member_status\",\"fashion_news_frequency\"]],how=\"left\",on=\"customer_id\")\n            df_train = df_train.append(add_feat(list_train[i],day_start[i],day_end[i],day_start_rec[i],day_end_rec[i],day_start_hist[i],day_end_hist[i],feat,dict_vec))\n            del feat\n        del list_train\n        gc.collect()\n\n        # train lgbm\n        X_train = df_train.drop([\"customer_id\",\"product_code\",\"product_type_no\",\"department_no\",\"target\"],axis=1)\n        y_train = df_train[\"target\"]\n        del df_train\n\n        list_model = []\n        if n_splits == 1:\n            X_tr, X_va, y_tr, y_va = train_test_split(X_train,y_train,stratify = y_train)\n            d_tr = lgb.Dataset(X_tr, label=y_tr,  free_raw_data=False)\n            d_va = lgb.Dataset(X_va, label=y_va,  free_raw_data=False)\n            list_model.append(lgb.train(lgb_params, train_set=d_tr, num_boost_round=n_round, valid_sets=[d_tr,d_va], verbose_eval=500, early_stopping_rounds=100))\n        else:\n            folds = StratifiedKFold(n_splits = n_splits, shuffle = True, random_state = rand)\n            for tr_idx,va_idx in folds.split(X_train,y_train):\n                X_tr, X_va, y_tr, y_va = X_train.iloc[tr_idx], X_train.iloc[va_idx], y_train.iloc[tr_idx], y_train.iloc[va_idx] \n                d_tr = lgb.Dataset(X_tr, label=y_tr,  free_raw_data=False)\n                d_va = lgb.Dataset(X_va, label=y_va,  free_raw_data=False)\n                list_model.append(lgb.train(lgb_params, train_set=d_tr, num_boost_round=n_round, valid_sets=[d_tr,d_va], verbose_eval=500, early_stopping_rounds=100))\n        # save model\n        pd.to_pickle(list_model,f\"/kaggle/working/models_{iter_train}.pkl\")\n        del X_train, y_train, X_tr, X_va, y_tr, y_va, d_tr, d_va\n        gc.collect()\n    del df_trans, df_art, df_cust\n    gc.collect()\n    return 0","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.700469Z","iopub.execute_input":"2022-12-24T15:25:08.700789Z","iopub.status.idle":"2022-12-24T15:25:08.709656Z","shell.execute_reply.started":"2022-12-24T15:25:08.700762Z","shell.execute_reply":"2022-12-24T15:25:08.708377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommend_train(datetime.datetime(2020,9,23))","metadata":{"execution":{"iopub.status.busy":"2022-12-24T15:25:08.712003Z","iopub.execute_input":"2022-12-24T15:25:08.712563Z","iopub.status.idle":"2022-12-24T15:25:08.723319Z","shell.execute_reply.started":"2022-12-24T15:25:08.712529Z","shell.execute_reply":"2022-12-24T15:25:08.722310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##############################################\n#                Test                        #\n##############################################","metadata":{}},{"cell_type":"code","source":"# !pip install cudf\nimport cudf\nfrom cuml import ForestInference","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:43:32.878558Z","iopub.execute_input":"2023-02-06T13:43:32.879115Z","iopub.status.idle":"2023-02-06T13:43:32.888788Z","shell.execute_reply.started":"2023-02-06T13:43:32.879069Z","shell.execute_reply":"2023-02-06T13:43:32.887657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article2vec_path =\"/kaggle/input/h-m-rapids-article2vec/articles.npy\"","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:43:35.119274Z","iopub.execute_input":"2023-02-06T13:43:35.120390Z","iopub.status.idle":"2023-02-06T13:43:35.125092Z","shell.execute_reply.started":"2023-02-06T13:43:35.120354Z","shell.execute_reply":"2023-02-06T13:43:35.123939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport gc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nimport datetime\nimport itertools\nimport os\nfrom contextlib import redirect_stdout\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:43:36.477663Z","iopub.execute_input":"2023-02-06T13:43:36.478279Z","iopub.status.idle":"2023-02-06T13:43:36.484998Z","shell.execute_reply.started":"2023-02-06T13:43:36.478239Z","shell.execute_reply":"2023-02-06T13:43:36.483868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand = 64\nlgb_params = {}\nlgb_params[\"class\"] = {\n    \"objective\": \"binary\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"subsample\": 0.8,\n    \"subsample_freq\": 1,\n    \"bagging_seed\": rand,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 100,\n    \"lambda_l1\": 0,\n    \"lambda_l2\": 0,\n    \"random_state\": rand,\n    \"metric\": \"auc\",#\"binary_logloss\",\n    \"verbose\": -1\n}\n\nlgb_params[\"rank\"] = {\n    \"objective\": \"lambdarank\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 40,\n    \"subsample\": 0.8,\n    \"subsample_freq\": 1,\n    \"bagging_seed\": rand,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 100,\n    \"lambda_l1\": 0,\n    \"lambda_l2\": 0,\n    \"random_state\": rand,\n    \"metric\": \"map\",\n    \"eval_at\": 12,\n    \"verbose\": -1\n}","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:43:39.824535Z","iopub.execute_input":"2023-02-06T13:43:39.824928Z","iopub.status.idle":"2023-02-06T13:43:39.833042Z","shell.execute_reply.started":"2023-02-06T13:43:39.824897Z","shell.execute_reply":"2023-02-06T13:43:39.831937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tran_dtypes = {\"t_dat\":\"str\",\n               \"customer_id\":\"str\",\n               \"article_id\":\"int\",\n               \"product_code\":\"int\",\n               \"price\":\"float\",\n               \"sales_channel_id\":\"int\"}\nart_dtypes = {\"article_id\":\"int\",\n              \"product_code\":\"int\",\n              \"product_type_no\":\"int\",\n              \"graphical_appearance_no\":\"int\",\n              \"colour_group_code\":\"int\",\n              \"department_no\":\"int\",\n              \"index_code\":\"str\",\n              \"index_group_no\":\"int\",\n              \"section_no\":\"int\",\n              \"garment_group_no\":\"int\"}\ncust_dtypes = {\"customer_id\":\"str\"}\n\nobj = \"class\" # \"class\" or \"rank\"\nN = 15000\nN_div = 20\nn_iter = 2 # num of iteration\nidx_file = \"exp05\"\nlen_hist = 366\nn_round = 4000\nn_splits = 1\ntmp_top = 200\ntr_set = [1,8,15,22] # set of train date\nlen_tr = 7 # length of validation period\nnobuy = 20 # num of negative samples","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:43:42.777418Z","iopub.execute_input":"2023-02-06T13:43:42.778128Z","iopub.status.idle":"2023-02-06T13:43:42.786313Z","shell.execute_reply.started":"2023-02-06T13:43:42.778088Z","shell.execute_reply":"2023-02-06T13:43:42.785257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv\ncandidates = []\ni = 0\nwith open('/kaggle/input/lgbmmodel/submissionx.csv', mode ='r')as file:\n    csvFile = csv.reader(file)\n    for lines in csvFile:\n        candidates.append(lines)\n#         i+=1\n#         if i == 300:\n#             break","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:43:48.374462Z","iopub.execute_input":"2023-02-06T13:43:48.375164Z","iopub.status.idle":"2023-02-06T13:44:02.763052Z","shell.execute_reply.started":"2023-02-06T13:43:48.375115Z","shell.execute_reply":"2023-02-06T13:44:02.761989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates = candidates[1:]","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:44:27.256935Z","iopub.execute_input":"2023-02-06T13:44:27.257638Z","iopub.status.idle":"2023-02-06T13:44:27.290633Z","shell.execute_reply.started":"2023-02-06T13:44:27.257594Z","shell.execute_reply":"2023-02-06T13:44:27.289484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(candidates)","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:44:31.073601Z","iopub.execute_input":"2023-02-06T13:44:31.074673Z","iopub.status.idle":"2023-02-06T13:44:31.081668Z","shell.execute_reply.started":"2023-02-06T13:44:31.074610Z","shell.execute_reply":"2023-02-06T13:44:31.080514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_trans = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\",dtype=tran_dtypes)\ndf_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"],format=\"%Y-%m-%d\")\ndf_trans = df_trans.drop_duplicates([\"customer_id\",\"article_id\",\"t_dat\"])\n\nday_end_valtmp = df_trans[\"t_dat\"].max()\nday_start_valtmp = day_end_valtmp - datetime.timedelta(days=6)  \ndf_trans_val_1 = df_trans.query(\"(t_dat >= @day_start_valtmp) and (t_dat <= @day_end_valtmp)\").copy()\ndf_trans_val_1[\"article_id\"] = df_trans_val_1[\"article_id\"].astype(str).str.zfill(10)\ndf_agg_val_1 = df_trans_val_1.groupby(\"customer_id\")[\"article_id\"].apply(list).reset_index()\ndf_agg_val_1 = df_agg_val_1[df_agg_val_1[\"article_id\"].apply(len) != 0]\n\ndel df_trans, df_trans_val_1\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:44:34.180626Z","iopub.execute_input":"2023-02-06T13:44:34.181009Z","iopub.status.idle":"2023-02-06T13:45:24.483665Z","shell.execute_reply.started":"2023-02-06T13:44:34.180977Z","shell.execute_reply":"2023-02-06T13:45:24.482638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(day_oldest):\n    df_art = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\",dtype=art_dtypes)\n    le = LabelEncoder()\n    le.fit(df_art[\"index_code\"].unique())\n    df_art[\"index_code\"] = le.transform(df_art[\"index_code\"])\n#     display(df_art[\"index_code\"].unique())\n\n    df_cust = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv\",dtype=cust_dtypes)\n    df_cust[\"age\"] = df_cust[\"age\"].fillna(df_cust[\"age\"].mean())\n    df_cust[[\"FN\",\"Active\"]] = df_cust[[\"FN\",\"Active\"]].fillna(0)\n    df_cust[\"club_member_status\"] = df_cust[\"club_member_status\"].apply(lambda x:1 if x == \"ACTIVE\" else 0)\n    df_cust[\"fashion_news_frequency\"] = df_cust[\"fashion_news_frequency\"].apply(lambda x:0 if x == \"NONE\" else 1)\n\n    df_trans = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\",dtype=tran_dtypes)\n    df_trans[\"t_dat\"] = pd.to_datetime(df_trans[\"t_dat\"],format=\"%Y-%m-%d\")\n    df_trans = df_trans.query(f\"t_dat >= '{day_oldest}'\").copy()\n    df_trans = df_trans.drop_duplicates([\"customer_id\",\"article_id\",\"t_dat\"])\n    df_trans = df_trans.merge(df_art[[\"article_id\",\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",on=\"article_id\")\n    df_trans = df_trans.merge(df_cust[[\"customer_id\",\"age\"]],how=\"left\",on=\"customer_id\")\n    \n    dict_vec = {}\n    vec_art = np.load(article2vec_path)\n    df_vec = pd.concat([df_art[\"article_id\"],pd.DataFrame(vec_art)],axis=1)\n    for i in range(len(vec_art)):\n        dict_vec[df_art[\"article_id\"][i]] = vec_art[i]\n    del vec_art,df_vec\n\n    df_art = df_art.set_index(\"article_id\")\n    df_cust = df_cust.set_index(\"customer_id\")\n    print(\"end read_data\")\n\n    return df_trans,cudf.from_pandas(df_art),cudf.from_pandas(df_cust),dict_vec","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:46:00.778601Z","iopub.execute_input":"2023-02-06T13:46:00.779330Z","iopub.status.idle":"2023-02-06T13:46:00.790518Z","shell.execute_reply.started":"2023-02-06T13:46:00.779294Z","shell.execute_reply":"2023-02-06T13:46:00.789486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_store(df_trans,l_cust,ds,de,dsr,der,dsh,deh):\n    feat ={}\n\n    df_trans_yesterday = df_trans.query(\"(t_dat == @der)\")\n    df_trans_recent = df_trans.query(\"(t_dat >= @dsr) and (t_dat <= @der)\")\n    df_trans_hist = df_trans.query(\"(t_dat >= @dsh) and (t_dat <= @deh)\")\n\n    feat[\"art_buy_hist\"] = df_trans_hist.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_hist=\"count\")\n    feat[\"art_buy_recent\"] = df_trans_recent.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_recent=\"count\")\n    feat[\"art_buy_yesterday\"] = df_trans_yesterday.groupby([\"article_id\"])[\"t_dat\"].agg(art_buy_yesterday=\"count\")\n    df_buy1 = df_trans_hist.groupby(\"article_id\")[\"customer_id\"].nunique().reset_index().rename(columns={\"customer_id\":\"cnt_buy1\"})\n    df_buy2 = df_trans_hist[df_trans_hist.duplicated([\"customer_id\",\"article_id\"])].copy()\n    df_buy2 = df_buy2.drop_duplicates([\"customer_id\",\"article_id\"])\n    df_buy2 = df_buy2.groupby(\"article_id\")[\"article_id\"].agg(cnt_buy2='count').reset_index()\n    df_buy = pd.merge(df_buy1,df_buy2,how=\"left\",on=\"article_id\").fillna(0)\n    df_buy[\"rebuy_rate\"] = df_buy[\"cnt_buy2\"]/df_buy[\"cnt_buy1\"]\n    feat[\"rebuy_rate\"] = df_buy[[\"article_id\",\"rebuy_rate\"]]\n\n    df_trans_yesterday = df_trans_yesterday.query(\"(customer_id in @l_cust)\")\n    df_trans_recent = df_trans_recent.query(\"(customer_id in @l_cust)\")\n    df_trans_hist = df_trans_hist.query(\"(customer_id in @l_cust)\")\n    feat[\"rate_sales_channel_hist\"] = df_trans_hist.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_hist=\"mean\")\n    feat[\"rate_sales_channel_recent\"] = df_trans_recent.groupby([\"customer_id\"])[\"sales_channel_id\"].agg(rate_sales_channel_recent=\"mean\")\n    feat[\"n_buy_hist\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_hist=\"count\")\n    feat[\"n_buy_recent\"] = df_trans_recent.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(n_buy_recent=\"count\")\n    feat[\"days_after_buy\"] = df_trans_hist.groupby([\"customer_id\",\"article_id\"])[\"t_dat\"].agg(days_after_buy=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_hist_all=\"count\")\n    feat[\"n_buy_recent_all\"] = df_trans_recent.groupby([\"customer_id\"])[\"t_dat\"].agg(n_buy_recent_all=\"count\")\n    feat[\"days_after_buy_all\"] = df_trans_hist.groupby([\"customer_id\"])[\"t_dat\"].agg(days_after_buy_all=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_hist_prod=\"count\")\n    feat[\"n_buy_recent_prod\"] = df_trans_recent.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(n_buy_recent_prod=\"count\")\n    feat[\"days_after_buy_prod\"] = df_trans_hist.groupby([\"customer_id\",\"product_code\"])[\"t_dat\"].agg(days_after_buy_prod=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_hist_ptype=\"count\")\n    feat[\"n_buy_recent_ptype\"] = df_trans_recent.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(n_buy_recent_ptype=\"count\")\n    feat[\"days_after_buy_ptype\"] = df_trans_hist.groupby([\"customer_id\",\"product_type_no\"])[\"t_dat\"].agg(days_after_buy_ptype=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_hist_graph=\"count\")\n    feat[\"n_buy_recent_graph\"] = df_trans_recent.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(n_buy_recent_graph=\"count\")\n    feat[\"days_after_buy_graph\"] = df_trans_hist.groupby([\"customer_id\",\"graphical_appearance_no\"])[\"t_dat\"].agg(days_after_buy_graph=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_hist_col=\"count\")\n    feat[\"n_buy_recent_col\"] = df_trans_recent.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(n_buy_recent_col=\"count\")\n    feat[\"days_after_buy_col\"] = df_trans_hist.groupby([\"customer_id\",\"colour_group_code\"])[\"t_dat\"].agg(days_after_buy_col=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_hist_dep=\"count\")\n    feat[\"n_buy_recent_dep\"] = df_trans_recent.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(n_buy_recent_dep=\"count\")\n    feat[\"days_after_buy_dep\"] = df_trans_hist.groupby([\"customer_id\",\"department_no\"])[\"t_dat\"].agg(days_after_buy_dep=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_hist_idx=\"count\")\n    feat[\"n_buy_recent_idx\"] = df_trans_recent.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(n_buy_recent_idx=\"count\")\n    feat[\"days_after_buy_idx\"] = df_trans_hist.groupby([\"customer_id\",\"index_code\"])[\"t_dat\"].agg(days_after_buy_idx=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_hist_idxg=\"count\")\n    feat[\"n_buy_recent_idxg\"] = df_trans_recent.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(n_buy_recent_idxg=\"count\")\n    feat[\"days_after_buy_idxg\"] = df_trans_hist.groupby([\"customer_id\",\"index_group_no\"])[\"t_dat\"].agg(days_after_buy_idxg=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_hist_sec=\"count\")\n    feat[\"n_buy_recent_sec\"] = df_trans_recent.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(n_buy_recent_sec=\"count\")\n    feat[\"days_after_buy_sec\"] = df_trans_hist.groupby([\"customer_id\",\"section_no\"])[\"t_dat\"].agg(days_after_buy_sec=lambda x:(ds - max(x)).days)\n    feat[\"n_buy_hist_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_hist_garm=\"count\")\n    feat[\"n_buy_recent_garm\"] = df_trans_recent.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(n_buy_recent_garm=\"count\")\n    feat[\"days_after_buy_garm\"] = df_trans_hist.groupby([\"customer_id\",\"garment_group_no\"])[\"t_dat\"].agg(days_after_buy_garm=lambda x:(ds - max(x)).days)\n    feat[\"art_id_recent\"] = df_trans_recent.groupby(\"customer_id\")[\"article_id\"].apply(list).rename(\"art_id_recent\")\n\n    for k in feat.keys():\n        feat[k] = cudf.from_pandas(feat[k])\n\n    del df_trans_yesterday, df_trans_recent, df_trans_hist, df_buy1, df_buy2, df_buy\n    gc.collect()\n    print(\"end feat\")\n    return feat","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:46:04.913458Z","iopub.execute_input":"2023-02-06T13:46:04.913955Z","iopub.status.idle":"2023-02-06T13:46:04.957079Z","shell.execute_reply.started":"2023-02-06T13:46:04.913913Z","shell.execute_reply":"2023-02-06T13:46:04.956130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_feat(df,ds,de,dsr,der,dsh,deh,feat,dict_vec):\n  # rate_sales_channel_hist\n  df = df.merge(feat[\"rate_sales_channel_hist\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # rate_sales_channel_recent\n  df = df.merge(feat[\"rate_sales_channel_recent\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)  \n  # art_buy_hist\n  df = df.merge(feat[\"art_buy_hist\"],how=\"left\",left_on=[\"article_id\"], right_index=True)\n  # art_buy_recent\n  df = df.merge(feat[\"art_buy_recent\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # art_buy_yesterday\n  df = df.merge(feat[\"art_buy_yesterday\"],how=\"left\",left_on=[\"article_id\"], right_index=True)  \n  # n_buy_hist\n  df = df.merge(feat[\"n_buy_hist\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_recent\n  df = df.merge(feat[\"n_buy_recent\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # days_after_buy\n  df = df.merge(feat[\"days_after_buy\"],how=\"left\",left_on=[\"customer_id\",\"article_id\"], right_index=True)\n  # n_buy_hist_all\n  df = df.merge(feat[\"n_buy_hist_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_recent_all\n  df = df.merge(feat[\"n_buy_recent_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # days_after_buy_all\n  df = df.merge(feat[\"days_after_buy_all\"],how=\"left\",left_on=[\"customer_id\"], right_index=True)\n  # n_buy_hist_prod\n  df = df.merge(feat[\"n_buy_hist_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_recent_prod\n  df = df.merge(feat[\"n_buy_recent_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # days_after_buy_prod\n  df = df.merge(feat[\"days_after_buy_prod\"],how=\"left\",left_on=[\"customer_id\",\"product_code\"], right_index=True)\n  # n_buy_hist_ptype\n  df = df.merge(feat[\"n_buy_hist_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_recent_ptype\n  df = df.merge(feat[\"n_buy_recent_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # days_after_buy_ptype\n  df = df.merge(feat[\"days_after_buy_ptype\"],how=\"left\",left_on=[\"customer_id\",\"product_type_no\"], right_index=True)\n  # n_buy_hist_graph\n  df = df.merge(feat[\"n_buy_hist_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_recent_graph\n  df = df.merge(feat[\"n_buy_recent_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # days_after_buy_graph\n  df = df.merge(feat[\"days_after_buy_graph\"],how=\"left\",left_on=[\"customer_id\",\"graphical_appearance_no\"], right_index=True)\n  # n_buy_hist_col\n  df = df.merge(feat[\"n_buy_hist_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_recent_col\n  df = df.merge(feat[\"n_buy_recent_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # days_after_buy_col\n  df = df.merge(feat[\"days_after_buy_col\"],how=\"left\",left_on=[\"customer_id\",\"colour_group_code\"], right_index=True)\n  # n_buy_hist_dep\n  df = df.merge(feat[\"n_buy_hist_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_recent_dep\n  df = df.merge(feat[\"n_buy_recent_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # days_after_buy_dep\n  df = df.merge(feat[\"days_after_buy_dep\"],how=\"left\",left_on=[\"customer_id\",\"department_no\"], right_index=True)\n  # n_buy_hist_idx\n  df = df.merge(feat[\"n_buy_hist_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_recent_idx\n  df = df.merge(feat[\"n_buy_recent_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # days_after_buy_idx\n  df = df.merge(feat[\"days_after_buy_idx\"],how=\"left\",left_on=[\"customer_id\",\"index_code\"], right_index=True)\n  # n_buy_hist_idxg\n  df = df.merge(feat[\"n_buy_hist_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_recent_idxg\n  df = df.merge(feat[\"n_buy_recent_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # days_after_buy_idxg\n  df = df.merge(feat[\"days_after_buy_idxg\"],how=\"left\",left_on=[\"customer_id\",\"index_group_no\"], right_index=True)\n  # n_buy_hist_sec\n  df = df.merge(feat[\"n_buy_hist_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_recent_sec\n  df = df.merge(feat[\"n_buy_recent_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # days_after_buy_sec\n  df = df.merge(feat[\"days_after_buy_sec\"],how=\"left\",left_on=[\"customer_id\",\"section_no\"], right_index=True)\n  # n_buy_hist_garm\n  df = df.merge(feat[\"n_buy_hist_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # n_buy_recent_garm\n  df = df.merge(feat[\"n_buy_recent_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # days_after_buy_garm\n  df = df.merge(feat[\"days_after_buy_garm\"],how=\"left\",left_on=[\"customer_id\",\"garment_group_no\"], right_index=True)\n  # rebuy_rate\n  df = df.merge(feat[\"rebuy_rate\"],how=\"left\",on=\"article_id\")\n  # sim_article\n  df = df.merge(feat[\"art_id_recent\"],how=\"left\",left_on=\"customer_id\", right_index = True)\n  sim_max,sim_sum,sim_mean = [],[],[]\n  # display(df[[\"article_id\",\"art_id_recent\"]].head())\n  tmp = df[[\"article_id\",\"art_id_recent\"]].to_pandas().values\n  for i in range(len(df)):\n      if not isinstance(tmp[i][1],list):\n        sim_max.append(0);sim_sum.append(0);sim_mean.append(0)\n      else:\n        list_sim = [cos_sim(dict_vec[tmp[i][0]],dict_vec[x]) for x in tmp[i][1]]\n        sim_max.append(max(list_sim))\n        sim_sum.append(sum(list_sim))\n        sim_mean.append(np.mean(list_sim))\n  df[\"sim_max\"] = sim_max\n  df[\"sim_sum\"] = sim_sum\n  df[\"sim_mean\"] = sim_mean\n  df = df.drop([\"art_id_recent\"], axis = 1)\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]] =\\\n  df[[\"n_buy_hist\",\"n_buy_recent\",\"n_buy_hist_all\",\"n_buy_recent_all\",\"n_buy_hist_prod\",\"n_buy_recent_prod\",\"n_buy_hist_ptype\",\"n_buy_recent_ptype\",\"n_buy_hist_graph\",\"n_buy_recent_graph\",\n      \"n_buy_hist_col\",\"n_buy_recent_col\",\"n_buy_hist_dep\",\"n_buy_recent_dep\",\"n_buy_hist_idx\",\"n_buy_recent_idx\",\"n_buy_hist_idxg\",\"n_buy_recent_idxg\",\"n_buy_hist_sec\",\"n_buy_recent_sec\",\n      \"n_buy_hist_garm\",\"n_buy_recent_garm\",\"art_buy_yesterday\",\"art_buy_recent\",\"art_buy_hist\",\"rebuy_rate\", \"sim_max\", \"sim_sum\", \"sim_mean\"]].fillna(0)\n\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]] = \\\n  df[[\"days_after_buy\",\"days_after_buy_all\",\"days_after_buy_prod\",\"days_after_buy_ptype\",\"days_after_buy_graph\",\"days_after_buy_col\",\"days_after_buy_dep\",\"days_after_buy_idx\",\n      \"days_after_buy_idxg\",\"days_after_buy_sec\",\"days_after_buy_garm\"]].fillna(10+len_hist)\n\n  df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]] = df[[\"rate_sales_channel_hist\",\"rate_sales_channel_recent\"]].fillna(1.5)\n  print(\"end add feat\")\n  return df.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:46:14.994643Z","iopub.execute_input":"2023-02-06T13:46:14.995029Z","iopub.status.idle":"2023-02-06T13:46:15.026779Z","shell.execute_reply.started":"2023-02-06T13:46:14.994997Z","shell.execute_reply":"2023-02-06T13:46:15.025613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"day_start_val=day_end_valtmp+datetime.timedelta(days=1)\nday_start = [day_start_val - datetime.timedelta(days=i-1+len_tr) for i in tr_set]\nday_end = [day_start_val - datetime.timedelta(days=i) for i in tr_set]\nday_start_rec = [x - datetime.timedelta(days=7) for x in day_start]\nday_end_rec = [x - datetime.timedelta(days=1) for x in day_start]\nday_start_hist = [x - datetime.timedelta(days=len_hist) for x in day_start]\nday_end_hist = [x - datetime.timedelta(days=1) for x in day_start]\nday_start_rec_test = day_start_val - datetime.timedelta(days=7)\nday_end_rec_test = day_start_val - datetime.timedelta(days=1)\nday_start_hist_test = day_start_val - datetime.timedelta(days=1+len_hist)\nday_end_hist_test = day_start_val - datetime.timedelta(days=1)\n\nday_end_val = day_start_val + datetime.timedelta(days=6)\n\ndf_trans, df_art, df_cust,  dict_vec = read_data(day_oldest = day_start_hist[-1])","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:46:19.465701Z","iopub.execute_input":"2023-02-06T13:46:19.466799Z","iopub.status.idle":"2023-02-06T13:47:30.326981Z","shell.execute_reply.started":"2023-02-06T13:46:19.466753Z","shell.execute_reply":"2023-02-06T13:47:30.325775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend_pred(series_cust,series_can,day_start_val,submit=False,sub_no = 0):\n    \n#     q_date = \"\"\n#     for i in range(len(day_start)):\n#       if i == 0: q_date = f\"((t_dat >= '{day_start[0]}') and (t_dat <= '{day_end[0]}'))\"\n#       else: q_date = q_date + f\" or ((t_dat >= '{day_start[i]}') and (t_dat <= '{day_end[i]}'))\"\n#     top_art_all = df_trans.query(q_date).groupby(\"article_id\")[\"t_dat\"].count().sort_values(ascending = False).index[:N].tolist()\n\n#     list_sl = list(range(0,N,N_div))\n#     if list_sl[-1] != N:list_sl.append(N)\n    df_ans = pd.DataFrame()\n\n    feat = feat_store(df_trans,series_cust.tolist(),day_start_val,day_end_val,day_start_rec_test,day_end_rec_test,day_start_hist_test,day_end_hist_test)\n\n#     for iter_art in tqdm(range(len(list_sl)-1)):\n#       top_art = top_art_all[list_sl[iter_art]:list_sl[iter_art+1]]\n    can = []\n    for i in series_can:\n        can.extend(itertools.product([i[0]],i[1:][0].split()))\n    df_test = cudf.from_pandas(pd.DataFrame(can,columns=[\"customer_id\",\"article_id\"]))\n    df_test = df_test.merge(df_art[[\"product_code\",\"product_type_no\",\"graphical_appearance_no\",\"colour_group_code\",\"department_no\",\"index_code\",\"index_group_no\",\"section_no\",\"garment_group_no\"]],how=\"left\",left_on=\"article_id\",right_index=True)\n    df_test = df_test.merge(df_cust[[\"age\",\"FN\",\"Active\",\"club_member_status\",\"fashion_news_frequency\"]],how=\"left\",left_on=\"customer_id\",right_index=True)\n\n    df_test = add_feat(df_test,day_start_val,day_end_val,day_start_rec_test,day_end_rec_test,day_start_hist_test,day_end_hist_test,feat,dict_vec)\n\n    df_pred = df_test[[\"customer_id\",\"article_id\"]].copy()\n    df_test = df_test.drop([\"customer_id\",\"product_code\",\"product_type_no\",\"department_no\"],axis=1)\n    pred = np.zeros(len(df_pred))\n    for iter_train in range(n_iter):\n        list_model = pd.read_pickle(f\"/kaggle/input/lgbmmodel/models_{iter_train}.pkl\")\n        for i in range(max(1,n_splits)):\n          list_model[i].save_model(f\"/kaggle/working/lgbm_{idx_file}.model\")\n          if obj == \"rank\":\n            with redirect_stdout(open(os.devnull, 'w')):\n              fm = ForestInference.load(filename=f\"/kaggle/working/lgbm_{idx_file}.model\",model_type='lightgbm')\n            pred += fm.predict(df_test) / (max(1,n_splits) * n_iter)\n          else:\n            with redirect_stdout(open(os.devnull, 'w')):\n              fm = ForestInference.load(filename=f\"/kaggle/working/lgbm_{idx_file}.model\",output_class=True,model_type='lightgbm')\n            pred += fm.predict_proba(df_test)[:,1] / (max(1,n_splits) * n_iter)      \n    df_pred[\"pred\"] = pred\n\n    df_ans = df_ans.append(df_pred)\n    df_ans = df_ans.sort_values([\"customer_id\",\"pred\"],ascending = False)\n    df_ans = df_ans.groupby(\"customer_id\").head(tmp_top)\n    del list_model,df_test,df_pred,pred\n    gc.collect()\n        \n    df_ans[\"article_id\"] = df_ans[\"article_id\"].astype(str).str.zfill(10)\n    if submit: df_ans.groupby(\"customer_id\").head(tmp_top).to_csv(f\"/kaggle/working/oof_{idx_file}_{day_start_val.date()}_{sub_no}.csv\",index = False)\n    else: df_ans.groupby(\"customer_id\").head(tmp_top).to_csv(path+f\"/kaggle/working/output/oof_{idx_file}_{day_start_val.date()}.csv\",index = False)\n    df_ans = df_ans.groupby(\"customer_id\").head(12)\n    if not submit:\n      mapk_val = mapk(df_agg_val_1[\"article_id\"].tolist(),df_ans.groupby(\"customer_id\")[\"article_id\"].apply(list).tolist())\n      print(f\"mapk:{mapk_val} \")\n    df_ans = df_ans.groupby(\"customer_id\")[\"article_id\"].apply(list).reset_index()\n    df_ans = df_ans.rename({'article_id':'pred'},axis=1)\n    gc.collect()\n    return df_ans","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:47:32.960263Z","iopub.execute_input":"2023-02-06T13:47:32.960748Z","iopub.status.idle":"2023-02-06T13:47:32.987105Z","shell.execute_reply.started":"2023-02-06T13:47:32.960707Z","shell.execute_reply":"2023-02-06T13:47:32.985744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\")\ndf_sub.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:47:37.685952Z","iopub.execute_input":"2023-02-06T13:47:37.686673Z","iopub.status.idle":"2023-02-06T13:47:43.307482Z","shell.execute_reply.started":"2023-02-06T13:47:37.686632Z","shell.execute_reply":"2023-02-06T13:47:43.306387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nsize_block = 5000\n# df_sub = df_sub[:300]\nlist_slice = list(range(0,len(df_sub),size_block))\nif list_slice[-1] != len(df_sub):list_slice.append(len(df_sub))","metadata":{"execution":{"iopub.status.busy":"2023-02-06T12:32:20.410319Z","iopub.execute_input":"2023-02-06T12:32:20.411483Z","iopub.status.idle":"2023-02-06T12:32:20.420388Z","shell.execute_reply.started":"2023-02-06T12:32:20.411444Z","shell.execute_reply":"2023-02-06T12:32:20.419164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nsize_block = 10000\n# df_sub = df_sub[:300]\nlist_slice = list(range(0,len(df_sub),size_block))\nif list_slice[-1] != len(df_sub):list_slice.append(len(df_sub))\n# os.makedirs(path+f\"output/div{idx_file}\",exist_ok = True)\nfor i in tqdm(range(len(list_slice)-1)):\n  time.sleep(1)\n  if not os.path.exists(f\"/kaggle/input/lgbmmodel/submissiony_{i}.csv\"):\n    df_sub_0 = df_sub[list_slice[i]:list_slice[i+1]].copy()\n    df_can_0 = candidates[list_slice[i]:list_slice[i+1]].copy()\n    df_ans = recommend_pred(df_sub_0[\"customer_id\"],df_can_0,day_end_valtmp+datetime.timedelta(days=1),submit=True,sub_no = i)\n    df_ans[\"prediction\"] = df_ans[\"pred\"].apply(lambda x:' '.join(x))\n    df_ans[[\"customer_id\",\"prediction\"]].to_csv(f\"/kaggle/working/submissiony_{i}.csv\",index = False)\n    del df_sub_0,df_ans\n    gc.collect()\n  else:\n    print(f\"submission_{i}.csv is found, so skip to next step!\")\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-06T13:47:48.399110Z","iopub.execute_input":"2023-02-06T13:47:48.399494Z","iopub.status.idle":"2023-02-06T15:57:22.642572Z","shell.execute_reply.started":"2023-02-06T13:47:48.399460Z","shell.execute_reply":"2023-02-06T15:57:22.641469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"can = []\nfor i in candidates:\n    can.extend(itertools.product([i[0]],i[1:][0].split()))\ndf_test = cudf.from_pandas(pd.DataFrame(can,columns=[\"customer_id\",\"article_id\"]))","metadata":{"execution":{"iopub.status.busy":"2023-02-06T12:01:39.256072Z","iopub.execute_input":"2023-02-06T12:01:39.256434Z","iopub.status.idle":"2023-02-06T12:01:39.278259Z","shell.execute_reply.started":"2023-02-06T12:01:39.256403Z","shell.execute_reply":"2023-02-06T12:01:39.277352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_article.corr()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:10:35.667459Z","iopub.execute_input":"2022-12-26T16:10:35.667825Z","iopub.status.idle":"2022-12-26T16:10:35.716907Z","shell.execute_reply.started":"2022-12-26T16:10:35.667794Z","shell.execute_reply":"2022-12-26T16:10:35.715867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\nfig = px.imshow(df_article.corr())","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:11:30.490656Z","iopub.execute_input":"2022-12-26T16:11:30.491573Z","iopub.status.idle":"2022-12-26T16:11:31.983778Z","shell.execute_reply.started":"2022-12-26T16:11:30.491531Z","shell.execute_reply":"2022-12-26T16:11:31.982781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in df_article.columns:\n    print(c)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T02:53:23.893034Z","iopub.execute_input":"2022-12-27T02:53:23.894004Z","iopub.status.idle":"2022-12-27T02:53:23.901824Z","shell.execute_reply.started":"2022-12-27T02:53:23.893965Z","shell.execute_reply":"2022-12-27T02:53:23.900751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_article['colour_group_name'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:27:45.367858Z","iopub.execute_input":"2022-12-26T16:27:45.368341Z","iopub.status.idle":"2022-12-26T16:27:45.391059Z","shell.execute_reply.started":"2022-12-26T16:27:45.368309Z","shell.execute_reply":"2022-12-26T16:27:45.389232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_article['perceived_colour_value_name'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:28:47.169973Z","iopub.execute_input":"2022-12-26T16:28:47.170726Z","iopub.status.idle":"2022-12-26T16:28:47.193223Z","shell.execute_reply.started":"2022-12-26T16:28:47.170659Z","shell.execute_reply":"2022-12-26T16:28:47.190503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_article['perceived_colour_master_name'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:27:38.160293Z","iopub.execute_input":"2022-12-26T16:27:38.161188Z","iopub.status.idle":"2022-12-26T16:27:38.178068Z","shell.execute_reply.started":"2022-12-26T16:27:38.161141Z","shell.execute_reply":"2022-12-26T16:27:38.177165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub0 = pd.read_csv(\"/kaggle/working/submissiony_0.csv\")    \nfor i in range(137):\n    subx = pd.read_csv(\"/kaggle/working/submissiony_\"+ str(i+1)+\".csv\")\n    sub0 = pd.concat([sub0, subx], axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:06:32.460865Z","iopub.execute_input":"2023-02-06T16:06:32.461283Z","iopub.status.idle":"2023-02-06T16:06:40.305087Z","shell.execute_reply.started":"2023-02-06T16:06:32.461248Z","shell.execute_reply":"2023-02-06T16:06:40.303916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub0.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:08:24.222406Z","iopub.execute_input":"2023-02-06T16:08:24.223496Z","iopub.status.idle":"2023-02-06T16:08:29.091684Z","shell.execute_reply.started":"2023-02-06T16:08:24.223450Z","shell.execute_reply":"2023-02-06T16:08:29.089602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# subx = sub0\nsubx.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:27:48.133042Z","iopub.execute_input":"2023-02-06T16:27:48.133451Z","iopub.status.idle":"2023-02-06T16:27:48.144694Z","shell.execute_reply.started":"2023-02-06T16:27:48.133417Z","shell.execute_reply":"2023-02-06T16:27:48.143638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subxx = subx['prediction']","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:17:04.531395Z","iopub.execute_input":"2023-02-06T16:17:04.531805Z","iopub.status.idle":"2023-02-06T16:17:04.536630Z","shell.execute_reply.started":"2023-02-06T16:17:04.531765Z","shell.execute_reply":"2023-02-06T16:17:04.535571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cus = ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor i in range(len(subxx)):\n    ls = subxx.values[i].split()\n    ls = ['0' + pred for pred in ls]\n    ls = [pred[:-2] for pred in ls] \n    preds.append(ls)","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:26:37.530667Z","iopub.execute_input":"2023-02-06T16:26:37.531718Z","iopub.status.idle":"2023-02-06T16:26:46.383169Z","shell.execute_reply.started":"2023-02-06T16:26:37.531677Z","shell.execute_reply":"2023-02-06T16:26:46.382037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = [\" \".join(pred) for pred in preds]","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:26:49.455211Z","iopub.execute_input":"2023-02-06T16:26:49.455630Z","iopub.status.idle":"2023-02-06T16:26:50.619084Z","shell.execute_reply.started":"2023-02-06T16:26:49.455591Z","shell.execute_reply":"2023-02-06T16:26:50.618001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(preds)","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:27:02.293425Z","iopub.execute_input":"2023-02-06T16:27:02.294290Z","iopub.status.idle":"2023-02-06T16:27:02.304675Z","shell.execute_reply.started":"2023-02-06T16:27:02.294239Z","shell.execute_reply":"2023-02-06T16:27:02.303749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subb = subx.drop('prediction', axis=1)\nsubb['prediction'] = preds","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:28:39.400263Z","iopub.execute_input":"2023-02-06T16:28:39.400696Z","iopub.status.idle":"2023-02-06T16:28:39.568835Z","shell.execute_reply.started":"2023-02-06T16:28:39.400659Z","shell.execute_reply":"2023-02-06T16:28:39.567828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subb.to_csv(\"submissionxx.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-06T16:29:21.786368Z","iopub.execute_input":"2023-02-06T16:29:21.786797Z","iopub.status.idle":"2023-02-06T16:29:26.384752Z","shell.execute_reply.started":"2023-02-06T16:29:21.786755Z","shell.execute_reply":"2023-02-06T16:29:26.383695Z"},"trusted":true},"execution_count":null,"outputs":[]}]}