{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"collapsed_sections":[],"machine_shape":"hm","name":"H&M 17th place solution.ipynb","provenance":[]},"gpuClass":"standard","accelerator":"GPU","kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Solution Summary**\n\nMy Solution: [discussion](https://www.kaggle.com/competitions/h-and-m-personalized-fashion-recommendations/discussion/324595)\n\nI solved this task as a binary classification problem to predict whether or not each product was bought in the last week of `transactions_train.csv`. To create features, I used\nthe purchase data from the previous weeks.\n\n","metadata":{"id":"Dbg911SON80e"}},{"cell_type":"markdown","source":"**about this format**\n\nOn both Kaggle and Colab, training and inference can be run on this single notebook!\n\n1. Training\n \n Set `CFG.train = True` and run.\n\n2. Inference\n\n Set `CFG.train = False` and run.\n\nThe duration of `transactions_df` and `target_df` is automatically changed by True/False of `CFG.train`.","metadata":{"id":"uLXUw84cOWbT"}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"id":"TLNV_u2j3hk0","outputId":"d22f3a66-218a-4e58-c814-4eb471cbaa0b","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:11:40.539766Z","iopub.execute_input":"2025-12-04T18:11:40.540082Z","iopub.status.idle":"2025-12-04T18:11:40.834474Z","shell.execute_reply.started":"2025-12-04T18:11:40.540047Z","shell.execute_reply":"2025-12-04T18:11:40.833553Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Libraries","metadata":{"id":"OL4dWeDU3GLB"}},{"cell_type":"code","source":"# ====================================================\n# import libraries\n# ====================================================\n\nimport warnings\nwarnings.filterwarnings('ignore')\nimport math\nimport json\nimport os\nimport numpy as np\nimport pandas as pd\nimport gc\nimport time\nimport torch\nimport random\nimport sys\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport itertools\nimport collections\nimport joblib\nfrom gensim.models import word2vec\nimport datetime\nfrom datetime import timedelta\nimport hashlib\nimport matplotlib.pyplot as plt\nfrom sklearn.base import BaseEstimator, TransformerMixin\nimport numpy as np\nfrom collections import defaultdict, Counter\nfrom PIL import Image\nfrom pathlib import Path\nimport pickle\nfrom contextlib import contextmanager\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import RidgeCV\nimport lightgbm as lgb\nimport typing as tp\nfrom sklearn.preprocessing import LabelEncoder\nimport seaborn as sns\nfrom sklearn.utils.class_weight import compute_sample_weight\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.decomposition import TruncatedSVD\nimport torch\nfrom logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n\ntqdm.pandas()\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\n\nif torch.cuda.is_available():\n device = torch.device('cuda')\nelse:\n device = torch.device('cpu')\n \nprint(f'Using device: {device}')","metadata":{"id":"ZAX1B3vO3GLF","outputId":"44e93739-83b8-4155-d782-7464a991cf27","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:11:40.836296Z","iopub.execute_input":"2025-12-04T18:11:40.836916Z","iopub.status.idle":"2025-12-04T18:12:12.157541Z","shell.execute_reply.started":"2025-12-04T18:11:40.836889Z","shell.execute_reply":"2025-12-04T18:12:12.156763Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Config","metadata":{"id":"low8Ncyj3GLG"}},{"cell_type":"code","source":"# ==============================================\n# Config\n# ==============================================\n\nclass CFG:\n colab = \"google.colab\" in sys.modules\n exp = \"120\"\n train = True\n submit = False\n api_path = '/content/drive/My Drive/kaggle.json'\n seed = 42\n used_fold = [0,1,2,3,4]\n fold = 5\n part = 0\n target = \"target\"\n \nval_start_date = '2020-09-16'","metadata":{"id":"CpIy7UsA3GLG","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:12.158665Z","iopub.execute_input":"2025-12-04T18:12:12.159426Z","iopub.status.idle":"2025-12-04T18:12:12.163599Z","shell.execute_reply.started":"2025-12-04T18:12:12.159401Z","shell.execute_reply":"2025-12-04T18:12:12.162860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==============================================\n# Catboost parameters\n# ==============================================\n\nDROP_COLS = [\n \"customer_id\",\n \"article_id\",\n #\"product_code\",\n #\"product_type_no\",\n #\"graphical_appearance_no\",\n #\"colour_group_code\",\n #\"index_group_no\",\n #\"section_no\",\n #\"garment_group_no\",\n]\n\n\nCATEGORICAL_COLS = [\n # ---------------------------------------------------------\n # Article features\n # ---------------------------------------------------------\n #\"product_code\",\n #\"product_type_no\",\n #\"graphical_appearance_no\",\n #\"colour_group_code\",\n #\"index_group_no\",\n #\"section_no\",\n #\"garment_group_no\",\n # ---------------------------------------------------------\n # Customer features\n # ---------------------------------------------------------\n #\"FN\",\n #\"Active\",\n #\"club_member_status\",\n #\"fashion_news_frequency\",\n #\"postal_code\",\n]\n\nPARAMS = {\n 'loss_function': 'Logloss',\n 'learning_rate': 0.02,\n 'max_depth': 6,\n 'random_state': CFG.seed,\n 'thread_count': 2,\n 'task_type': 'GPU',\n 'scale_pos_weight': 100,\n 'num_boost_round': 15000,\n}","metadata":{"id":"gSuGMbVUAu3m","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:12.165305Z","iopub.execute_input":"2025-12-04T18:12:12.165605Z","iopub.status.idle":"2025-12-04T18:12:12.175013Z","shell.execute_reply.started":"2025-12-04T18:12:12.165578Z","shell.execute_reply":"2025-12-04T18:12:12.174263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG.colab:\n print(\"==============================================\")\n print(\"This environment is Google Colab\")\n print(\"==============================================\")\n\n # Google Drive\n from google.colab import drive, files\n drive.mount('/content/drive')\n %cd \"drive/My Drive/h-and-m/\"\n\n # Kaggle API\n f = open(CFG.api_path, 'r')\n json_data = json.load(f) \n os.environ[\"KAGGLE_USERNAME\"] = json_data[\"username\"]\n os.environ[\"KAGGLE_KEY\"] = json_data[\"key\"]\n\n # Directory Setting\n if not os.path.exists(f\"output/exp{CFG.exp}/\"):\n os.makedirs(f\"output/exp{CFG.exp}/\")\n\n DATA_DIR = \"input/\"\n OUTPUT_DIR = f\"output/exp{CFG.exp}/\"\n MODEL_DIR = OUTPUT_DIR\n\n if not os.path.exists(OUTPUT_DIR + f\"exp{CFG.exp}/\"):\n os.makedirs(OUTPUT_DIR + f\"exp{CFG.exp}/\")\n \n # Data Loading\n if not os.path.isfile(os.path.join(DATA_DIR, \"transactions_train.csv.zip\")):\n !kaggle competitions download -c h-and-m-personalized-fashion-recommendations -p $DATA_DIR\n \n # Libraries\n !pip install -q catboost\n !pip install -q transformers\n \nelse:\n print(\"==============================================\")\n print(\" This environment is Kaggle Notebook\")\n print(\"==============================================\")\n\n # Directory Setting\n OUTPUT_DIR = \"./\"\n DATA_DIR = \"../input/h-and-m-personalized-fashion-recommendations/\"\n\n# ====================================================\n# import libraries2\n# ====================================================\n\nfrom catboost import CatBoost, Pool\nimport transformers","metadata":{"id":"HV9Do4Qs4ida","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:12.175861Z","iopub.execute_input":"2025-12-04T18:12:12.176112Z","iopub.status.idle":"2025-12-04T18:12:13.677679Z","shell.execute_reply.started":"2025-12-04T18:12:12.176085Z","shell.execute_reply":"2025-12-04T18:12:13.676743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper Functions","metadata":{"id":"lGwvMFZXxj2E"}},{"cell_type":"code","source":"def get_swap_dict(d):\n return {v: k for k, v in d.items()}","metadata":{"id":"TrirIqxMhTAT","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.678589Z","iopub.execute_input":"2025-12-04T18:12:13.680048Z","iopub.status.idle":"2025-12-04T18:12:13.683695Z","shell.execute_reply.started":"2025-12-04T18:12:13.680024Z","shell.execute_reply":"2025-12-04T18:12:13.682849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n start_mem = df.memory_usage().sum() / 1024**2 \n for col in df.columns:\n col_type = df[col].dtypes\n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64) \n end_mem = df.memory_usage().sum() / 1024**2\n if verbose: print('Memory usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n return df","metadata":{"id":"VeqV1MGS5a_e","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.684564Z","iopub.execute_input":"2025-12-04T18:12:13.684916Z","iopub.status.idle":"2025-12-04T18:12:13.696999Z","shell.execute_reply.started":"2025-12-04T18:12:13.684890Z","shell.execute_reply":"2025-12-04T18:12:13.696345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compare_vectors(v1, v2):\n intersection = len(set(v1) & set(v2))\n denominator = np.sqrt(len(v1) * len(v2))\n return intersection / denominator\n\ndef flatten(x):\n return [e for i in x for e in i]","metadata":{"id":"xO1GfvYY61qS","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.697772Z","iopub.execute_input":"2025-12-04T18:12:13.698040Z","iopub.status.idle":"2025-12-04T18:12:13.711277Z","shell.execute_reply.started":"2025-12-04T18:12:13.698016Z","shell.execute_reply":"2025-12-04T18:12:13.710646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@contextmanager\ndef timer(name: str):\n t0 = time.time()\n print(f\"[{name}] start\")\n yield\n msg = f\"[{name}] done in {time.time() - t0:.0f} s\"\n print(msg)","metadata":{"id":"-kZhIkMGOpCg","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.711909Z","iopub.execute_input":"2025-12-04T18:12:13.712220Z","iopub.status.idle":"2025-12-04T18:12:13.722135Z","shell.execute_reply.started":"2025-12-04T18:12:13.712192Z","shell.execute_reply":"2025-12-04T18:12:13.721492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed=42):\n random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\ndef hashfxn(x):\n return int(hashlib.md5(str(x).encode()).hexdigest(), 16)\n\nseed_everything(CFG.seed)","metadata":{"id":"QsnGPtqmLUfe","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.724568Z","iopub.execute_input":"2025-12-04T18:12:13.724847Z","iopub.status.idle":"2025-12-04T18:12:13.736925Z","shell.execute_reply.started":"2025-12-04T18:12:13.724830Z","shell.execute_reply":"2025-12-04T18:12:13.736189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def init_logger(log_file=OUTPUT_DIR+'train.log'):\n logger = getLogger(__name__)\n logger.setLevel(INFO)\n logger.hasHandlers()\n handler1 = StreamHandler()\n handler1.setFormatter(Formatter(\"%(message)s\"))\n handler2 = FileHandler(filename=log_file)\n handler2.setFormatter(Formatter(\"%(message)s\"))\n logger.addHandler(handler1)\n logger.addHandler(handler2)\n return logger\n\nLOGGER = init_logger()","metadata":{"id":"HHkDY3YXEx1X","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.737713Z","iopub.execute_input":"2025-12-04T18:12:13.737958Z","iopub.status.idle":"2025-12-04T18:12:13.743759Z","shell.execute_reply.started":"2025-12-04T18:12:13.737935Z","shell.execute_reply":"2025-12-04T18:12:13.743161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ====================================================\n# Calculate MAP@12\n# ====================================================\n# Reference: https://www.kaggle.com/kaerunantoka/h-m-how-to-calculate-map-12\n\ndef apk(actual, predicted, k=12):\n \"\"\"\n Computes the average precision at k.\n This function computes the average prescision at k between two lists of\n items.\n Parameters\n ----------\n actual : list\n A list of elements that are to be predicted (order doesn't matter)\n predicted : list\n A list of predicted elements (order does matter)\n k : int, optional\n The maximum number of predicted elements\n Returns\n -------\n score : double\n The average precision at k over the input lists\n \"\"\"\n if len(predicted)>k:\n predicted = predicted[:k]\n\n score = 0.0\n num_hits = 0.0\n\n for i,p in enumerate(predicted):\n if p in actual and p not in predicted[:i]:\n num_hits += 1.0\n score += num_hits / (i+1.0)\n\n if not actual:\n return 0.0\n\n return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n \"\"\"\n Computes the mean average precision at k.\n This function computes the mean average prescision at k between two lists\n of lists of items.\n Parameters\n ----------\n actual : list\n A list of lists of elements that are to be predicted \n (order doesn't matter in the lists)\n predicted : list\n A list of lists of predicted elements\n (order matters in the lists)\n k : int, optional\n The maximum number of predicted elements\n Returns\n -------\n score : double\n The mean average precision at k over the input lists\n \"\"\"\n return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"id":"hAS43i-wAklN","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.744498Z","iopub.execute_input":"2025-12-04T18:12:13.745005Z","iopub.status.idle":"2025-12-04T18:12:13.757246Z","shell.execute_reply.started":"2025-12-04T18:12:13.744981Z","shell.execute_reply":"2025-12-04T18:12:13.756660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ARTICLE_FEATURES = [\"product_code\", \"product_type_no\", \"graphical_appearance_no\", \"colour_group_code\", \"index_group_no\", \"section_no\", \"garment_group_no\"]","metadata":{"id":"Hb1dClzAyf6t","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.758003Z","iopub.execute_input":"2025-12-04T18:12:13.758240Z","iopub.status.idle":"2025-12-04T18:12:13.771128Z","shell.execute_reply.started":"2025-12-04T18:12:13.758223Z","shell.execute_reply":"2025-12-04T18:12:13.770504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading","metadata":{"id":"CPrIe0GM3GLH"}},{"cell_type":"code","source":"with timer(\"Data Loading\"):\n transactions_df = reduce_mem_usage(pd.read_csv(DATA_DIR + \"transactions_train.csv\"))\n articles_df = reduce_mem_usage(pd.read_csv(DATA_DIR + \"articles.csv\"))\n customers_df = reduce_mem_usage(pd.read_csv(DATA_DIR + \"customers.csv\"))\n sub_df = reduce_mem_usage(pd.read_csv(DATA_DIR + \"sample_submission.csv\"))\n transactions_df['customer_id'] = transactions_df['customer_id'].apply(lambda x: int(x, 16))\n sub_df['customer_id'] = sub_df['customer_id'].apply(lambda x: int(x[-16:], 16))","metadata":{"id":"OagzUEps3GLH","outputId":"5e6a1e65-64f8-4a27-efc2-b94e5feedf60","trusted":true,"execution":{"iopub.status.busy":"2025-12-04T18:12:13.771826Z","iopub.execute_input":"2025-12-04T18:12:13.772018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG.train:\n target_df = transactions_df.query(f\"t_dat >= '{val_start_date}'\").reset_index(drop=True)\n transactions_df = transactions_df.query(f\"t_dat < '{val_start_date}'\").reset_index(drop=True)\nelse:\n target_df = sub_df.copy()\n del sub_df\n gc.collect()\n\ndisplay(transactions_df.head(5))\ndisplay(target_df.head(5))","metadata":{"id":"buJpnANHwUzy","outputId":"e57ae6cc-f8c9-429f-d584-529509ae5125","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def original_df_features(original_df):\n original_df[\"t_dat\"] = pd.to_datetime(original_df[\"t_dat\"])\n \n original_df[\"week\"] = ((original_df.t_dat.max() - original_df.t_dat).dt.days // 7).astype(\"int8\")\n original_df[\"day\"] = ((original_df.t_dat.max() - original_df.t_dat).dt.days).astype(\"int16\")\n original_df[\"sales_channel_id\"] = original_df[\"sales_channel_id\"]-1\n return original_df","metadata":{"id":"RDKNxr84ZuMF","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_df(original_df):\n\n # ==========================================================================\n # Step1: Top300 popular items\n # ==========================================================================\n\n tmp1 = pd.DataFrame(target_df[\"customer_id\"].unique()).rename(columns={0: \"customer_id\"})\n\n # For memory measures, the customer is divided into 14 parts when inferring.\n if not(CFG.train):\n a = 100000*CFG.part\n b = 100000*(CFG.part+1)\n tmp1 = tmp1.query(f\"{a}<=customer_id<{b}\").reset_index(drop=True)\n\n if CFG.train:\n tmp2 = pd.DataFrame(target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300])\n else:\n tmp2 = pd.DataFrame(original_df.query(f\"t_dat >= '{val_start_date}'\").groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300])\n\n tmp1[\"tmp\"] = 0\n tmp2[\"tmp\"] = 0\n\n df = pd.merge(tmp1, tmp2, on=\"tmp\", how=\"outer\")\n\n # ==========================================================================\n # Step2: items with the same product code as previously purchased items\n # ==========================================================================\n\n tmp3 = original_df[[\"article_id\", \"customer_id\"]].reset_index(drop=True)\n tmp3 = tmp3[tmp3[\"customer_id\"].isin(tmp1[\"customer_id\"])].reset_index(drop=True)\n tmp3 = pd.merge(tmp3, articles_df[[\"article_id\", \"product_code\"]], on=\"article_id\", how=\"left\")\n tmp3 = tmp3.drop_duplicates([\"customer_id\", \"product_code\"])[[\"customer_id\", \"product_code\"]].reset_index(drop=True)\n\n tmp4 = articles_df[articles_df[\"product_code\"].isin(tmp3[\"product_code\"].unique())][[\"article_id\", \"product_code\"]].reset_index(drop=True)\n tmp4 = pd.merge(tmp3, tmp4, on=\"product_code\", how=\"outer\")[[\"customer_id\", \"article_id\"]]\n\n if CFG.train:\n tmp4 = tmp4[tmp4[\"article_id\"].isin(target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index)].reset_index(drop=True)\n else:\n tmp4 = tmp4[tmp4[\"article_id\"].isin(original_df.query(f\"t_dat >= '{val_start_date}'\").groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index)].reset_index(drop=True)\n\n df = pd.concat([df, tmp4]).drop_duplicates([\"customer_id\", \"article_id\"], keep=\"last\").reset_index(drop=True)\n\n # ==========================================================================\n # Make label\n # ==========================================================================\n if CFG.train:\n tmp5 = target_df.drop_duplicates([\"customer_id\", \"article_id\"])[[\"article_id\", \"customer_id\"]].reset_index(drop=True)\n tmp5[\"target\"] = 1\n\n df = pd.merge(df, tmp5, on=[\"customer_id\", \"article_id\"], how=\"left\")\n df[\"target\"] = df[\"target\"].fillna(0).astype(\"int8\")\n\n del tmp5\n\n del tmp1, tmp2, tmp3, tmp4, df[\"tmp\"]\n gc.collect()\n\n return df","metadata":{"id":"Ju2R2H5wMToS","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_article_information(df):\n df = pd.merge(df, articles_df[[\"article_id\"]+ARTICLE_FEATURES], on=\"article_id\", how=\"left\")\n return df","metadata":{"id":"hs8hsCrdRbbY","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_customer_information(df):\n df = pd.merge(df, customers_df[[\"customer_id\", \"age\"]], on=\"customer_id\", how=\"left\")\n return df","metadata":{"id":"jKSpW8gSWuKZ","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cf_features(original_df, df):\n def get_sim_item(df, user_col, item_col, use_iif=False): \n user_item_ = df.groupby(user_col)[item_col].agg(set).reset_index() \n user_item_dict = dict(zip(user_item_[user_col], user_item_[item_col]))\n\n del user_item_\n gc.collect()\n \n sim_item = {}\n item_cnt = defaultdict(int) \n for user, items in tqdm(user_item_dict.items()): \n for i in items:\n item_cnt[i] += 1\n sim_item.setdefault(i, {})\n for relate_item in items: \n if not relate_item in popular_item: \n continue \n sim_item[i].setdefault(relate_item, 0) \n if not use_iif: \n sim_item[i][relate_item] += 1 \n else: \n sim_item[i][relate_item] += 1 / math.log(1 + len(items)) \n sim_item_corr = sim_item.copy() \n for i, related_items in tqdm(sim_item.items()): \n for j, cij in related_items.items(): \n sim_item_corr[i][j] = cij/math.sqrt(item_cnt[i]*item_cnt[j]) \n \n return sim_item_corr, user_item_dict\n\n def recommend(sim_item_corr, user_item_dict, user_id): \n rank = {}\n interacted_items = user_item_dict[user_id] \n for i in interacted_items:\n for j, wij in sorted(sim_item_corr[i].items(), key=lambda d: d[1], reverse=True): \n rank.setdefault(j, 0) \n rank[j] += wij \n return sorted(rank.items(), key=lambda d: d[1], reverse=True)\n \n popular_item = target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n item_sim_list, user_item = get_sim_item(transactions_df, user_col = 'customer_id', item_col = 'article_id', use_iif=True)\n\n # Lấy danh sách unique customers và chia thành 14 phần\n unique_customers = transactions_df[\"customer_id\"].unique()\n total_customers = len(unique_customers)\n customers_per_part = total_customers // 14\n \n for i in range(14):\n # Tính chỉ số bắt đầu và kết thúc cho phần thứ i\n start_idx = i * customers_per_part\n if i == 13: # Phần cuối cùng lấy hết customers còn lại\n end_idx = total_customers\n else:\n end_idx = (i + 1) * customers_per_part\n \n # Lấy customers cho phần hiện tại\n selected_customers = unique_customers[start_idx:end_idx]\n \n new_df = []\n for c in tqdm(selected_customers):\n if c in user_item: # Kiểm tra customer có trong user_item dict\n tmp = pd.DataFrame(recommend(item_sim_list, user_item, c))\n tmp[\"customer_id\"] = c\n new_df.append(tmp)\n \n if len(new_df) > 0: # Chỉ concat nếu có data\n new_df = pd.concat(new_df).reset_index(drop=True)\n new_df = new_df.rename(columns={0: \"article_id\", 1: \"cf_score\"})\n with open(OUTPUT_DIR + f\"cf_score_test{i}.pickle\", mode=\"wb\") as f:\n pickle.dump(new_df, f, protocol=4)\n\n for feature in [\"article_id\"]+ARTICLE_FEATURES:\n if CFG.train:\n with open(OUTPUT_DIR + f\"cf_score_{feature}.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n else:\n with open(OUTPUT_DIR + f\"cf_score_{feature}_test{CFG.part}.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n \n df = pd.merge(df, cf_score_df, on=[feature, \"customer_id\"], how=\"left\")\n df[f\"cf_score_{feature}\"] = df[f\"cf_score_{feature}\"].astype(\"float16\")\n \n del cf_score_df\n gc.collect()\n \n if CFG.train:\n with open(OUTPUT_DIR + f\"cf_score_article_with_channel.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n else:\n with open(OUTPUT_DIR + f\"cf_score_article_with_channel_test{CFG.part}.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n \n cf_score_df[\"sales_channel_id\"] = cf_score_df[\"article_id\"].apply(lambda x: int(x[-1]))\n cf_score_df[\"article_id\"] = cf_score_df[\"article_id\"].apply(lambda x: int(x[:-2]))\n\n tmp1 = cf_score_df.pivot_table(values=['cf_score_article_with_channel'], index=['customer_id', 'article_id'], columns=['sales_channel_id'], aggfunc='sum').reset_index()\n tmp1.columns = [\"customer_id\", \"article_id\", \"cf_score_article_offline\", \"cf_score_article_online\"]\n tmp1 = tmp1.fillna(0)\n\n tmp2 = cf_score_df.groupby([\"customer_id\", \"article_id\"])[\"cf_score_article_with_channel\"].agg(\"sum\").reset_index()\n\n df = pd.merge(df, tmp1, on=[\"article_id\", \"customer_id\"], how=\"left\")\n df = pd.merge(df, tmp2, on=[\"article_id\", \"customer_id\"], how=\"left\")\n\n df[\"cf_score_article_with_channel\"] = df[\"cf_score_article_with_channel\"].astype(\"float16\")\n df[\"cf_score_article_offline\"] = df[\"cf_score_article_offline\"].astype(\"float16\")\n df[\"cf_score_article_online\"] = df[\"cf_score_article_online\"].astype(\"float16\")\n \n return df","metadata":{"id":"5Tvz0ytKcAKe","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cf_features2(original_df, df):\n def get_sim_item2(df, user_col, item_col, use_iif=False): \n user_item_ = df.groupby(user_col)[item_col].agg(set).reset_index() \n user_item_dict = dict(zip(user_item_[user_col], user_item_[item_col]))\n\n del user_item_\n gc.collect()\n \n sim_item = {}\n item_cnt = defaultdict(int) \n for user, items in tqdm(user_item_dict.items()): \n for i in items:\n item_cnt[i] += 1\n sim_item.setdefault(i, {})\n for relate_item in items: \n if not relate_item in popular_item: \n continue \n sim_item[i].setdefault(relate_item, 0) \n if not use_iif: \n sim_item[i][relate_item] += 1 \n else: \n sim_item[i][relate_item] += 1 / math.log(1 + len(items)) \n sim_item_corr = sim_item.copy() \n for i, related_items in tqdm(sim_item.items()): \n for j, cij in related_items.items(): \n sim_item_corr[i][j] = cij/item_cnt[i] \n \n return sim_item_corr, user_item_dict\n\n def recommend(sim_item_corr, user_item_dict, user_id): \n rank = {}\n interacted_items = user_item_dict[user_id] \n for i in interacted_items:\n for j, wij in sorted(sim_item_corr[i].items(), key=lambda d: d[1], reverse=True): \n rank.setdefault(j, 0) \n rank[j] += wij \n return sorted(rank.items(), key=lambda d: d[1], reverse=True)\n \n popular_item = target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n item_sim_list, user_item = get_sim_item2(transactions_df, user_col = 'customer_id', item_col = 'article_id', use_iif=False)\n\n for i in range(14):\n a = 100000*i\n b = 100000*(i+1)\n new_df = []\n for c in tqdm(transactions_df.query(f\"{a}<=customer_id<{b}\")[\"customer_id\"].unique()):\n tmp = pd.DataFrame(recommend(item_sim_list, user_item, c))\n tmp[\"customer_id\"] = c\n new_df.append(tmp)\n new_df = pd.concat(new_df).reset_index(drop=True)\n new_df = new_df.rename(columns={0: \"article_id\", 1: \"cf_score\"})\n with open(OUTPUT_DIR + f\"cf_score_test{i}.pickle\", mode=\"wb\") as f:\n pickle.dump(new_df, f, protocol=4)\n\n if CFG.train:\n with open(OUTPUT_DIR + \"cf_score_v2.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n else:\n with open(OUTPUT_DIR + f\"cf_score_v2_test{CFG.part}.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n \n cf_score_df = cf_score_df.rename(columns={\"cf_score\": \"cf_score2\"})\n \n df = pd.merge(df, cf_score_df, on=[\"article_id\", \"customer_id\"], how=\"left\")\n df[\"cf_score2\"] = df[\"cf_score2\"].astype(\"float16\")\n \n del cf_score_df\n gc.collect()\n \n return df","metadata":{"id":"D0IpQ5zBO2AY","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_sim_item2(df, user_col, item_col, use_iif=False): \n user_item_ = df.groupby(user_col)[item_col].agg(set).reset_index() \n user_item_dict = dict(zip(user_item_[user_col], user_item_[item_col]))\n\n del user_item_\n gc.collect()\n \n sim_item = {}\n item_cnt = defaultdict(int) \n for user, items in tqdm(user_item_dict.items()): \n for i in items:\n item_cnt[i] += 1\n sim_item.setdefault(i, {})\n for relate_item in items: \n if not relate_item in popular_item: \n continue \n sim_item[i].setdefault(relate_item, 0) \n if not use_iif: \n sim_item[i][relate_item] += 1 \n else: \n sim_item[i][relate_item] += 1 / math.log(1 + len(items)) \n sim_item_corr = sim_item.copy() \n for i, related_items in tqdm(sim_item.items()): \n for j, cij in related_items.items(): \n sim_item_corr[i][j] = cij/item_cnt[i] \n\n return sim_item_corr, user_item_dict","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cf_features3(original_df, df):\n def get_sim_item3(df, user_col, item_col, use_iif=False): \n user_item_ = df.groupby(user_col)[item_col].agg(set).reset_index() \n user_item_dict = dict(zip(user_item_[user_col], user_item_[item_col]))\n\n del user_item_\n gc.collect()\n \n sim_item = {}\n item_cnt = defaultdict(int) \n for user, items in tqdm(user_item_dict.items()): \n for i in items:\n item_cnt[i] += 1\n sim_item.setdefault(i, {})\n for relate_item in items: \n if not relate_item in popular_item: \n continue \n sim_item[i].setdefault(relate_item, 0) \n if not use_iif: \n sim_item[i][relate_item] += 1 \n else: \n sim_item[i][relate_item] += 1 / math.log(1 + len(items)) \n sim_item_corr = sim_item.copy() \n for i, related_items in tqdm(sim_item.items()): \n for j, cij in related_items.items(): \n sim_item_corr[i][j] = cij/item_cnt[j]\n \n return sim_item_corr, user_item_dict\n\n def recommend(sim_item_corr, user_item_dict, user_id): \n rank = {}\n interacted_items = user_item_dict[user_id] \n for i in interacted_items:\n for j, wij in sorted(sim_item_corr[i].items(), key=lambda d: d[1], reverse=True): \n rank.setdefault(j, 0) \n rank[j] += wij \n return sorted(rank.items(), key=lambda d: d[1], reverse=True)\n \n popular_item = target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n item_sim_list, user_item = get_sim_item2(transactions_df, user_col = 'customer_id', item_col = 'article_id', use_iif=False)\n\n for i in range(14):\n a = 100000*i\n b = 100000*(i+1)\n new_df = []\n for c in tqdm(transactions_df.query(f\"{a}<=customer_id<{b}\")[\"customer_id\"].unique()):\n tmp = pd.DataFrame(recommend(item_sim_list, user_item, c))\n tmp[\"customer_id\"] = c\n new_df.append(tmp)\n new_df = pd.concat(new_df).reset_index(drop=True)\n new_df = new_df.rename(columns={0: \"article_id\", 1: \"cf_score\"})\n with open(OUTPUT_DIR + f\"cf_score_test{i}.pickle\", mode=\"wb\") as f:\n pickle.dump(new_df, f, protocol=4)\n\n if CFG.train:\n with open(OUTPUT_DIR + \"cf_score_v3.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n else:\n with open(OUTPUT_DIR + f\"cf_score_v3_test{CFG.part}.pickle\", 'rb') as f:\n cf_score_df = pickle.load(f)\n \n cf_score_df = cf_score_df.rename(columns={\"cf_score\": \"cf_score3\"})\n \n df = pd.merge(df, cf_score_df, on=[\"article_id\", \"customer_id\"], how=\"left\")\n df[\"cf_score3\"] = df[\"cf_score3\"].astype(\"float16\")\n \n del cf_score_df\n gc.collect()\n \n return df","metadata":{"id":"CuviW9PFQlm5","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pair_ratio(original_df, df):\n \"\"\"\n pair.pickle is made by this code.\n ----------------------------------------------------\n new_df = pd.DataFrame()\n for article in tqdm(df[\"article_id\"].unique()):\n USERS = original_df.loc[original_df.article_id==article, \"customer_id\"].unique()\n vc = original_df.loc[(original_df.customer_id.isin(USERS))&(original_df.article_id!=article)].drop_duplicates([\"customer_id\", \"article_id\"])[\"article_id\"].value_counts().reset_index()\n vc.columns = [\"article_id\", \"pair_score\"] # Đặt tên cột rõ ràng\n vc[\"pair_score\"] = (vc[\"pair_score\"]/len(USERS)).astype(\"float16\")\n tmp = pd.merge(original_df[[\"customer_id\", \"article_id\"]], vc, on=\"article_id\", how=\"left\")\n tmp = tmp.groupby(\"customer_id\")[\"pair_score\"].agg([\"max\", \"mean\"]).reset_index().rename(columns={\"max\": \"max_pair_score\", \"mean\": \"mean_pair_score\"})\n tmp[\"article_id\"] = article\n new_df = pd.concat([new_df, tmp]).reset_index(drop=True)\n ----------------------------------------------------\n \"\"\"\n \n # Tạo pair.pickle nếu chưa tồn tại - lưu vào /kaggle/working\n pair_file = \"/kaggle/working/\" + (\"pair.pickle\" if CFG.train else \"pair_test.pickle\")\n \n if not os.path.exists(pair_file):\n print(f\"Creating {pair_file}...\")\n new_df = pd.DataFrame()\n \n for article in tqdm(df[\"article_id\"].unique()):\n USERS = original_df.loc[original_df.article_id==article, \"customer_id\"].unique()\n vc = original_df.loc[(original_df.customer_id.isin(USERS))&(original_df.article_id!=article)].drop_duplicates([\"customer_id\", \"article_id\"])[\"article_id\"].value_counts().reset_index()\n \n # Sửa lỗi: đặt tên cột rõ ràng sau value_counts\n vc.columns = [\"article_id\", \"pair_score\"]\n vc[\"pair_score\"] = (vc[\"pair_score\"]/len(USERS)).astype(\"float16\")\n \n tmp = pd.merge(original_df[[\"customer_id\", \"article_id\"]], vc, on=\"article_id\", how=\"left\")\n tmp = tmp.groupby(\"customer_id\")[\"pair_score\"].agg([\"max\", \"mean\"]).reset_index().rename(columns={\"max\": \"max_pair_score\", \"mean\": \"mean_pair_score\"})\n tmp[\"article_id\"] = article\n new_df = pd.concat([new_df, tmp]).reset_index(drop=True)\n \n # Lưu pair_ratio_df vào /kaggle/working\n with open(pair_file, \"wb\") as f:\n pickle.dump(new_df, f, protocol=4)\n print(f\"Saved {pair_file}\")\n pair_ratio_df = new_df\n else:\n # Load pair.pickle từ /kaggle/working nếu đã tồn tại\n with open(pair_file, \"rb\") as f:\n pair_ratio_df = pickle.load(f)\n \n df = pd.merge(df, pair_ratio_df, on=[\"customer_id\", \"article_id\"], how=\"left\")\n df[\"max_pair_score\"] = df[\"max_pair_score\"].fillna(0).astype(\"float16\")\n df[\"mean_pair_score\"] = df[\"mean_pair_score\"].fillna(0).astype(\"float16\")\n\n del pair_ratio_df\n gc.collect()\n\n return df","metadata":{"id":"wNNfeCfSey5r","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def last_purchase_date(original_df, df):\n \n for feature in [\"article_id\"]+ARTICLE_FEATURES:\n tmp = original_df.groupby([\"customer_id\", f\"{feature}\"])[\"day\"].agg(\"min\").reset_index().rename(columns={\"day\": f\"last_{feature}\"})\n df = pd.merge(df, tmp, on=[\"customer_id\", f\"{feature}\"], how=\"left\")\n df[f\"last_{feature}\"] = df[f\"last_{feature}\"].fillna(9999).astype(\"int16\")\n\n del tmp\n gc.collect()\n\n return df","metadata":{"id":"OObP46atSUpg","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weekly_count_features(original_df, df, week=5):\n\n for col in [\"article_id\", \"product_code\"]:\n tmp = original_df.query(f\"week<{week}\").groupby([col, \"week\"])[\"customer_id\"].nunique().unstack()\n tmp.columns = [f\"{col}_week{i}_count\" for i in range(week)]\n tmp = tmp.reset_index()\n df = pd.merge(df, tmp, on=col, how=\"left\")\n df[[f\"{col}_week{i}_count\" for i in range(week)]] = df[[f\"{col}_week{i}_count\" for i in range(week)]].fillna(0).astype(\"int16\")\n df[f\"{col}_week0_change_rate\"] = (df[f\"{col}_week0_count\"]/df[f\"{col}_week1_count\"]).replace([np.inf, -np.inf], np.nan).astype(\"float16\")\n\n del tmp\n gc.collect()\n\n return df","metadata":{"id":"XjuHMK9NAFJR","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def daily_count_features(original_df, df, day=7):\n\n for col in [\"article_id\", \"product_code\"]:\n tmp = original_df.query(f\"day<{day}\").groupby([col, \"day\"])[\"customer_id\"].nunique().unstack()\n tmp.columns = [f\"{col}_day{i}_count\" for i in range(day)]\n tmp = tmp.reset_index()\n df = pd.merge(df, tmp, on=col, how=\"left\")\n df[[f\"{col}_day{i}_count\" for i in range(day)]] = df[[f\"{col}_day{i}_count\" for i in range(day)]].fillna(0).astype(\"int16\")\n df[f\"{col}_day0_change_rate\"] = (df[f\"{col}_day0_count\"]/df[f\"{col}_day1_count\"]).replace([np.inf, -np.inf], np.nan).astype(\"float16\")\n\n del tmp\n gc.collect()\n\n return df","metadata":{"id":"mCjcgnrBHas0","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def last_year_count_features(original_df, df):\n\n if CFG.train:\n tmp1 = original_df.query(\"'2019-09-29'>=t_dat>='2019-09-09'\")\n else:\n tmp1 = original_df.query(\"'2019-10-06'>=t_dat>='2019-09-16'\")\n\n for col in ARTICLE_FEATURES:\n tmp2 = tmp1.groupby([\"customer_id\", col])[\"t_dat\"].count().reset_index().rename(columns={\"t_dat\": f\"last_year_{col}_count\"})\n df = pd.merge(df, tmp2, on=[\"customer_id\", col], how=\"left\")\n df[f\"last_year_{col}_count\"] = df[f\"last_year_{col}_count\"].fillna(0).astype(\"int16\")\n\n del tmp2\n gc.collect()\n\n return df","metadata":{"id":"Z1PJe8npyym5","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def age_features(original_df, df):\n # Tạo bản copy và chuyển đổi kiểu dữ liệu age\n original_df_filtered = original_df[\n (original_df['age'].notna()) & \n (original_df['age'] > 0) & \n (original_df['age'] < 120)\n ].copy()\n \n # Chuyển age sang float32 hoặc int để tránh lỗi float16\n original_df_filtered['age'] = original_df_filtered['age'].astype('float32')\n \n for feature in [\"article_id\"] + ARTICLE_FEATURES:\n # Đếm số lượng unique customers cho mỗi age\n tmp1 = (original_df_filtered\n .groupby(\"age\")[\"customer_id\"]\n .nunique()\n .reset_index()\n .rename(columns={\"customer_id\": \"user_count\"}))\n \n # Đếm số lượng unique customers đã mua feature đó ở mỗi age\n tmp2 = (original_df_filtered\n .groupby([\"age\", feature])[\"customer_id\"]\n .nunique()\n .reset_index()\n .rename(columns={\"customer_id\": f\"{feature}_purchase_user_count\"}))\n \n # Merge và tính tỷ lệ\n tmp2 = pd.merge(tmp2, tmp1, on=\"age\", how=\"left\")\n \n # Tính purchase rate\n tmp2[f\"{feature}_purchase_rate_by_age\"] = (\n tmp2[f\"{feature}_purchase_user_count\"] / tmp2[\"user_count\"]\n ).fillna(0).astype(\"float16\")\n \n # Merge vào df chính\n df = pd.merge(\n df, \n tmp2[[\"age\", feature, f\"{feature}_purchase_rate_by_age\"]], \n on=[\"age\", feature], \n how=\"left\"\n )\n \n # Fill NaN với 0\n df[f\"{feature}_purchase_rate_by_age\"] = (\n df[f\"{feature}_purchase_rate_by_age\"]\n .fillna(0)\n .astype(\"float16\")\n )\n \n del tmp1, tmp2\n gc.collect()\n\n del original_df_filtered\n gc.collect()\n \n return df","metadata":{"id":"an3AcCqsA4FX","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def price_features(original_df, df):\n tmp1 = original_df.groupby(\"customer_id\")[\"price\"].agg([\"max\", \"mean\"]).reset_index().rename(columns={\"max\": \"max_price_by_customer\", \"mean\": \"mean_price_by_customer\"})\n df = pd.merge(df, tmp1, on=\"customer_id\", how=\"left\")\n \n tmp2 = original_df.query(\"week<4\").groupby(\"article_id\")[\"price\"].agg(\"mean\").reset_index().rename(columns={\"price\": \"mean_price_4w\"})\n df = pd.merge(df, tmp2, on=\"article_id\", how=\"left\")\n \n del tmp1, tmp2\n gc.collect()\n\n df[\"higher_than_max_price\"] = 0\n target_index = df[df[\"mean_price_4w\"]<=df[\"max_price_by_customer\"]].index\n df.loc[target_index, \"higher_than_max_price\"] = 1\n df[\"higher_than_max_price\"] = df[\"higher_than_max_price\"].astype(\"int8\")\n\n df[\"higher_than_mean_price\"] = 0\n target_index = df[df[\"mean_price_4w\"]<=df[\"mean_price_by_customer\"]].index\n df.loc[target_index, \"higher_than_mean_price\"] = 1\n df[\"higher_than_mean_price\"] = df[\"higher_than_mean_price\"].astype(\"int8\")\n \n return df","metadata":{"id":"x-UNSgTpVUB3","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def channel_features(original_df, df):\n\n for w in [999, 3, 0]:\n tmp1 = original_df.query(f\"week<={w}\").groupby(\"customer_id\")[\"sales_channel_id\"].agg([\"mean\", \"max\"])\n tmp1.columns = [f\"mean_channel_{w}w_by_customer\", f\"max_channel_{w}w_by_customer\"]\n df = pd.merge(df, tmp1, on=\"customer_id\", how=\"left\")\n\n df[f\"max_channel_{w}w_by_customer\"] = df[f\"max_channel_{w}w_by_customer\"].fillna(0).astype(\"int8\")\n df[f\"mean_channel_{w}w_by_customer\"] = df[f\"mean_channel_{w}w_by_customer\"].astype(\"float16\")\n\n tmp2 = original_df.query(f\"week<={w}\").groupby(\"article_id\")[\"sales_channel_id\"].agg([\"mean\", \"max\"])\n tmp2.columns = [f\"mean_channel_{w}w_by_article\", f\"max_channel_{w}w_by_article\"]\n df = pd.merge(df, tmp2, on=\"article_id\", how=\"left\")\n\n df[f\"max_channel_{w}w_by_article\"] = df[f\"max_channel_{w}w_by_article\"].fillna(0).astype(\"int8\")\n df[f\"mean_channel_{w}w_by_article\"] = df[f\"mean_channel_{w}w_by_article\"].astype(\"float16\")\n \n df[f\"channel_score_mean_mean_{w}w\"] = (df[f\"mean_channel_{w}w_by_customer\"]*df[f\"mean_channel_{w}w_by_article\"] + (1-df[f\"mean_channel_{w}w_by_customer\"])*(1-df[f\"mean_channel_{w}w_by_article\"])).astype(\"float16\")\n df[f\"channel_score_max_mean_{w}w\"] = (df[f\"max_channel_{w}w_by_customer\"]*df[f\"mean_channel_{w}w_by_article\"] + (1-df[f\"max_channel_{w}w_by_customer\"])*(1-df[f\"mean_channel_{w}w_by_article\"])).astype(\"float16\")\n df[f\"channel_score_mean_max_{w}w\"] = (df[f\"mean_channel_{w}w_by_customer\"]*df[f\"max_channel_{w}w_by_article\"] + (1-df[f\"mean_channel_{w}w_by_customer\"])*(1-df[f\"max_channel_{w}w_by_article\"])).astype(\"float16\")\n df[f\"channel_score_max_max_{w}w\"] = (df[f\"max_channel_{w}w_by_customer\"]*df[f\"max_channel_{w}w_by_article\"] + (1-df[f\"max_channel_{w}w_by_customer\"])*(1-df[f\"max_channel_{w}w_by_article\"])).astype(\"float16\")\n\n del tmp1, tmp2\n gc.collect()\n \n return df","metadata":{"id":"wzNZVrGyTxLi","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def purchase_ratio(original_df, df):\n for feature in ARTICLE_FEATURES:\n tmp1 = original_df.groupby([\"customer_id\", feature])[\"t_dat\"].count().reset_index().rename(columns={\"t_dat\": f\"purchase_ratio_{feature}\"})\n tmp2 = original_df.groupby([\"customer_id\"])[\"t_dat\"].count().reset_index().rename(columns={\"t_dat\": \"tmp\"})\n tmp1 = pd.merge(tmp1, tmp2, on=\"customer_id\", how=\"left\")\n tmp1[f\"purchase_ratio_{feature}\"] = tmp1[f\"purchase_ratio_{feature}\"]/tmp1[\"tmp\"]\n df = pd.merge(df, tmp1[[\"customer_id\", feature, f\"purchase_ratio_{feature}\"]], on=[\"customer_id\", feature], how=\"left\")\n df[f\"purchase_ratio_{feature}\"] = df[f\"purchase_ratio_{feature}\"].fillna(0).astype(\"float16\")\n\n del tmp1\n gc.collect()\n\n return df","metadata":{"id":"xLiQfp2KZNz6","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gender_featuers(original_df, df):\n \"\"\"\n \"gender\" row is made using this function.\n ----------------------------------------------------\n def gender_flg(row):\n\n row[\"gender\"] = \"Unknown\"\n if row[\"index_group_name\"] in [\"Ladieswear\", \"Divided\"]:\n row[\"gender\"] = \"Female\"\n if row[\"index_group_name\"] == \"Menswear\":\n row[\"gender\"] = \"Male\"\n if row[\"index_group_name\"] == \"Sport\":\n if \"men\" in row[\"department_name\"].lower():\n row[\"gender\"] = \"Male\"\n if \"ladies\" in row[\"department_name\"].lower():\n row[\"gender\"] = \"Female\"\n\n return row[\"gender\"]\n ----------------------------------------------------\n \"\"\"\n\n with open(DATA_DIR + f\"gender.pickle\", 'rb') as f:\n gender_df = pickle.load(f)\n \n original_df = pd.merge(original_df, gender_df, on=\"article_id\", how=\"left\")\n df = pd.merge(df, gender_df, on=\"article_id\", how=\"left\")\n\n tmp = original_df[original_df[\"gender\"]!=\"Unknown\"].reset_index(drop=True)\n tmp[\"gender\"] = tmp[\"gender\"].apply(lambda x: 1 if x==\"Male\" else 0)\n tmp = tmp.groupby([\"customer_id\"])[\"gender\"].agg(\"mean\").reset_index().rename(columns={\"gender\": \"gender_mean_by_customer\"})\n df = pd.merge(df, tmp, on=\"customer_id\", how=\"left\")\n\n target_index = df[df[\"gender\"]==\"Male\"].index\n df[\"gender_score\"] = 1 - df[\"gender_mean_by_customer\"]\n target_value = df.loc[:, \"gender_score\"].to_numpy()\n target_value[target_index] = df[df[\"gender\"]==\"Male\"][\"gender_mean_by_customer\"].to_numpy()\n df[\"gender_score\"] = target_value\n\n df[\"gender_score\"] = df[\"gender_score\"].astype(\"float16\")\n df[\"gender_mean_by_customer\"] = df[\"gender_mean_by_customer\"].astype(\"float16\")\n\n del df[\"gender\"]\n gc.collect()\n\n return df","metadata":{"id":"OnSVp2mXQioI","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def datetime_features(original_df, df):\n tmp = original_df.groupby([\"article_id\"])[\"day\"].agg(\"max\").reset_index().rename(columns={\"day\": \"release_date\"})\n df = pd.merge(df, tmp, on=\"article_id\", how=\"left\")\n\n original_df = pd.merge(original_df, tmp, on=\"article_id\", how=\"left\")\n tmp = original_df.groupby([\"customer_id\"])[\"release_date\"].agg(\"mean\").reset_index().rename(columns={\"release_date\": \"mean_release_date_by_customer\"})\n df = pd.merge(df, tmp, on=\"customer_id\", how=\"left\")\n df[\"diff_mean_release_date\"] = df[\"mean_release_date_by_customer\"] - df[\"release_date\"]\n\n df[\"release_date\"] = df[\"release_date\"].fillna(-1).astype(\"int16\")\n df[\"diff_mean_release_date\"] = df[\"diff_mean_release_date\"].astype(\"float16\")\n df[\"mean_release_date_by_customer\"] = df[\"mean_release_date_by_customer\"].astype(\"float16\")\n\n return df","metadata":{"id":"d023ej5l2s19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def w2v_features(original_df, df):\n \"\"\"\n model_w2v is made by this code.\n ----------------------------------------------------\n popular_item = transactions_df.query(\"week <= 3\").groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:3000]\n train_item2vec = transactions_df[transactions_df[\"article_id\"].isin(popular_item)].reset_index(drop=True)\n train_item2vec = train_item2vec.query(f\"week <= 4\").reset_index(drop=True)\n display(train_item2vec.head(5))\n\n train_w2v = []\n train_w2v_dict = {}\n for i, v in train_item2vec.groupby(['customer_id']):\n train_w2v.append(list(v.article_id))\n train_w2v_dict[i] = list(v.article_id)\n\n model_w2v = word2vec.Word2Vec(train_w2v,\n sg=1,\n hs=0,\n ns_exponent=0.5,\n window=4000,\n min_count=1,\n negative=15,\n seed=CFG.seed,\n workers=5,\n hashfxn=hashfxn,\n epochs=100)\n ----------------------------------------------------\n \"\"\"\n\n with open(DATA_DIR + f\"model_w2v.pickle\", \"rb\") as f:\n model_w2v = pickle.load(f)\n\n if CFG.train:\n popular_item3000 = target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:3000]\n popular_item300 = target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n else:\n popular_item3000 = original_df.query(f\"t_dat >= '{val_start_date}'\").groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:3000]\n popular_item300 = original_df.query(f\"t_dat >= '{val_start_date}'\").groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n \n df = original_df[original_df[\"article_id\"].isin(popular_item3000)].reset_index(drop=True)\n df = df.query(f\"week <= 3\").reset_index(drop=True)\n\n w2v_list = []\n for i, v in df.groupby(['customer_id']):\n w2v_list.append(list(v.article_id))\n \n customer_vec = np.array([np.sum(model_w2v.wv[v], axis=0) for v in w2v_list]).reshape(-1, 100)\n popular_item300_vec = model_w2v.wv[list(popular_item300)]\n\n new_df = pd.DataFrame(cosine_similarity(customer_vec.reshape(-1, 100), popular_item300_vec.reshape(-1, 100)))\n new_df.columns = list(popular_item300)\n new_df.index = df.sort_values(\"customer_id\")[\"customer_id\"].unique()\n new_df = new_df.unstack().reset_index()\n new_df.columns = [\"article_id\", \"customer_id\", \"w2v_cosine_similarity\"]\n\n df = pd.merge(df, new_df, on=[\"customer_id\", \"article_id\"], how=\"left\")\n\n del df, new_df\n gc.collect()\n\n return df","metadata":{"id":"XypVYmYfQkvz","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def repeat_features(original_df, df):\n tmp1 = original_df.drop_duplicates([\"customer_id\", \"article_id\", \"t_dat\"]).reset_index(drop=True)\n tmp1 = tmp1.groupby([\"customer_id\", \"article_id\"])[\"t_dat\"].agg(\"count\").reset_index().rename(columns={\"t_dat\": \"repeat_count\"})\n\n tmp2 = tmp1.groupby([\"customer_id\"])[\"repeat_count\"].agg([\"mean\", \"max\", \"std\", \"count\", \"sum\"])\n tmp2.columns = [\"mean_repeat_count_by_customer\", \"max_repeat_count_by_customer\", \"std_repeat_count_by_customer\", \"unique_count_by_customer\", \"count_by_customer\"]\n tmp2[\"repeat_rate_by_customer\"] = (tmp2[\"count_by_customer\"]-tmp2[\"unique_count_by_customer\"])/tmp2[\"count_by_customer\"]\n\n tmp3 = tmp1.groupby([\"article_id\"])[\"repeat_count\"].agg([\"mean\", \"max\", \"std\", \"count\", \"sum\"])\n tmp3.columns = [\"mean_repeat_count_by_article_id\", \"max_repeat_count_by_article_id\", \"std_repeat_count_by_article_id\", \"unique_count_by_article_id\", \"count_by_article_id\"]\n tmp3[\"repeat_rate_by_article_id\"] = (tmp3[\"count_by_article_id\"]-tmp3[\"unique_count_by_article_id\"])/tmp3[\"count_by_article_id\"]\n\n tmp4 = original_df.groupby([\"customer_id\"])[\"day\"].agg(\"max\").reset_index().rename(columns={\"day\": \"max_day_by_customer\"})\n\n df = pd.merge(df, tmp2, on=\"customer_id\", how=\"left\")\n df = pd.merge(df, tmp3, on=\"article_id\", how=\"left\")\n df = pd.merge(df, tmp4, on=\"customer_id\", how=\"left\")\n\n del tmp1, tmp2, tmp3, tmp4\n gc.collect()\n\n df[\"multi_repeat_rate\"] = (df[\"repeat_rate_by_customer\"]*df[\"repeat_rate_by_article_id\"]).astype(\"float16\")\n df[\"sub_repeat_rate\"] = (df[\"repeat_rate_by_customer\"]-df[\"repeat_rate_by_article_id\"]).astype(\"float16\")\n df[\"span_by_customer\"] = (df[\"max_day_by_customer\"]/df[\"count_by_customer\"]).astype(\"float16\")\n\n df[[\"max_repeat_count_by_article_id\", \"unique_count_by_article_id\", \"count_by_article_id\"]] = df[[\"max_repeat_count_by_article_id\", \"unique_count_by_article_id\", \"count_by_article_id\"]].fillna(0).astype(\"int16\")\n df[[\"max_repeat_count_by_customer\", \"unique_count_by_customer\", \"count_by_customer\"]] = df[[\"max_repeat_count_by_customer\", \"unique_count_by_customer\", \"count_by_customer\"]].fillna(0).astype(\"int16\")\n df[\"max_day_by_customer\"] = df[\"max_day_by_customer\"].fillna(-1).astype(\"int16\")\n\n df[[\"mean_repeat_count_by_article_id\", \"std_repeat_count_by_article_id\", \"repeat_rate_by_article_id\"]] = df[[\"mean_repeat_count_by_article_id\", \"std_repeat_count_by_article_id\", \"repeat_rate_by_article_id\"]].astype(\"float16\")\n df[[\"mean_repeat_count_by_customer\", \"std_repeat_count_by_customer\", \"repeat_rate_by_customer\"]] = df[[\"mean_repeat_count_by_customer\", \"std_repeat_count_by_customer\", \"repeat_rate_by_customer\"]].astype(\"float16\")\n return df","metadata":{"id":"qL4I05dnxvsb","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def time_features(original_df, df):\n \"\"\"Thêm features về thời gian mua hàng\"\"\"\n \n # Ngày trong tuần (0=Monday, 6=Sunday)\n original_df[\"day_of_week\"] = original_df[\"t_dat\"].dt.dayofweek\n \n # Customer mua vào ngày nào trong tuần nhiều nhất?\n tmp = original_df.groupby(\"customer_id\")[\"day_of_week\"].agg([\"mean\", \"std\"]).reset_index()\n tmp.columns = [\"customer_id\", \"mean_dow_by_customer\", \"std_dow_by_customer\"]\n df = pd.merge(df, tmp, on=\"customer_id\", how=\"left\")\n df[\"mean_dow_by_customer\"] = df[\"mean_dow_by_customer\"].astype(\"float16\")\n df[\"std_dow_by_customer\"] = df[\"std_dow_by_customer\"].fillna(0).astype(\"float16\")\n \n # Article được mua vào ngày nào nhiều nhất?\n tmp = original_df.groupby(\"article_id\")[\"day_of_week\"].agg(\"mean\").reset_index()\n tmp.columns = [\"article_id\", \"mean_dow_by_article\"]\n df = pd.merge(df, tmp, on=\"article_id\", how=\"left\")\n df[\"mean_dow_by_article\"] = df[\"mean_dow_by_article\"].astype(\"float16\")\n \n # Độ khớp ngày mua giữa customer và article\n df[\"dow_match_score\"] = (1 - abs(df[\"mean_dow_by_customer\"] - df[\"mean_dow_by_article\"]) / 7).astype(\"float16\")\n \n del tmp\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def customer_activity_features(original_df, df):\n \"\"\"Features về mức độ active của customer\"\"\"\n \n tmp = original_df.groupby(\"customer_id\").agg({\n \"t_dat\": [\"count\", \"nunique\"], # Số lần mua, số ngày khác nhau\n \"article_id\": \"nunique\", # Số article unique đã mua\n \"price\": [\"sum\", \"std\"], # Tổng chi tiêu, độ biến động giá\n })\n tmp.columns = [\n \"total_purchases\", \"active_days\", \n \"unique_articles_bought\", \n \"total_spending\", \"price_std_by_customer\"\n ]\n tmp = tmp.reset_index()\n \n # Tần suất mua (purchases per active day)\n tmp[\"purchase_frequency\"] = (tmp[\"total_purchases\"] / tmp[\"active_days\"]).astype(\"float16\")\n \n # Diversity score (unique articles / total purchases)\n tmp[\"diversity_score\"] = (tmp[\"unique_articles_bought\"] / tmp[\"total_purchases\"]).astype(\"float16\")\n \n # Convert dtypes\n tmp[\"total_purchases\"] = tmp[\"total_purchases\"].astype(\"int16\")\n tmp[\"active_days\"] = tmp[\"active_days\"].astype(\"int16\")\n tmp[\"unique_articles_bought\"] = tmp[\"unique_articles_bought\"].astype(\"int16\")\n tmp[\"total_spending\"] = tmp[\"total_spending\"].astype(\"float16\")\n tmp[\"price_std_by_customer\"] = tmp[\"price_std_by_customer\"].fillna(0).astype(\"float16\")\n \n df = pd.merge(df, tmp, on=\"customer_id\", how=\"left\")\n \n del tmp\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def interaction_depth_features(original_df, df):\n \"\"\"Độ sâu tương tác giữa customer và các thuộc tính article\"\"\"\n \n # Đảm bảo original_df có các cột ARTICLE_FEATURES\n if \"product_code\" not in original_df.columns:\n original_df = pd.merge(original_df, articles_df[[\"article_id\"]+ARTICLE_FEATURES], on=\"article_id\", how=\"left\")\n \n for feature in ARTICLE_FEATURES:\n # Số lượng unique values của feature mà customer đã mua\n tmp = original_df.groupby(\"customer_id\")[feature].nunique().reset_index()\n tmp.columns = [\"customer_id\", f\"unique_{feature}_count\"]\n tmp[f\"unique_{feature}_count\"] = tmp[f\"unique_{feature}_count\"].astype(\"int16\")\n df = pd.merge(df, tmp, on=\"customer_id\", how=\"left\")\n \n del tmp\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def price_sensitivity_features(original_df, df):\n \"\"\"Features về độ nhạy giá\"\"\"\n \n # Đảm bảo original_df có cột product_code\n if \"product_code\" not in original_df.columns:\n original_df = pd.merge(original_df, articles_df[[\"article_id\", \"product_code\"]], on=\"article_id\", how=\"left\")\n \n # Giá trung bình theo product_code\n tmp1 = original_df.groupby(\"product_code\")[\"price\"].mean().reset_index()\n tmp1.columns = [\"product_code\", \"mean_price_by_product_code\"]\n \n # Giá trung bình của article\n tmp2 = original_df.groupby(\"article_id\")[\"price\"].mean().reset_index()\n tmp2.columns = [\"article_id\", \"article_price\"]\n \n # Merge qua article_id trước (df đã có article_id và product_code từ add_article_information)\n df = pd.merge(df, tmp2, on=\"article_id\", how=\"left\")\n \n # Merge tmp1 qua product_code (df đã có product_code)\n if \"product_code\" in df.columns:\n df = pd.merge(df, tmp1, on=\"product_code\", how=\"left\")\n else:\n # Nếu df chưa có product_code, merge qua article_id\n tmp1_with_article = pd.merge(articles_df[[\"article_id\", \"product_code\"]], tmp1, on=\"product_code\", how=\"left\")\n df = pd.merge(df, tmp1_with_article[[\"article_id\", \"mean_price_by_product_code\"]], on=\"article_id\", how=\"left\")\n \n # Article đắt/rẻ hơn trung bình của product_code?\n df[\"price_vs_product_code\"] = ((df[\"article_price\"] - df[\"mean_price_by_product_code\"]) / (df[\"mean_price_by_product_code\"] + 0.01)).astype(\"float16\")\n \n df[\"mean_price_by_product_code\"] = df[\"mean_price_by_product_code\"].astype(\"float16\")\n df[\"article_price\"] = df[\"article_price\"].astype(\"float16\")\n \n del tmp1, tmp2\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def sequence_features(original_df, df):\n \"\"\"Features về chuỗi mua hàng - sản phẩm thường được mua sau sản phẩm nào\"\"\"\n \n # Sort theo thời gian\n tmp_df = original_df.sort_values([\"customer_id\", \"t_dat\"]).reset_index(drop=True)\n tmp_df[\"prev_article\"] = tmp_df.groupby(\"customer_id\")[\"article_id\"].shift(1)\n \n # Đếm cặp (prev_article, article) xuất hiện bao nhiêu lần\n pair_count = tmp_df.dropna(subset=[\"prev_article\"]).groupby(\n [\"prev_article\", \"article_id\"]\n ).size().reset_index(name=\"pair_freq\")\n \n # Normalize theo prev_article\n prev_total = pair_count.groupby(\"prev_article\")[\"pair_freq\"].sum().reset_index(name=\"prev_total\")\n pair_count = pd.merge(pair_count, prev_total, on=\"prev_article\")\n pair_count[\"transition_prob\"] = (pair_count[\"pair_freq\"] / pair_count[\"prev_total\"]).astype(\"float16\")\n \n # Last article của mỗi customer\n last_article = original_df.sort_values(\"t_dat\").groupby(\"customer_id\")[\"article_id\"].last().reset_index()\n last_article.columns = [\"customer_id\", \"last_bought_article\"]\n \n df = pd.merge(df, last_article, on=\"customer_id\", how=\"left\")\n \n # Merge transition probability\n pair_count = pair_count[[\"prev_article\", \"article_id\", \"transition_prob\"]]\n pair_count.columns = [\"last_bought_article\", \"article_id\", \"transition_prob\"]\n \n df = pd.merge(df, pair_count, on=[\"last_bought_article\", \"article_id\"], how=\"left\")\n df[\"transition_prob\"] = df[\"transition_prob\"].fillna(0).astype(\"float16\")\n \n # Clean up\n del df[\"last_bought_article\"]\n del tmp_df, pair_count, prev_total, last_article\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recency_weighted_features(original_df, df):\n \"\"\"Features có trọng số theo recency - ngày gần hơn = weight cao hơn\"\"\"\n \n # Đảm bảo original_df có các cột cần thiết\n if \"product_code\" not in original_df.columns:\n original_df = pd.merge(original_df, articles_df[[\"article_id\", \"product_code\", \"colour_group_code\"]], on=\"article_id\", how=\"left\")\n \n # Tính weight theo recency (chỉ 1 lần)\n original_df[\"recency_weight\"] = (1 / (original_df[\"day\"] + 1)).astype(\"float32\")\n \n # Weighted popularity cho article\n tmp = original_df.groupby(\"article_id\")[\"recency_weight\"].sum().reset_index()\n tmp.columns = [\"article_id\", \"recency_weighted_popularity\"]\n tmp[\"recency_weighted_popularity\"] = tmp[\"recency_weighted_popularity\"].astype(\"float16\")\n \n df = pd.merge(df, tmp, on=\"article_id\", how=\"left\")\n df[\"recency_weighted_popularity\"] = df[\"recency_weighted_popularity\"].fillna(0).astype(\"float16\")\n \n del tmp\n gc.collect()\n \n # Tính customer total 1 lần duy nhất\n customer_total = original_df.groupby(\"customer_id\")[\"recency_weight\"].sum().reset_index()\n customer_total.columns = [\"customer_id\", \"recency_weight_total\"]\n \n # Weighted category preference cho customer\n for feature in [\"product_code\", \"colour_group_code\"]:\n tmp1 = original_df.groupby([\"customer_id\", feature])[\"recency_weight\"].sum().reset_index()\n tmp1 = pd.merge(tmp1, customer_total, on=\"customer_id\")\n tmp1[f\"recency_weighted_{feature}_pref\"] = (tmp1[\"recency_weight\"] / tmp1[\"recency_weight_total\"]).astype(\"float16\")\n \n # Merge qua article_id nếu df không có feature\n if feature in df.columns:\n df = pd.merge(df, tmp1[[\"customer_id\", feature, f\"recency_weighted_{feature}_pref\"]], \n on=[\"customer_id\", feature], how=\"left\")\n else:\n # Merge qua article_id\n tmp1_with_article = pd.merge(articles_df[[\"article_id\", feature]], \n tmp1[[\"customer_id\", feature, f\"recency_weighted_{feature}_pref\"]], \n on=feature, how=\"left\")\n df = pd.merge(df, tmp1_with_article[[\"customer_id\", \"article_id\", f\"recency_weighted_{feature}_pref\"]], \n on=[\"customer_id\", \"article_id\"], how=\"left\")\n \n df[f\"recency_weighted_{feature}_pref\"] = df[f\"recency_weighted_{feature}_pref\"].fillna(0).astype(\"float16\")\n \n del tmp1\n gc.collect()\n \n # Clean up\n del original_df[\"recency_weight\"], customer_total\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def article_trend_features(original_df, df):\n \"\"\"Xu hướng popularity của article theo thời gian\"\"\"\n \n # Tính popularity tuần 0 vs tuần 2-3 (trend dài hạn hơn)\n week0 = original_df.query(\"week==0\").groupby(\"article_id\")[\"customer_id\"].nunique()\n week23 = original_df.query(\"week>=2 and week<=3\").groupby(\"article_id\")[\"customer_id\"].nunique() / 2\n \n trend = (week0 - week23) / (week23 + 1) # +1 để tránh chia 0\n trend = trend.reset_index()\n trend.columns = [\"article_id\", \"popularity_trend\"]\n \n df = pd.merge(df, trend, on=\"article_id\", how=\"left\")\n df[\"popularity_trend\"] = df[\"popularity_trend\"].fillna(0).astype(\"float16\")\n \n # Rank của article trong tuần gần nhất (percentile)\n rank = original_df.query(\"week==0\").groupby(\"article_id\")[\"customer_id\"].nunique()\n rank = rank.rank(ascending=False, pct=True).reset_index()\n rank.columns = [\"article_id\", \"popularity_rank_pct\"]\n \n df = pd.merge(df, rank, on=\"article_id\", how=\"left\")\n df[\"popularity_rank_pct\"] = df[\"popularity_rank_pct\"].fillna(1).astype(\"float16\")\n \n del trend, rank\n gc.collect()\n \n return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_dataset(original_df=transactions_df):\n\n with timer(\"Make original_df features\"):\n original_df = original_df_features(original_df=original_df)\n \n with timer(\"Make df\"):\n df = make_df(original_df=original_df)\n \n # with timer(\"Make w2v features\"):\n # df = w2v_features(original_df=original_df, df=df)\n\n with timer(\"Add article features\"):\n original_df = add_article_information(df=original_df)\n df = add_article_information(df=df)\n\n with timer(\"Add customer features\"):\n original_df = add_customer_information(df=original_df)\n df = add_customer_information(df=df)\n \n with timer(\"Make repeat features\"):\n df = repeat_features(original_df=original_df, df=df)\n \n with timer(\"Make last purchase features\"):\n df = last_purchase_date(original_df=original_df, df=df)\n\n with timer(\"Make weekly count features\"):\n df = weekly_count_features(original_df=original_df, df=df)\n \n with timer(\"Make daily count features\"):\n df = daily_count_features(original_df=original_df, df=df)\n \n with timer(\"Make last year count features\"):\n df = last_year_count_features(original_df=original_df, df=df)\n\n with timer(\"Make price features\"):\n df = price_features(original_df=original_df, df=df)\n \n with timer(\"Make channel features\"):\n df = channel_features(original_df=original_df, df=df)\n\n with timer(\"Make purchase ratio features\"):\n df = purchase_ratio(original_df=original_df, df=df)\n \n with timer(\"Make datetime features\"):\n df = datetime_features(original_df=original_df, df=df)\n\n # with timer(\"Make cf features\"):\n # df = cf_features(original_df=original_df, df=df)\n \n # with timer(\"Make cf features2\"):\n # df = cf_features2(original_df=original_df, df=df)\n\n # with timer(\"Make time features\"):\n # df = time_features(original_df=original_df, df=df)\n \n # with timer(\"Make customer activity features\"):\n # df = customer_activity_features(original_df=original_df, df=df)\n \n # with timer(\"Make article trend features\"):\n # df = article_trend_features(original_df=original_df, df=df)\n \n # with timer(\"Make interaction depth features\"):\n # df = interaction_depth_features(original_df=original_df, df=df)\n \n with timer(\"Make price sensitivity features\"):\n df = price_sensitivity_features(original_df=original_df, df=df)\n \n # with timer(\"Make sequence features\"):\n # df = sequence_features(original_df=original_df, df=df)\n \n # with timer(\"Make recency weighted features\"):\n # df = recency_weighted_features(original_df=original_df, df=df)\n\n \n display(df.head(10))\n\n return df","metadata":{"id":"TkT1mF5QIg6i","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def calculate_and_save_cf_features(original_df, target_df):\n# \"\"\"\n# Tính toán và lưu các cf_score features\n# \"\"\"\n# def get_sim_item(df, user_col, item_col, use_iif=False): \n# user_item_ = df.groupby(user_col)[item_col].agg(set).reset_index() \n# user_item_dict = dict(zip(user_item_[user_col], user_item_[item_col]))\n\n# del user_item_\n# gc.collect()\n \n# sim_item = {}\n# item_cnt = defaultdict(int) \n# for user, items in tqdm(user_item_dict.items()): \n# for i in items:\n# item_cnt[i] += 1\n# sim_item.setdefault(i, {})\n# for relate_item in items: \n# if not relate_item in popular_item: \n# continue \n# sim_item[i].setdefault(relate_item, 0) \n# if not use_iif: \n# sim_item[i][relate_item] += 1 \n# else: \n# sim_item[i][relate_item] += 1 / math.log(1 + len(items)) \n# sim_item_corr = sim_item.copy() \n# for i, related_items in tqdm(sim_item.items()): \n# for j, cij in related_items.items(): \n# sim_item_corr[i][j] = cij/math.sqrt(item_cnt[i]*item_cnt[j]) \n \n# return sim_item_corr, user_item_dict\n\n# def recommend(sim_item_corr, user_item_dict, user_id): \n# rank = {}\n# interacted_items = user_item_dict[user_id] \n# for i in interacted_items:\n# for j, wij in sorted(sim_item_corr[i].items(), key=lambda d: d[1], reverse=True): \n# rank.setdefault(j, 0) \n# rank[j] += wij \n# return sorted(rank.items(), key=lambda d: d[1], reverse=True)\n \n# # Lấy top 300 popular items\n# if CFG.train:\n# popular_item = target_df.groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n# else:\n# popular_item = original_df.query(f\"t_dat >= '{val_start_date}'\").groupby([\"article_id\"])[\"customer_id\"].nunique().sort_values(ascending=False).index[:300]\n \n# print(\"Calculating item similarity matrix...\")\n# item_sim_list, user_item = get_sim_item(original_df, user_col='customer_id', item_col='article_id', use_iif=True)\n\n# # Tính cho article_id\n# print(\"\\n=== Calculating cf_score for article_id ===\")\n# if CFG.train:\n# new_df = []\n# for c in tqdm(original_df[\"customer_id\"].unique()):\n# if c in user_item:\n# tmp = pd.DataFrame(recommend(item_sim_list, user_item, c))\n# tmp[\"customer_id\"] = c\n# new_df.append(tmp)\n# new_df = pd.concat(new_df).reset_index(drop=True)\n# new_df = new_df.rename(columns={0: \"article_id\", 1: \"cf_score_article_id\"})\n# with open(OUTPUT_DIR + \"cf_score_article_id.pickle\", mode=\"wb\") as f:\n# pickle.dump(new_df, f, protocol=4)\n# print(f\"Saved cf_score_article_id.pickle\")\n# else:\n# # Chia thành 14 phần cho inference\n# unique_customers = original_df[\"customer_id\"].unique()\n# total_customers = len(unique_customers)\n# customers_per_part = total_customers // 14\n \n# for i in range(14):\n# start_idx = i * customers_per_part\n# if i == 13:\n# end_idx = total_customers\n# else:\n# end_idx = (i + 1) * customers_per_part\n \n# selected_customers = unique_customers[start_idx:end_idx]\n \n# new_df = []\n# for c in tqdm(selected_customers):\n# if c in user_item:\n# tmp = pd.DataFrame(recommend(item_sim_list, user_item, c))\n# tmp[\"customer_id\"] = c\n# new_df.append(tmp)\n \n# if len(new_df) > 0:\n# new_df = pd.concat(new_df).reset_index(drop=True)\n# new_df = new_df.rename(columns={0: \"article_id\", 1: \"cf_score_article_id\"})\n# with open(OUTPUT_DIR + f\"cf_score_article_id_test{i}.pickle\", mode=\"wb\") as f:\n# pickle.dump(new_df, f, protocol=4)\n# print(f\"Saved cf_score_article_id_test{i}.pickle\")\n\n# # Merge với articles_df để có các ARTICLE_FEATURES\n# original_df_with_features = pd.merge(\n# original_df[['customer_id', 'article_id']], \n# articles_df[['article_id'] + ARTICLE_FEATURES], \n# on='article_id', \n# how='left'\n# )\n \n# # Tính cho từng feature trong ARTICLE_FEATURES\n# for feature in tqdm(ARTICLE_FEATURES):\n# print(f\"\\n=== Calculating cf_score for {feature} ===\")\n \n# # Tạo sim_item cho feature này\n# feature_sim_list, feature_user_item = get_sim_item(\n# original_df_with_features, \n# user_col='customer_id', \n# item_col=feature, \n# use_iif=True\n# )\n \n# if CFG.train:\n# new_df = []\n# for c in tqdm(original_df[\"customer_id\"].unique()):\n# if c in feature_user_item:\n# tmp = pd.DataFrame(recommend(feature_sim_list, feature_user_item, c))\n# tmp[\"customer_id\"] = c\n# new_df.append(tmp)\n# new_df = pd.concat(new_df).reset_index(drop=True)\n# new_df = new_df.rename(columns={0: feature, 1: f\"cf_score_{feature}\"})\n# with open(OUTPUT_DIR + f\"cf_score_{feature}.pickle\", mode=\"wb\") as f:\n# pickle.dump(new_df, f, protocol=4)\n# print(f\"Saved cf_score_{feature}.pickle\")\n# else:\n# # Chia thành 14 phần\n# unique_customers = original_df[\"customer_id\"].unique()\n# total_customers = len(unique_customers)\n# customers_per_part = total_customers // 14\n \n# for i in range(14):\n# start_idx = i * customers_per_part\n# if i == 13:\n# end_idx = total_customers\n# else:\n# end_idx = (i + 1) * customers_per_part\n \n# selected_customers = unique_customers[start_idx:end_idx]\n \n# new_df = []\n# for c in tqdm(selected_customers):\n# if c in feature_user_item:\n# tmp = pd.DataFrame(recommend(feature_sim_list, feature_user_item, c))\n# tmp[\"customer_id\"] = c\n# new_df.append(tmp)\n \n# if len(new_df) > 0:\n# new_df = pd.concat(new_df).reset_index(drop=True)\n# new_df = new_df.rename(columns={0: feature, 1: f\"cf_score_{feature}\"})\n# with open(OUTPUT_DIR + f\"cf_score_{feature}_test{i}.pickle\", mode=\"wb\") as f:\n# pickle.dump(new_df, f, protocol=4)\n# print(f\"Saved cf_score_{feature}_test{i}.pickle\")\n\n# # Tính cf_score_article_with_channel\n# print(\"\\n=== Calculating cf_score_article_with_channel ===\")\n \n# # Thêm cột sales_channel_id vào article_id để tạo key kết hợp\n# original_df_channel = original_df.copy()\n# original_df_channel['article_channel'] = original_df_channel['article_id'].astype(str) + '_' + original_df_channel['sales_channel_id'].astype(str)\n \n# channel_sim_list, channel_user_item = get_sim_item(\n# original_df_channel,\n# user_col='customer_id',\n# item_col='article_channel',\n# use_iif=True\n# )\n \n# if CFG.train:\n# new_df = []\n# for c in tqdm(original_df[\"customer_id\"].unique()):\n# if c in channel_user_item:\n# tmp = pd.DataFrame(recommend(channel_sim_list, channel_user_item, c))\n# tmp[\"customer_id\"] = c\n# new_df.append(tmp)\n# new_df = pd.concat(new_df).reset_index(drop=True)\n# new_df = new_df.rename(columns={0: \"article_channel\", 1: \"cf_score_article_with_channel\"})\n \n# # Tách article_id và sales_channel_id\n# new_df[['article_id', 'sales_channel_id']] = new_df['article_channel'].str.split('_', expand=True)\n# new_df['article_id'] = new_df['article_id'].astype(int)\n# new_df['sales_channel_id'] = new_df['sales_channel_id'].astype(int)\n# new_df = new_df.drop('article_channel', axis=1)\n \n# with open(OUTPUT_DIR + \"cf_score_article_with_channel.pickle\", mode=\"wb\") as f:\n# pickle.dump(new_df, f, protocol=4)\n# print(f\"Saved cf_score_article_with_channel.pickle\")\n# else:\n# # Chia thành 14 phần\n# unique_customers = original_df[\"customer_id\"].unique()\n# total_customers = len(unique_customers)\n# customers_per_part = total_customers // 14\n \n# for i in range(14):\n# start_idx = i * customers_per_part\n# if i == 13:\n# end_idx = total_customers\n# else:\n# end_idx = (i + 1) * customers_per_part\n \n# selected_customers = unique_customers[start_idx:end_idx]\n \n# new_df = []\n# for c in tqdm(selected_customers):\n# if c in channel_user_item:\n# tmp = pd.DataFrame(recommend(channel_sim_list, channel_user_item, c))\n# tmp[\"customer_id\"] = c\n# new_df.append(tmp)\n \n# if len(new_df) > 0:\n# new_df = pd.concat(new_df).reset_index(drop=True)\n# new_df = new_df.rename(columns={0: \"article_channel\", 1: \"cf_score_article_with_channel\"})\n \n# # Tách article_id và sales_channel_id\n# new_df[['article_id', 'sales_channel_id']] = new_df['article_channel'].str.split('_', expand=True)\n# new_df['article_id'] = new_df['article_id'].astype(int)\n# new_df['sales_channel_id'] = new_df['sales_channel_id'].astype(int)\n# new_df = new_df.drop('article_channel', axis=1)\n \n# with open(OUTPUT_DIR + f\"cf_score_article_with_channel_test{i}.pickle\", mode=\"wb\") as f:\n# pickle.dump(new_df, f, protocol=4)\n# print(f\"Saved cf_score_article_with_channel_test{i}.pickle\")\n\n# print(\"\\n=== Finished calculating all cf_features ===\")\n\n# # Gọi hàm này trước khi chạy make_dataset\n# calculate_and_save_cf_features(transactions_df, target_df)","metadata":{"id":"PiPeSalycYZ-","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = make_dataset(original_df=transactions_df)\ndf = df.drop('age', axis=1)","metadata":{"id":"yjo-qerZU0Kv","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Create the directory if it doesn't exist\nexp_dir = os.path.join(OUTPUT_DIR, f\"exp{CFG.exp}\")\nos.makedirs(exp_dir, exist_ok=True)\n\nif CFG.train:\n with open(os.path.join(exp_dir, \"df.pickle\"), mode=\"wb\") as f:\n pickle.dump(df, f, protocol=4)\nelse:\n with open(os.path.join(exp_dir, f\"test{CFG.part}.pickle\"), mode=\"wb\") as f:\n pickle.dump(df, f, protocol=4)","metadata":{"id":"duoMpRH-4K17","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del transactions_df\ngc.collect()","metadata":{"id":"yc8xaIoR_0Oo","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"id":"od_0VyUTcIFM"}},{"cell_type":"code","source":"def make_fold(df, seed=CFG.seed, folds=CFG.fold):\n unique_customer = df[\"customer_id\"].unique()\n fold = np.zeros(len(df), dtype=int)\n kf = KFold(n_splits=folds, shuffle=True, random_state=seed)\n for i_fold, (tr_group_idx, va_group_idx) in enumerate(kf.split(unique_customer)):\n tr_groups, va_groups = unique_customer[tr_group_idx], unique_customer[va_group_idx]\n is_va = df[df[\"customer_id\"].isin(va_groups)].index\n fold[is_va] = i_fold\n return fold","metadata":{"id":"qxizlIvPiR5M","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calc_score(X_val, val_pred):\n X_val[\"prediction\"] = val_pred\n X_val = X_val.sort_values(\"prediction\", ascending=False).reset_index(drop=True)\n new_df = X_val.groupby([\"customer_id\"])[\"article_id\"].apply(list).reset_index().rename(columns={\"article_id\": \"prediction\"})\n new_df[\"prediction\"] = new_df[\"prediction\"].apply(lambda x: x[:12])\n \n # Sửa ở đây: cần group article_id từ target_df thành list\n target_grouped = target_df.groupby(\"customer_id\")[\"article_id\"].apply(list).reset_index()\n new_df = pd.merge(new_df, target_grouped, on=\"customer_id\", how=\"left\")\n \n score = mapk(new_df['article_id'], new_df['prediction'], k=12)\n return score","metadata":{"id":"lld5OoschGxQ","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_catboost(param, df):\n \n oof_pred = np.zeros(len(df))\n feature_importance_df = pd.DataFrame()\n score_list = []\n \n folds_idx = make_fold(df)\n oof_pred = np.zeros(len(df))\n\n for fold in range(CFG.fold):\n\n if fold in CFG.used_fold:\n\n LOGGER.info(f\"==============================================\")\n LOGGER.info(f\"▶︎ Start fold{fold} Training\")\n LOGGER.info(f\"==============================================\")\n\n tr_idx = np.argwhere(folds_idx != fold).reshape(-1)\n va_idx = np.argwhere(folds_idx == fold).reshape(-1)\n X_trn, X_val = df.loc[tr_idx].reset_index(drop=True), df.loc[va_idx].reset_index(drop=True)\n y_trn, y_val = df.loc[tr_idx, CFG.target].reset_index(drop=True), df.loc[va_idx, CFG.target].reset_index(drop=True)\n\n LOGGER.info(f\"train_shape: {X_trn.shape}, val_shape: {X_val.shape}\")\n\n train = Pool(X_trn.drop(columns=[CFG.target]+DROP_COLS), y_trn, cat_features=CATEGORICAL_COLS)\n valid = Pool(X_val.drop(columns=[CFG.target]+DROP_COLS), y_val, cat_features=CATEGORICAL_COLS)\n\n model = CatBoost(param)\n model = model.fit(\n train,\n eval_set=valid,\n use_best_model=True,\n early_stopping_rounds=100,\n verbose_eval=200\n )\n \n # ==============================================\n # Feature Importances\n # ==============================================\n\n fold_importance_df = pd.DataFrame()\n fold_importance_df[\"feature\"] = model.feature_names_\n fold_importance_df[\"importance\"] = model.feature_importances_\n fold_importance_df[\"fold\"] = fold\n feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n\n # ==============================================\n # Calculate Score\n # ==============================================\n val_pred = model.predict(X_val.drop(columns=[CFG.target]+DROP_COLS), prediction_type='Probability').T[1]\n score = calc_score(X_val, val_pred)\n\n LOGGER.info(f\"fold{fold} score: {score:.6f}\")\n\n oof_pred[va_idx] = val_pred\n score_list.append([fold, score])\n\n # ==============================================\n # Save model\n # ==============================================\n pickle.dump(model, open(OUTPUT_DIR + f'model_fold{fold}.pkl', 'wb'))\n \n del model, X_trn, X_val, y_trn, y_val, train, valid\n gc.collect()\n \n score_df = pd.DataFrame(\n score_list, columns=[\"fold\", \"IoU\"])\n \n return oof_pred, score_df, feature_importance_df\n\ndef show_feature_importance(feature_importance_df):\n order = list(feature_importance_df.groupby(\"feature\").mean().sort_values(\"importance\", ascending=False).index)\n plt.figure(figsize=(10, 15))\n sns.barplot(x=\"importance\", y=\"feature\", data=feature_importance_df, order=order)\n plt.title(\"feature importance\")\n plt.tight_layout()\n plt.savefig(OUTPUT_DIR+'feature_importance.png')","metadata":{"id":"9DF7kcKPCy_K","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG.train:\n oof_pred, score_df, feature_importance_df = run_catboost(PARAMS, df)\n display(score_df)","metadata":{"id":"exUpRIsZCzCA","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference","metadata":{"id":"HI8LD3bFQqD4"}},{"cell_type":"code","source":"def inference(X_test):\n tmp_test_pred = np.zeros(len(X_test))\n for fold in tqdm(range(CFG.fold)):\n if fold in CFG.used_fold:\n with open(MODEL_DIR + f\"model_fold{fold}.pkl\", 'rb') as f:\n model = pickle.load(f)\n tmp_test_pred += model.predict(X_test.drop(columns=DROP_COLS), prediction_type='Probability').T[1]\n tmp_test_pred = tmp_test_pred / len(CFG.used_fold)\n X_test = X_test[[\"customer_id\", \"article_id\"]]\n X_test[\"prediction\"] = tmp_test_pred\n return X_test","metadata":{"id":"kyaCFpIOCzFx","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(14):\n with open(DATA_DIR + f\"exp{CFG.exp}/test{i}.pickle\", \"rb\") as f:\n test = pickle.load(f)\n test = inference(test)\n with open(OUTPUT_DIR + f\"exp{CFG.exp}/test{i}.pickle\", mode=\"wb\") as f:\n pickle.dump(test, f)\n del test\n gc.collect()","metadata":{"id":"IMDsL-9kCzH6","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not(CFG.train):\n\n output_df = pd.DataFrame()\n for i in tqdm(range(14)):\n with open(OUTPUT_DIR + f\"exp{CFG.exp}/test{i}.pickle\", \"rb\") as f:\n df = pickle.load(f)\n df = df[[\"customer_id\", \"article_id\", \"prediction\"]]\n\n # convert to submission format\n df = df.sort_values(\"prediction\", ascending=False)\n df = df.groupby(\"customer_id\")[\"article_id\"].apply(list).reset_index()\n df = df.rename(columns={\"article_id\": \"prediction\"})\n df[\"prediction\"] = df[\"prediction\"].apply(lambda x: x[:12])\n\n output_df = pd.concat([output_df, df]).reset_index(drop=True)\n del df\n gc.collect()\n\n # convert numbers to customer_id/article_id\n with open(MODEL_DIR + \"num_to_customer_id.pkl\", 'rb') as f:\n customer_id_dict = pickle.load(f)\n with open(MODEL_DIR + \"num_to_article_id.pkl\", 'rb') as f:\n article_id_dict = pickle.load(f)\n\n output_df[\"customer_id\"] = output_df[\"customer_id\"].map(customer_id_dict)\n output_df[\"prediction\"] = output_df[\"prediction\"].apply(lambda x: sorted(set(x), key=x.index)[:12])\n output_df[\"prediction\"] = output_df[\"prediction\"].apply(lambda x: [article_id_dict[e] for e in x])\n output_df[\"prediction\"] = output_df[\"prediction\"].apply(lambda x: \" \".join(x))\n\n output_df[[\"customer_id\", \"prediction\"]].to_csv(OUTPUT_DIR + \"submission.csv\", index=False)\n# !kaggle competitions submit -c h-and-m-personalized-fashion-recommendations -f\"{OUTPUT_DIR}submission.csv\" -m f\"exp{CFG.exp}\"","metadata":{"id":"tk8q0UpQQzSd","trusted":true},"outputs":[],"execution_count":null}]}