{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n\ndf = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\", dtype={\"article_id\": str})\nprint(df.shape)\ndf.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-15T04:36:41.285131Z","iopub.execute_input":"2022-08-15T04:36:41.285465Z","iopub.status.idle":"2022-08-15T04:37:41.837939Z","shell.execute_reply.started":"2022-08-15T04:36:41.285374Z","shell.execute_reply":"2022-08-15T04:37:41.837175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"t_dat\"] = pd.to_datetime(df[\"t_dat\"])\ndf[\"t_dat\"].max()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T04:37:41.839839Z","iopub.execute_input":"2022-08-15T04:37:41.840436Z","iopub.status.idle":"2022-08-15T04:37:46.781322Z","shell.execute_reply.started":"2022-08-15T04:37:41.840394Z","shell.execute_reply":"2022-08-15T04:37:46.780607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"active_articles = df.groupby(\"article_id\")[\"t_dat\"].max().reset_index()\nactive_articles = active_articles[active_articles[\"t_dat\"] >= \"2019-09-01\"].reset_index()\nactive_articles.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-15T04:37:46.782681Z","iopub.execute_input":"2022-08-15T04:37:46.782960Z","iopub.status.idle":"2022-08-15T04:37:51.395307Z","shell.execute_reply.started":"2022-08-15T04:37:46.782925Z","shell.execute_reply":"2022-08-15T04:37:51.394598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df[\"article_id\"].isin(active_articles[\"article_id\"])].reset_index(drop=True)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-15T04:37:51.397265Z","iopub.execute_input":"2022-08-15T04:37:51.397531Z","iopub.status.idle":"2022-08-15T04:37:58.233619Z","shell.execute_reply.started":"2022-08-15T04:37:51.397495Z","shell.execute_reply":"2022-08-15T04:37:58.232961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"week\"] = (df[\"t_dat\"].max() - df[\"t_dat\"]).dt.days // 7\ndf[\"week\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T04:37:58.235066Z","iopub.execute_input":"2022-08-15T04:37:58.235321Z","iopub.status.idle":"2022-08-15T04:37:59.659037Z","shell.execute_reply.started":"2022-08-15T04:37:58.235286Z","shell.execute_reply":"2022-08-15T04:37:59.658176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n\narticle_ids = np.concatenate([[\"placeholder\"], np.unique(df[\"article_id\"].values)])\n\nle_article = LabelEncoder()\nle_article.fit(article_ids)\ndf[\"article_id\"] = le_article.transform(df[\"article_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-15T04:37:59.660454Z","iopub.execute_input":"2022-08-15T04:37:59.660715Z","iopub.status.idle":"2022-08-15T04:38:45.829350Z","shell.execute_reply.started":"2022-08-15T04:37:59.660682Z","shell.execute_reply":"2022-08-15T04:38:45.828565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WEEK_HIST_MAX = 5\n\ndef create_dataset(df, week):\n    hist_df = df[(df[\"week\"] > week) & (df[\"week\"] <= week + WEEK_HIST_MAX)]\n    hist_df = hist_df.groupby(\"customer_id\").agg({\"article_id\": list, \"week\": list}).reset_index()\n    hist_df.rename(columns={\"week\": 'week_history'}, inplace=True)\n    \n    target_df = df[df[\"week\"] == week]\n    target_df = target_df.groupby(\"customer_id\").agg({\"article_id\": list}).reset_index()\n    target_df.rename(columns={\"article_id\": \"target\"}, inplace=True)\n    target_df[\"week\"] = week\n    \n    return target_df.merge(hist_df, on=\"customer_id\", how=\"left\")\n\nval_weeks = [0]\ntrain_weeks = [1, 2, 3, 4]\n\n\nval_df = pd.concat([create_dataset(df, w) for w in val_weeks]).reset_index(drop=True)\ntrain_df = pd.concat([create_dataset(df, w) for w in train_weeks]).reset_index(drop=True)\ntrain_df.shape, val_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-15T04:38:45.830693Z","iopub.execute_input":"2022-08-15T04:38:45.830967Z","iopub.status.idle":"2022-08-15T04:39:16.932311Z","shell.execute_reply.started":"2022-08-15T04:38:45.830930Z","shell.execute_reply":"2022-08-15T04:39:16.931595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nimport torch\nfrom tqdm import tqdm\n\nclass HMDataset(Dataset):\n    def __init__(self, df, seq_len, is_test=False):\n        self.df = df.reset_index(drop=True)\n        self.seq_len = seq_len\n        self.is_test = is_test\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        \n        if self.is_test:\n            target = torch.zeros(2).float()\n        else:\n            if not row.target:\n                target = torch.tensor([0]).int()\n            else:\n                rand_target = np.random.choice(row.target,1)\n                target = torch.tensor(rand_target).squeeze().int()\n\n            \n        article_hist = torch.zeros(self.seq_len).long()\n        week_hist = torch.ones(self.seq_len).float()\n        \n        \n        if isinstance(row.article_id, list):\n            if len(row.article_id) >= self.seq_len:\n                article_hist = torch.LongTensor(row.article_id[-self.seq_len:])\n                week_hist = (torch.LongTensor(row.week_history[-self.seq_len:]) - row.week)/WEEK_HIST_MAX/2\n            else:\n                article_hist[-len(row.article_id):] = torch.LongTensor(row.article_id)\n                week_hist[-len(row.article_id):] = (torch.LongTensor(row.week_history) - row.week)/WEEK_HIST_MAX/2\n                \n        return article_hist, week_hist, target\n    \nHMDataset(val_df, 64)[2]","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:41:55.577422Z","iopub.execute_input":"2022-08-15T05:41:55.577710Z","iopub.status.idle":"2022-08-15T05:41:55.632123Z","shell.execute_reply.started":"2022-08-15T05:41:55.577677Z","shell.execute_reply":"2022-08-15T05:41:55.631371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_lr(optimizer, epoch):\n    if epoch < 1:\n        lr = 5e-5\n    elif epoch < 6:\n        lr = 1e-3\n    elif epoch < 9:\n        lr = 1e-4\n    else:\n        lr = 1e-5\n\n    for p in optimizer.param_groups:\n        p['lr'] = lr\n    return lr\n    \ndef get_optimizer(net):\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=3e-4, betas=(0.9, 0.999),\n                                 eps=1e-08)\n    return optimizer","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:41:59.127207Z","iopub.execute_input":"2022-08-15T05:41:59.127469Z","iopub.status.idle":"2022-08-15T05:41:59.134285Z","shell.execute_reply.started":"2022-08-15T05:41:59.127440Z","shell.execute_reply":"2022-08-15T05:41:59.132358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:22:04.543207Z","iopub.execute_input":"2022-08-15T05:22:04.543486Z","iopub.status.idle":"2022-08-15T05:22:04.549661Z","shell.execute_reply.started":"2022-08-15T05:22:04.543455Z","shell.execute_reply":"2022-08-15T05:22:04.548791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HierarchicalSoftmax(nn.Module):\n    def __init__(self, ntokens, nhid, ntokens_per_class = None):\n        super(HierarchicalSoftmax, self).__init__()\n\n        # Parameters\n        self.ntokens = ntokens#the number of ouput.(72582)\n        self.nhid = nhid#dimension: the same length of customer dimension.(512)\n\n        self.ntokens_per_class = ntokens_per_class#how many children one intermidiate node.(20)\n\n        self.nclasses = int(np.ceil(self.ntokens * 1. / self.ntokens_per_class))#intermidiate nodes.(3630)\n        self.ntokens_actual = self.nclasses * self.ntokens_per_class#72600\n\n        self.layer_top_W = nn.Parameter(torch.FloatTensor(self.nhid, self.nclasses), requires_grad=True)\n        self.layer_top_b = nn.Parameter(torch.FloatTensor(self.nclasses), requires_grad=True)\n\n        self.layer_bottom_W = nn.Parameter(torch.FloatTensor(self.ntokens_per_class, self.nhid), requires_grad=True)\n        self.layer_bottom_b = nn.Parameter(torch.FloatTensor(self.nclasses), requires_grad=True)\n\n        self.init_weights()\n\n    def init_weights(self):\n\n        initrange = 0.1\n        self.layer_top_W.data.uniform_(-initrange, initrange)\n        self.layer_top_b.data.fill_(0)\n        self.layer_bottom_W.data.uniform_(-initrange, initrange)\n        self.layer_bottom_b.data.fill_(0)\n\n\n    def forward(self, inputs):\n        labels = torch.arange(self.ntokens_actual)###72600 \n        batch_size, d = inputs.size()\n\n        label_position_top = (labels / self.ntokens_per_class).long()#which position is the top layer.###[0,0,..,0,....,3659,3659]\n        label_position_bottom = (labels % self.ntokens_per_class).long()#which position is the bottom layer.###[0,1,2,..,19,1,2,...,19,..]\n        \n        layer_top_logits = torch.matmul(inputs, self.layer_top_W) + self.layer_top_b###[256, 3630]\n\n        multi_bias = self.layer_bottom_b[label_position_bottom].repeat(batch_size,1)###[256,72600]\n        \n        layer_bottom_logits = torch.matmul(inputs,self.layer_bottom_W[label_position_bottom].T) + multi_bias###[256,72600]\n\n        layer_top_logits = layer_top_logits.repeat_interleave(self.ntokens_per_class,dim=1)###[256,72600]#match the top classes and the bottom classes.\n        \n        target_logits = torch.add(layer_top_logits,layer_bottom_logits)#get the final logits\n\n        return target_logits\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:43:36.161635Z","iopub.execute_input":"2022-08-15T05:43:36.161943Z","iopub.status.idle":"2022-08-15T05:43:36.173105Z","shell.execute_reply.started":"2022-08-15T05:43:36.161910Z","shell.execute_reply":"2022-08-15T05:43:36.172362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HMModel(nn.Module):\n    def __init__(self, article_shape):\n        super(HMModel, self).__init__()\n        \n        self.article_emb = nn.Embedding(article_shape[0], embedding_dim=article_shape[1])\n        self.hier = HierarchicalSoftmax(72582, 512,ntokens_per_class = 20)\n        \n    def forward(self, inputs):\n        article_hist, week_hist = inputs[0], inputs[1]\n        x = self.article_emb(article_hist)\n        x = F.normalize(x, dim=2)###[256, 16, 512]\n        \n        x, indices = x.max(axis=1)##customer_emb[256,512]\n\n        ###get logits rather than probability to generate loss function.\n        \n        logits = self.hier(x)\n        logits = logits[:,:72582]#remove virtual leaves.\n\n        return logits\n    \n    \nmodel = HMModel((len(le_article.classes_), 512))\nmodel = model.cuda()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:43:39.599521Z","iopub.execute_input":"2022-08-15T05:43:39.599813Z","iopub.status.idle":"2022-08-15T05:43:40.008550Z","shell.execute_reply.started":"2022-08-15T05:43:39.599776Z","shell.execute_reply":"2022-08-15T05:43:40.007764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\ndef calc_map(topk_preds, target_array, k=12):\n    metric = []\n    tp, fp = 0, 0\n    \n    for pred in topk_preds:\n        if target_array[pred]:\n            tp += 1\n            metric.append(tp/(tp + fp))\n        else:\n            fp += 1\n            \n    return np.sum(metric) / min(k, target_array.sum())\n\ndef read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\ndef validate(model, val_loader, k=12):\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    maps = []\n    \n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            logits = model(inputs)\n\n            _, indices = torch.topk(logits, k, dim=1)\n\n            indices = indices.detach().cpu().numpy()\n            target = target.detach().cpu().numpy()\n            \n            for i in range(indices.shape[0]):\n                maps.append(calc_map(indices[i], target[i]))\n        \n    \n    return np.mean(maps)\n\nSEQ_LEN = 16\n\nBS = 256\nNW = 8\n\nval_dataset = HMDataset(val_df, SEQ_LEN)\nval_loader = DataLoader(val_dataset, batch_size=BS, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:43:42.186302Z","iopub.execute_input":"2022-08-15T05:43:42.186985Z","iopub.status.idle":"2022-08-15T05:43:42.202333Z","shell.execute_reply.started":"2022-08-15T05:43:42.186949Z","shell.execute_reply":"2022-08-15T05:43:42.201445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train and validate","metadata":{}},{"cell_type":"code","source":"def dice_loss(y_pred, y_true):\n    y_pred = y_pred.sigmoid()\n    intersect = (y_true*y_pred).sum(axis=1)\n    \n    return 1 - (intersect/(intersect + y_true.sum(axis=1) + y_pred.sum(axis=1))).mean()\n\n\ndef train(model, train_loader, val_loader, epochs):\n    np.random.seed(SEED)\n    \n    optimizer = get_optimizer(model)\n    scaler = torch.cuda.amp.GradScaler()\n    \n    criterion = torch.nn.functional.cross_entropy\n    \n    for e in range(epochs):\n        model.train()\n        tbar = tqdm(train_loader, file=sys.stdout)\n        \n        lr = adjust_lr(optimizer, e)\n        \n        loss_list = []\n\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            optimizer.zero_grad()\n            \n            with torch.cuda.amp.autocast():\n                logits = model(inputs)\n#                 print(logits.shape)\n#                 print(logits)\n#                 print(target.shape)\n#                 print(target)\n#                 return ###\n                loss = criterion(logits, target.long())\n            #loss.backward()\n            scaler.scale(loss).backward()\n            #optimizer.step()\n            scaler.step(optimizer)\n            scaler.update()\n            \n            loss_list.append(loss.detach().cpu().item())\n            \n            avg_loss = np.round(100*np.mean(loss_list), 4)\n\n            tbar.set_description(f\"Epoch {e+1} Loss: {avg_loss} lr: {lr}\")\n            \n    return model\n\n\nMODEL_NAME = \"exp001\"\nSEED = 0\n\ntrain_dataset = HMDataset(train_df, SEQ_LEN)\ntrain_loader = DataLoader(train_dataset, batch_size=BS, shuffle=True, num_workers=NW,\n                          pin_memory=False, drop_last=True)\n\nmodel = train(model, train_loader, val_loader, epochs=10)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T05:43:44.943969Z","iopub.execute_input":"2022-08-15T05:43:44.944878Z","iopub.status.idle":"2022-08-15T05:43:58.269718Z","shell.execute_reply.started":"2022-08-15T05:43:44.944828Z","shell.execute_reply":"2022-08-15T05:43:58.267552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Finetune with more recent data for submission (include validation set)","metadata":{}},{"cell_type":"code","source":"train_dataset = HMDataset(train_df[train_df[\"week\"] < 4].append(val_df), SEQ_LEN)\ntrain_loader = DataLoader(train_dataset, batch_size=BS, shuffle=True, num_workers=NW,\n                          pin_memory=False, drop_last=True)\n\nmodel = train(model, train_loader, val_loader, epochs=10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv').drop(\"prediction\", axis=1)\nprint(test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T23:32:45.079468Z","iopub.execute_input":"2022-08-13T23:32:45.080067Z","iopub.status.idle":"2022-08-13T23:32:49.993336Z","shell.execute_reply.started":"2022-08-13T23:32:45.080029Z","shell.execute_reply":"2022-08-13T23:32:49.992564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_test_dataset(test_df):\n    week = -1\n    test_df[\"week\"] = week\n    \n    hist_df = df[(df[\"week\"] > week) & (df[\"week\"] <= week + WEEK_HIST_MAX)]\n    hist_df = hist_df.groupby(\"customer_id\").agg({\"article_id\": list, \"week\": list}).reset_index()\n    hist_df.rename(columns={\"week\": 'week_history'}, inplace=True)\n    \n    \n    return test_df.merge(hist_df, on=\"customer_id\", how=\"left\")\n\ntest_df = create_test_dataset(test_df)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T23:32:54.047829Z","iopub.execute_input":"2022-08-13T23:32:54.048633Z","iopub.status.idle":"2022-08-13T23:33:00.584837Z","shell.execute_reply.started":"2022-08-13T23:32:54.048581Z","shell.execute_reply":"2022-08-13T23:33:00.584118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"article_id\"].isnull().mean()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T13:19:46.235925Z","iopub.execute_input":"2022-03-18T13:19:46.236179Z","iopub.status.idle":"2022-03-18T13:19:46.313921Z","shell.execute_reply.started":"2022-03-18T13:19:46.236144Z","shell.execute_reply":"2022-03-18T13:19:46.313072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = HMDataset(test_df, SEQ_LEN, is_test=True)\ntest_loader = DataLoader(test_ds, batch_size=BS, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)\n\n\ndef inference(model, loader, k=12):\n    model.eval()\n    \n    tbar = tqdm(loader, file=sys.stdout)\n    \n    preds = []\n    \n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            logits = model(inputs)\n\n            _, indices = torch.topk(logits, k, dim=1)\n\n            indices = indices.detach().cpu().numpy()\n            target = target.detach().cpu().numpy()\n\n            for i in range(indices.shape[0]):\n                preds.append(\" \".join(list(le_article.inverse_transform(indices[i]))))\n        \n    \n    return preds\n\n\ntest_df[\"prediction\"] = inference(model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T23:33:09.284893Z","iopub.execute_input":"2022-08-13T23:33:09.285599Z","iopub.status.idle":"2022-08-13T23:33:30.579046Z","shell.execute_reply.started":"2022-08-13T23:33:09.285560Z","shell.execute_reply":"2022-08-13T23:33:30.576211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False, columns=[\"customer_id\", \"prediction\"])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T13:20:11.209147Z","iopub.status.idle":"2022-03-18T13:20:11.209811Z","shell.execute_reply.started":"2022-03-18T13:20:11.209573Z","shell.execute_reply":"2022-03-18T13:20:11.209598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}