{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# H&M Personalized Fashion Recommendations\n- 日本語コメント付きのベースラインモデルです\n- 以下のnotebookをリファクタしたものになります\n    - 元notebook\n        - H&M Pure Pytorch Baseline\n        - https://www.kaggle.com/code/aerdem4/h-m-pure-pytorch-baseline\n    - リファクタ内容\n        - DEBUGモード追加\n        - コメント付与\n        - 定数のみ切り出し","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:04:32.719933Z","iopub.execute_input":"2022-04-03T04:04:32.720630Z","iopub.status.idle":"2022-04-03T04:04:32.725061Z","shell.execute_reply.started":"2022-04-03T04:04:32.720591Z","shell.execute_reply":"2022-04-03T04:04:32.724067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = True\n\n# ファイルパス\n# baselineではTRANSACTIONのみ学習に使用している\nTRANSACTION_PATH = \"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\"\nARTICLES_PATH = \"../input/h-and-m-personalized-fashion-recommendations/articles.csv\"\nCUSTOMERS_PATH = \"../input/h-and-m-personalized-fashion-recommendations/customers.csv\"\nSAMPLE_SUB_PATH = '../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv'\nIMAGES_DIR = \"../input/h-and-m-personalized-fashion-recommendations/images/\"\n\n# 前処理\nACTIVE_DATE_TH = \"2019-09-01\" # この日付より最近購入された商品のみ学習/推論対象とする\nWEEK_HIST_MAX = 5 # 過去何日分の購買履歴を特徴量とするか\nVAL_WEEKS = [0] # どの週のデータをvalidationデータとするか\nTRAIN_WEEKS = [1, 2, 3, 4] # どの週のデータをtrainデータとするか\n\n# データセット\nSEQ_LEN = 16 # 何個分の購買履歴まで使うか\nBS = 256 # バッチサイズ\nNW = 8 # ワーカーズ\n\n# 学習\nDIM_ARTICLE = 512 # article_idを何次元の特徴量にembeddingするか\nINIT_LR = 3e-4\nBETAS = (0.9, 0.999)\nEPS=1e-08\nLR_DICT = { # n-epoch目の時、n < KEYならVALをlrとする\n    1: 5e-5,\n    6: 1e-3, \n    9: 1e-4,\n    100: 1e-5\n}\nMODEL_NAME = \"exp001\"\nSEED = 0\nBASE_EPOCHS = 10 # ベースモデルのエポック数\nFT_EPOCHS = 10 # fine tuningのエポック数\nFT_WEEKS = 4 # fine tuningする際使用するweek数","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:04:32.749732Z","iopub.execute_input":"2022-04-03T04:04:32.749942Z","iopub.status.idle":"2022-04-03T04:04:32.757295Z","shell.execute_reply.started":"2022-04-03T04:04:32.749916Z","shell.execute_reply":"2022-04-03T04:04:32.756642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# データ読み込み\ndf = pd.read_csv(TRANSACTION_PATH, dtype={\"article_id\": str})\nprint(df.shape)\ndf.head()\n\nif DEBUG:\n    print(\"DEBUGモードで処理を実行します\")\n    BASE_EPOCHS = 1\n    FT_EPOCHS = 1\n    SEQ_LEN = 5\n    DIM_ARTICLE = 8\n    df = df.sample(frac=0.005)\n    print(df.shape)\n    display(df.head())","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:04:32.775842Z","iopub.execute_input":"2022-04-03T04:04:32.776131Z","iopub.status.idle":"2022-04-03T04:05:11.974317Z","shell.execute_reply.started":"2022-04-03T04:04:32.776102Z","shell.execute_reply":"2022-04-03T04:05:11.973442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 前処理(1)","metadata":{}},{"cell_type":"code","source":"# datetime型に変換\ndf[\"t_dat\"] = pd.to_datetime(df[\"t_dat\"])\ndf[\"t_dat\"].max()","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:11.976041Z","iopub.execute_input":"2022-04-03T04:05:11.976308Z","iopub.status.idle":"2022-04-03T04:05:12.051196Z","shell.execute_reply.started":"2022-04-03T04:05:11.976270Z","shell.execute_reply":"2022-04-03T04:05:12.050322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 直近で購入された商品一覧を抽出する\nactive_articles = df.groupby(\"article_id\")[\"t_dat\"].max().reset_index()\nactive_articles = active_articles[active_articles[\"t_dat\"] >= ACTIVE_DATE_TH].reset_index()\nactive_articles.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.052587Z","iopub.execute_input":"2022-04-03T04:05:12.053292Z","iopub.status.idle":"2022-04-03T04:05:12.197089Z","shell.execute_reply.started":"2022-04-03T04:05:12.053250Z","shell.execute_reply":"2022-04-03T04:05:12.196302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 購買履歴のうち直近で購入された商品IDに対応する履歴のみを抽出\ndf = df[df[\"article_id\"].isin(active_articles[\"article_id\"])].reset_index(drop=True)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.199114Z","iopub.execute_input":"2022-04-03T04:05:12.199352Z","iopub.status.idle":"2022-04-03T04:05:12.328411Z","shell.execute_reply.started":"2022-04-03T04:05:12.199318Z","shell.execute_reply":"2022-04-03T04:05:12.327578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"week\"カラムを追加(weekが小さいほど最近)\ndf[\"week\"] = (df[\"t_dat\"].max() - df[\"t_dat\"]).dt.days // 7\ndf[\"week\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.329767Z","iopub.execute_input":"2022-04-03T04:05:12.330182Z","iopub.status.idle":"2022-04-03T04:05:12.346118Z","shell.execute_reply.started":"2022-04-03T04:05:12.330141Z","shell.execute_reply":"2022-04-03T04:05:12.345379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# articleIdをラベルに変換\nfrom sklearn.preprocessing import LabelEncoder\narticle_ids = np.concatenate([[\"placeholder\"], np.unique(df[\"article_id\"].values)])\nle_article = LabelEncoder()\nle_article.fit(article_ids)\ndf[\"article_id\"] = le_article.transform(df[\"article_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.347235Z","iopub.execute_input":"2022-04-03T04:05:12.348598Z","iopub.status.idle":"2022-04-03T04:05:12.664308Z","shell.execute_reply.started":"2022-04-03T04:05:12.348559Z","shell.execute_reply":"2022-04-03T04:05:12.663583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    print(\"データ(head)\")\n    display(df.head())\n    print(\"データ(describe)\")\n    display(df.describe())\n    print(\"データ(info)\")\n    display(df.info())","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.665490Z","iopub.execute_input":"2022-04-03T04:05:12.665735Z","iopub.status.idle":"2022-04-03T04:05:12.778212Z","shell.execute_reply.started":"2022-04-03T04:05:12.665702Z","shell.execute_reply":"2022-04-03T04:05:12.777373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 前処理(2)","metadata":{}},{"cell_type":"code","source":"def create_dataset(df, week, debug=False) -> pd.DataFrame:\n    \"\"\"\n    データセットを作成\n    各顧客が指定した週で購入したものと、過去n週で購入したもの\n    \n    【データの説明】\n        customer_id: 顧客ID\n        target: (その週に)何を購入したか←予測したいもの\n        week: 週\n        article_id: (過去n週で)何を購入したか\n        week_history: (過去n週の)どの週で購入したか\n        \n    【処理の流れ】\n    1.指定した週の過去n週間分のデータを抽出する\n    2.顧客(customer_id)毎に買った商品リスト(article_idリスト)、購入したweekリストを作成\n    3.購入したweekリストをweek_historyとする\n    4.指定した週のレコードをtarget_dfとして抽出\n    5.target_dfも顧客(customer_id)毎に買った商品リスト(article_idリスト)を作成\n    6.買った商品リストのカラムをtargetに変換\n    7.target_dfのweekカラムに指定した週の値を代入\n    8.customer_idでleft join\n    \"\"\"\n    hist_df = df[(df[\"week\"] > week) & (df[\"week\"] <= week + WEEK_HIST_MAX)]\n    hist_df = hist_df.groupby(\"customer_id\").agg({\"article_id\": list, \"week\": list}).reset_index()\n    hist_df.rename(columns={\"week\": 'week_history'}, inplace=True)\n    if debug:\n        display(hist_df.head())\n    \n    target_df = df[df[\"week\"] == week]\n    target_df = target_df.groupby(\"customer_id\").agg({\"article_id\": list}).reset_index()\n    target_df.rename(columns={\"article_id\": \"target\"}, inplace=True)\n    target_df[\"week\"] = week\n    result_df = target_df.merge(hist_df, on=\"customer_id\", how=\"left\")\n    if debug:\n        display(result_df.head())\n    return result_df","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.779616Z","iopub.execute_input":"2022-04-03T04:05:12.780339Z","iopub.status.idle":"2022-04-03T04:05:12.790204Z","shell.execute_reply.started":"2022-04-03T04:05:12.780299Z","shell.execute_reply":"2022-04-03T04:05:12.789504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# データセット作成\nval_df = pd.concat([create_dataset(df, w) for w in VAL_WEEKS]).reset_index(drop=True)\ntrain_df = pd.concat([create_dataset(df, w) for w in TRAIN_WEEKS]).reset_index(drop=True)\ntrain_df.shape, val_df.shape\n\nif DEBUG:\n    print(\"validationデータ(head)\")\n    display(val_df.head())\n    print(\"trainデータ(head + tail)\")\n    display(train_df.head())\n    display(train_df.tail())","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:12.792410Z","iopub.execute_input":"2022-04-03T04:05:12.793028Z","iopub.status.idle":"2022-04-03T04:05:13.728708Z","shell.execute_reply.started":"2022-04-03T04:05:12.792989Z","shell.execute_reply":"2022-04-03T04:05:13.727888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデル学習","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nimport torch\nfrom tqdm import tqdm\n\nclass HMDataset(Dataset):\n    \"\"\"\n    pytorch Datasetを継承する際には以下の2つの関数をoverrideする\n    ①.def __getitem__(self, index: int):\n        indexを与えられたときに対応するデータを返す  \n    ②.def __len__(self):\n        データのサンプル数を返す\n    \"\"\"\n    def __init__(self, df, seq_len, is_test=False):\n        \"\"\"\n        df: 前処理したデータセット\n        seq_len: 特徴量の次元数\n        is_test: テストデータか否か\n        \"\"\"\n        self.df = df.reset_index(drop=True)\n        self.seq_len = seq_len\n        self.is_test = is_test\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index):\n        \"\"\"\n        対象indexデータを加工して返す\n        \n        特徴量：\n        article_hist: 過去n週で購買した商品(要素数=seq_len)\n        week_hist: article_histの商品をいつ購入したかを正規化(0~1)した値(要素数=seq_len)\n        目的変数: \n        target: この週どの商品を購入したか(要素数=len(article_ids))\n        \"\"\"\n        \n        # dfのindex番目の行を抽出\n        row = self.df.iloc[index]\n        \n        if self.is_test:\n            # testならtargetは長さ2のlist\n            target = torch.zeros(2).float()\n        else:\n            # trainならtargetは長さが\"対象とする商品数\"のlist\n            target = torch.zeros(len(article_ids)).float()\n            \n            # targetのうち顧客が購入したもの(article_idのindex)を1.0にする\n            for t in row.target:\n                target[t] = 1.0\n            \n        # 購入履歴は長さseq_lenのlist\n        article_hist = torch.zeros(self.seq_len).long()\n        week_hist = torch.ones(self.seq_len).float()\n        \n        # 何かしらの購入履歴があるか\n        if isinstance(row.article_id, list):\n            \n            # 購入履歴の数がseq_lenより長いなら直近の履歴から抽出\n            if len(row.article_id) >= self.seq_len:\n                article_hist = torch.LongTensor(row.article_id[-self.seq_len:])\n                week_hist = (torch.LongTensor(row.week_history[-self.seq_len:]) - row.week)/WEEK_HIST_MAX/2\n            # 購入履歴の数がseq_lenより短いならarticle_hist/week_histの前半indexに購入履歴を格納\n            else:   \n                article_hist[-len(row.article_id):] = torch.LongTensor(row.article_id)\n                week_hist[-len(row.article_id):] = (torch.LongTensor(row.week_history) - row.week)/WEEK_HIST_MAX/2\n                \n        return article_hist, week_hist, target\n    \nif DEBUG:\n    print(\"train_dfでHMDataSetを作成し中身を確認\")\n    print(\"[0]article_hist: 過去n週で購入した商品一覧\")\n    print(\"[1]week_hist: 各商品を過去n週のうちどの週で購入したか一覧\")\n    print(\"[2]target: 今週どの商品を購入したか0:購入しなかった、1:購入した(要素数: len(article_ids)\")\n    print(\"-\"*60)\n    print()\n    print(\"データセットの中身\")\n    sample = HMDataset(train_df, 64)[0]\n    display(sample)\n    print()\n    print(\"各特徴量のlen\")\n    print(len(sample[0]))\n    print(len(sample[1]))\n    print(len(sample[2]))","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:13.731836Z","iopub.execute_input":"2022-04-03T04:05:13.732095Z","iopub.status.idle":"2022-04-03T04:05:13.756594Z","shell.execute_reply.started":"2022-04-03T04:05:13.732059Z","shell.execute_reply":"2022-04-03T04:05:13.755786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_lr(optimizer, epoch):\n    \"\"\"\n    エポックによって特徴量を変える\n    \"\"\"\n    \n    for epoch_th in LR_DICT.keys():\n        if epoch < epoch_th:\n            lr = LR_DICT[epoch_th]\n\n    for p in optimizer.param_groups:\n        p['lr'] = lr\n    return lr\n    \ndef get_optimizer(net):\n    \"\"\"\n    最適化関数を取得\n    \"\"\"\n    optimizer = torch.optim.Adam(\n        params=filter(lambda p: p.requires_grad, net.parameters()),\n        lr=INIT_LR,\n        betas=BETAS,\n        eps=EPS\n    )\n    return optimizer","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:13.758026Z","iopub.execute_input":"2022-04-03T04:05:13.758720Z","iopub.status.idle":"2022-04-03T04:05:13.765578Z","shell.execute_reply.started":"2022-04-03T04:05:13.758680Z","shell.execute_reply":"2022-04-03T04:05:13.764779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\nclass HMModel(nn.Module):\n    \"\"\"\n    ネットワークを定義\n    一方通行ではない複雑なモデル（ネットワーク）を構築するには、torch.nn.Moduleを継承したサブクラスを定義する\n    \n    基本的な書き方\n    __init__()\n    　- 使用するモジュール（レイヤー）のインスタンスを生成\n     ※super().__init__()を忘れないように注意\n    forward()\n    　- レイヤーを所望の順番で適用していく\n    \"\"\"\n    def __init__(self, article_shape):\n        \"\"\"\n        article_shape = (商品の種類数, ベクトルの次元)\n        \"\"\"\n        # おまじない\n        super(HMModel, self).__init__()\n        \n        # レイヤー(Sequentialはレイヤーのひとまとまり?)\n        # nn.Embedding: IDに対応する初期ベクトルを作成(IDをベクトルに埋め込む)\n        # nn.Parameter: レイヤーのパラメータ\n        self.article_emb = nn.Embedding(num_embeddings=article_shape[0], embedding_dim=article_shape[1])\n        self.article_likelihood = nn.Parameter(torch.zeros(article_shape[0]), requires_grad=True)\n        self.top = nn.Sequential(\n            nn.Conv1d(3, 32, kernel_size=1),\n            nn.LeakyReLU(),\n            nn.Conv1d(32, 8, kernel_size=1),\n            nn.LeakyReLU(),\n            nn.Conv1d(8, 1, kernel_size=1)\n        )\n        \n    def forward(self, inputs):\n        \"\"\"\n        inputs: 特徴量\n        \"\"\"\n        article_hist, week_hist = inputs[0], inputs[1]\n        \n        # TODO: 前処理\n        x = self.article_emb(article_hist)\n\n        x = F.normalize(x, dim=2)\n        x = x@F.normalize(self.article_emb.weight).T\n        \n        x, indices = x.max(axis=1)\n        x = x.clamp(1e-3, 0.999)\n        x = -torch.log(1/x - 1)\n        \n        max_week = week_hist.unsqueeze(2).repeat(1, 1, x.shape[-1]).gather(1, indices.unsqueeze(1).repeat(1, week_hist.shape[1], 1))\n        max_week = max_week.mean(axis=1).unsqueeze(1)\n        \n        x = torch.cat([x.unsqueeze(1), max_week,\n                       self.article_likelihood[None, None, :].repeat(x.shape[0], 1, 1)], axis=1)\n        \n        # ネットワークで計算\n        x = self.top(x).squeeze(1)\n        return x\n    \n    \nmodel = HMModel((len(le_article.classes_), DIM_ARTICLE))\nmodel = model.cuda()\nif DEBUG:\n    print(\"モデル構造を可視化\")\n    print(model)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:13.767140Z","iopub.execute_input":"2022-04-03T04:05:13.767941Z","iopub.status.idle":"2022-04-03T04:05:13.787753Z","shell.execute_reply.started":"2022-04-03T04:05:13.767903Z","shell.execute_reply":"2022-04-03T04:05:13.786837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\ndef calc_map(topk_preds, target_array, k=12):\n    \"\"\"\n    MAPスコアを計算\n    topk_perds: 予測結果\n    target_array: 正解\n    k: 予測件数(今回のコンペだと上位12件を予測)\n    \"\"\"\n    metric = []\n    tp, fp = 0, 0\n    \n    # 予測結果を1件ずつ見ていきtp/fpを計算\n    for pred in topk_preds:\n        if target_array[pred]:\n            tp += 1\n            metric.append(tp/(tp + fp))\n        else:\n            fp += 1\n            \n    return np.sum(metric) / min(k, target_array.sum())\n\ndef read_data(data):\n    \"\"\"\n    TODO: おまじない的なやつ?\n    \"\"\"\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\ndef validate(model, val_loader, k=12):\n    \"\"\"\n    バリデーション\n    \"\"\"\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    maps = []\n    \n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            # inputs(入力特徴量),target(正解)を抽出\n            inputs, target = read_data(data)\n            \n            # モデルで予測\n            logits = model(inputs)\n            \n            # topk(スコア上位k件を抽出)\n            _, indices = torch.topk(logits, k, dim=1)\n\n            # GPU→CPUへ\n            indices = indices.detach().cpu().numpy() # 予測結果\n            target = target.detach().cpu().numpy() # 正解\n\n            # スコア計算\n            for i in range(indices.shape[0]):\n                maps.append(calc_map(indices[i], target[i]))\n        \n    return np.mean(maps)\n\n# DataSet: 学習に使うデータ(前処理などを定義している)\nval_dataset = HMDataset(val_df, SEQ_LEN)\n\n# DataLoader: DataSetをバッチで処理するためのもの\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=BS,\n    shuffle=False,\n    num_workers=NW,\n    pin_memory=False,\n    drop_last=False\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:13.789224Z","iopub.execute_input":"2022-04-03T04:05:13.789586Z","iopub.status.idle":"2022-04-03T04:05:13.804139Z","shell.execute_reply.started":"2022-04-03T04:05:13.789548Z","shell.execute_reply":"2022-04-03T04:05:13.803399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train and validate","metadata":{}},{"cell_type":"code","source":"def dice_loss(y_pred, y_true):\n    \"\"\"\n    dice_loss\n    Dice損失は2つの要素の類似度の評価するために使われているDice係数(F値)を損失として用いたもの\n    \"\"\"\n    y_pred = y_pred.sigmoid()\n    intersect = (y_true*y_pred).sum(axis=1)\n    \n    return 1 - (intersect/(intersect + y_true.sum(axis=1) + y_pred.sum(axis=1))).mean()\n\n\ndef train(model, train_loader, val_loader, epochs):\n    np.random.seed(SEED)\n    \n    # 最適化関数を定義\n    optimizer = get_optimizer(model)\n    scaler = torch.cuda.amp.GradScaler() # ←高速化のための何かっぽい\n\n    # ロスを定義\n    criterion = torch.nn.BCEWithLogitsLoss()\n    \n    for e in range(epochs):\n        model.train()\n        tbar = tqdm(train_loader, file=sys.stdout)\n        \n        lr = adjust_lr(optimizer, e)\n        \n        loss_list = []\n\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            optimizer.zero_grad()\n            \n            with torch.cuda.amp.autocast():\n                logits = model(inputs)\n                \n                # ロスは2種類を組み合わせている(完全一致と類似度？)\n                loss = criterion(logits, target) + dice_loss(logits, target)\n            \n            #loss.backward()\n            scaler.scale(loss).backward()\n            #optimizer.step()\n            scaler.step(optimizer)\n            scaler.update()\n            \n            loss_list.append(loss.detach().cpu().item())\n            \n            avg_loss = np.round(100*np.mean(loss_list), 4)\n\n            tbar.set_description(f\"Epoch {e+1} Loss: {avg_loss} lr: {lr}\")\n            \n        val_map = validate(model, val_loader)\n\n        log_text = f\"Epoch {e+1}\\nTrain Loss: {avg_loss}\\nValidation MAP: {val_map}\\n\"\n            \n        print(log_text)\n        \n        #logfile = open(f\"models/{MODEL_NAME}_{SEED}.txt\", 'a')\n        #logfile.write(log_text)\n        #logfile.close()\n    return model\n\n\n\n\ntrain_dataset = HMDataset(train_df, SEQ_LEN)\ntrain_loader = DataLoader(train_dataset, batch_size=BS, shuffle=True, num_workers=NW,\n                          pin_memory=False, drop_last=True)\n\nmodel = train(model, train_loader, val_loader, epochs=BASE_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:13.805736Z","iopub.execute_input":"2022-04-03T04:05:13.806119Z","iopub.status.idle":"2022-04-03T04:05:18.747430Z","shell.execute_reply.started":"2022-04-03T04:05:13.806081Z","shell.execute_reply":"2022-04-03T04:05:18.746540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Finetune with more recent data for submission (include validation set)\n今回のコンペは購買履歴の次の週の結果を予測する  \n→ 直近のデータの方が重要と考えられる  \n→ 最後にバリデーションデータを使ってfine tuningし直す","metadata":{}},{"cell_type":"code","source":"train_dataset = HMDataset(train_df[train_df[\"week\"] < FT_WEEKS].append(val_df), SEQ_LEN)\ntrain_loader = DataLoader(train_dataset, batch_size=BS, shuffle=True, num_workers=NW,\n                          pin_memory=False, drop_last=True)\n\nmodel = train(model, train_loader, val_loader, epochs=FT_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:18.749245Z","iopub.execute_input":"2022-04-03T04:05:18.749566Z","iopub.status.idle":"2022-04-03T04:05:23.414934Z","shell.execute_reply.started":"2022-04-03T04:05:18.749520Z","shell.execute_reply":"2022-04-03T04:05:23.414078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(SAMPLE_SUB_PATH).drop(\"prediction\", axis=1)\nprint(test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:05:23.416579Z","iopub.execute_input":"2022-04-03T04:05:23.417414Z","iopub.status.idle":"2022-04-03T04:05:25.797446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_test_dataset(test_df):\n    \"\"\"\n    テストデータ(submitする際の予測データ)を作成する\n    \"\"\"\n    week = -1\n    test_df[\"week\"] = week\n    \n    # 過去n週間で買ったものを列として追加していく\n    hist_df = df[(df[\"week\"] > week) & (df[\"week\"] <= week + WEEK_HIST_MAX)]\n    hist_df = hist_df.groupby(\"customer_id\").agg({\"article_id\": list, \"week\": list}).reset_index()\n    hist_df.rename(columns={\"week\": 'week_history'}, inplace=True)\n    \n    return test_df.merge(hist_df, on=\"customer_id\", how=\"left\")\n\ntest_df = create_test_dataset(test_df)\ntest_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 8割くらいの人がnull\n# →80%の人はヒントなしになってしまっているので別の特徴量が必要そう\ntest_df[\"article_id\"].isnull().mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# テストデータに対する推論 + submission.csvの作成","metadata":{}},{"cell_type":"code","source":"test_ds = HMDataset(test_df, SEQ_LEN, is_test=True)\ntest_loader = DataLoader(test_ds, batch_size=BS, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)\n\n\ndef inference(model, loader, k=12):\n    model.eval()\n    \n    tbar = tqdm(loader, file=sys.stdout)\n    \n    preds = []\n    \n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            logits = model(inputs)\n\n            _, indices = torch.topk(logits, k, dim=1)\n\n            indices = indices.detach().cpu().numpy()\n            target = target.detach().cpu().numpy()\n\n            for i in range(indices.shape[0]):\n                preds.append(\" \".join(list(le_article.inverse_transform(indices[i]))))\n        \n    \n    return preds\n\n\ntest_df[\"prediction\"] = inference(model, test_loader)","metadata":{"execution":{"iopub.status.idle":"2022-04-03T04:19:51.452255Z","shell.execute_reply.started":"2022-04-03T04:05:27.124106Z","shell.execute_reply":"2022-04-03T04:19:51.451363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv(\"submission.csv\", index=False, columns=[\"customer_id\", \"prediction\"])","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:19:51.454008Z","iopub.execute_input":"2022-04-03T04:19:51.454439Z","iopub.status.idle":"2022-04-03T04:20:03.257097Z","shell.execute_reply.started":"2022-04-03T04:19:51.454400Z","shell.execute_reply":"2022-04-03T04:20:03.256341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fin!","metadata":{"execution":{"iopub.status.busy":"2022-04-03T04:20:03.258479Z","iopub.execute_input":"2022-04-03T04:20:03.258728Z","iopub.status.idle":"2022-04-03T04:20:03.263033Z","shell.execute_reply.started":"2022-04-03T04:20:03.258694Z","shell.execute_reply":"2022-04-03T04:20:03.262253Z"},"trusted":true},"execution_count":null,"outputs":[]}]}