{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n- PyTorch NNet + Entity Embeddings starter code\n- Swish activation \n- Weighted average cat using softmax layer insted of torch cat\n- BCEweighted loss + label smoothing\n\n# Improvements maybe\n- Use ArcFace or add triplet loss with bce for score improvement\n\n# acknowledgement\n- Y.NAKAMA great [notebook](https://www.kaggle.com/yasufuminakama/herbarium-2020-pytorch-resnet18-train/notebook)\n- XGBoost Starter - [0.793] [link](https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793)\n- stackoverflow for weighted average layer (late fusion) [link](https://stackoverflow.com/questions/62877879/implementing-late-fusion-in-keras)\n\nIf this notebook is helpful, feel free to upvote :)","metadata":{}},{"cell_type":"code","source":"!pip install -q --upgrade wandb","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:53:32.388661Z","iopub.execute_input":"2022-06-10T10:53:32.389078Z","iopub.status.idle":"2022-06-10T10:53:50.708625Z","shell.execute_reply.started":"2022-06-10T10:53:32.388990Z","shell.execute_reply":"2022-06-10T10:53:50.707359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport gc\nimport random\nimport math\nimport time\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\n\n\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.impute import KNNImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import StandardScaler\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, SequentialSampler, RandomSampler\nfrom torch.optim import Adam, SGD\nfrom fastai.layers import SigmoidRange\nfrom torch.optim.lr_scheduler import (CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau,\nOneCycleLR)\n\nfrom torch.cuda.amp import autocast, GradScaler\n\n\ndevice=(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:53:50.711291Z","iopub.execute_input":"2022-06-10T10:53:50.711724Z","iopub.status.idle":"2022-06-10T10:53:53.465928Z","shell.execute_reply.started":"2022-06-10T10:53:50.711680Z","shell.execute_reply":"2022-06-10T10:53:53.464687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Directory settings","metadata":{}},{"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:53:53.468898Z","iopub.execute_input":"2022-06-10T10:53:53.469913Z","iopub.status.idle":"2022-06-10T10:53:53.476193Z","shell.execute_reply.started":"2022-06-10T10:53:53.469849Z","shell.execute_reply":"2022-06-10T10:53:53.474942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet(\"../input/amex-data-engineering/train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:53:53.479803Z","iopub.execute_input":"2022-06-10T10:53:53.480600Z","iopub.status.idle":"2022-06-10T10:54:13.069207Z","shell.execute_reply.started":"2022-06-10T10:53:53.480321Z","shell.execute_reply":"2022-06-10T10:54:13.068127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:13.070804Z","iopub.execute_input":"2022-06-10T10:54:13.071225Z","iopub.status.idle":"2022-06-10T10:54:13.111051Z","shell.execute_reply.started":"2022-06-10T10:54:13.071183Z","shell.execute_reply":"2022-06-10T10:54:13.109814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:13.112257Z","iopub.execute_input":"2022-06-10T10:54:13.113152Z","iopub.status.idle":"2022-06-10T10:54:13.120976Z","shell.execute_reply.started":"2022-06-10T10:54:13.113110Z","shell.execute_reply":"2022-06-10T10:54:13.119706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef amex_metric(y_true, y_pred, return_components=False) -> float:\n    \"\"\"Amex metric for ndarrays\"\"\"\n    def top_four_percent_captured(df) -> float:\n        \"\"\"Corresponds to the recall for a threshold of 4 %\"\"\"\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(df) -> float:\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(df) -> float:\n        \"\"\"Corresponds to 2 * AUC - 1\"\"\"\n        df2 = pd.DataFrame({'target': df.target, 'prediction': df.target})\n        df2.sort_values('prediction', ascending=False, inplace=True)\n        return weighted_gini(df) / weighted_gini(df2)\n\n    df = pd.DataFrame({'target': y_true.ravel(), 'prediction': y_pred.ravel()})\n    df.sort_values('prediction', ascending=False, inplace=True)\n    g = normalized_weighted_gini(df)\n    d = top_four_percent_captured(df)\n    print(\"G: {:.6f}, D: {:.6f}, ALL: {:6f}\".format(g, d, 0.5*(g+d)))\n    if return_components: return g, d, 0.5 * (g + d)\n    return 0.5 * (g + d)\n\ndef get_score(y_true, y_pred):\n    score = amex_metric(y_true, y_pred)\n    return score\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch(seed=45)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:13.122846Z","iopub.execute_input":"2022-06-10T10:54:13.124031Z","iopub.status.idle":"2022-06-10T10:54:13.149238Z","shell.execute_reply.started":"2022-06-10T10:54:13.123969Z","shell.execute_reply":"2022-06-10T10:54:13.148089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CAT_FEATURES = [col for col in train.columns if (col.split(\"_\")[-1] in ['count', 'nunique']) | (col in [\"B_30_last\",\"B_38_last\",\"D_114_last\",\"D_116_last\",\"D_117_last\",\"D_120_last\",\"D_126_last\",\"D_63_last\",\"D_64_last\",\"D_66_last\",\"D_68_last\"])]\nNUM_FEATURES = [col for col in train.columns if (col.split(\"_\")[-1] in ['mean', 'std', 'min', 'max', 'last']) & (col not in [\"B_30_last\",\"B_38_last\",\"D_114_last\",\"D_116_last\",\"D_117_last\",\"D_120_last\",\"D_126_last\",\"D_63_last\",\"D_64_last\",\"D_66_last\",\"D_68_last\"])]","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:13.151586Z","iopub.execute_input":"2022-06-10T10:54:13.152608Z","iopub.status.idle":"2022-06-10T10:54:13.162035Z","shell.execute_reply.started":"2022-06-10T10:54:13.152566Z","shell.execute_reply":"2022-06-10T10:54:13.160695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Encoding","metadata":{}},{"cell_type":"code","source":"def encode(df,cols):\n    enc =  {}\n    for col in cols:\n        print(col)\n        lbencoder = LabelEncoder()\n        lb = lbencoder.fit(df[col].values)\n        df[col]=lb.transform(df[col].values)\n        enc[col]=lb\n        \n    return df,enc\n\ntrain ,_ = encode(train, CAT_FEATURES)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:13.163626Z","iopub.execute_input":"2022-06-10T10:54:13.164782Z","iopub.status.idle":"2022-06-10T10:54:14.867100Z","shell.execute_reply.started":"2022-06-10T10:54:13.164739Z","shell.execute_reply":"2022-06-10T10:54:14.864504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    apex=False\n    debug=False\n    print_freq=100\n    num_workers=2\n    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts','OneCycleLR']\n    epochs=20\n    # CosineAnnealingLR params\n    cosanneal_params={\n        'T_max':6,\n        'eta_min':1e-5,\n        'last_epoch':-1\n    }\n    #ReduceLROnPlateau params\n    reduce_params={\n        'mode':'min',\n        'factor':0.2,\n        'patience':4,\n        'eps':1e-6,\n        'verbose':True\n    }\n    # CosineAnnealingWarmRestarts params\n    cosanneal_res_params={\n        'T_0':5,\n        'eta_min':1e-4,\n        'T_mult':1,\n        'last_epoch':-1\n    }\n    onecycle_params={\n        'pct_start':0.1,\n        'div_factor':1e2,\n        'max_lr':1e-3\n    }\n    batch_size=32\n    lr=4e-3\n    weight_decay=1e-3\n    gradient_accumulation_steps=2\n    max_grad_norm=1000\n    target_size=1\n    nfolds=5\n    trn_folds=[0, 1, 2, 3, 4]\n    target_col=\"target\"\n    model_name=\"nnet\"\n    train=True\n    cat_dims = [train[col].nunique() for col in CAT_FEATURES]\n    cat_embs = [(dim, min(50,(dim+1)//2)) for dim in cat_dims]\n    #num_features=len(num_columns)\n    seed=42\n    \nif CFG.debug:\n    CFG.epochs=2\n    train=train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True).fillna(method=\"ffill\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:14.874363Z","iopub.execute_input":"2022-06-10T10:54:14.877251Z","iopub.status.idle":"2022-06-10T10:54:15.093399Z","shell.execute_reply.started":"2022-06-10T10:54:14.877208Z","shell.execute_reply":"2022-06-10T10:54:15.092242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV Split","metadata":{}},{"cell_type":"code","source":"skf = KFold(n_splits=5, shuffle=True, random_state=42)\nfor fold, (trn_idx, vld_idx) in enumerate(skf.split(train, train[\"target\"])):\n    train.loc[vld_idx, \"folds\"] = int(fold)\ntrain[\"folds\"] = train[\"folds\"].astype(int)\n\ndel skf\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:15.098535Z","iopub.execute_input":"2022-06-10T10:54:15.101625Z","iopub.status.idle":"2022-06-10T10:54:15.360653Z","shell.execute_reply.started":"2022-06-10T10:54:15.101563Z","shell.execute_reply":"2022-06-10T10:54:15.359442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# W&B","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_key\")\n\nimport wandb\nwandb.login(key=wandb_api)\n\ndef class2dict(f):\n    return dict((name, getattr(f, name)) for name in dir(f) if not name.startswith('__'))\n\nrun = wandb.init(project=\"Amex\", \n                 name=CFG.model_name,\n                 config=class2dict(CFG),\n                 group=CFG.model_name,\n                 job_type=\"train\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:15.362582Z","iopub.execute_input":"2022-06-10T10:54:15.363174Z","iopub.status.idle":"2022-06-10T10:54:22.571421Z","shell.execute_reply.started":"2022-06-10T10:54:15.363130Z","shell.execute_reply":"2022-06-10T10:54:22.570238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class AmexDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        self.num_data = df[NUM_FEATURES].values\n        self.cat_data = df[CAT_FEATURES].values\n        self.targets = df[\"target\"].values \n    \n    def __len__(self):\n        \n        return len(self.num_data)\n    \n    def __getitem__(self, idx):\n            \n            dct = {\n            'num_data' : torch.tensor(self.num_data[idx], dtype=torch.float),\n            'cat_data' : torch.tensor(self.cat_data[idx], dtype=torch.int),\n            'y' : torch.tensor(self.targets[idx], dtype=torch.float)}\n            \n            return dct","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:22.573658Z","iopub.execute_input":"2022-06-10T10:54:22.574351Z","iopub.status.idle":"2022-06-10T10:54:22.586519Z","shell.execute_reply.started":"2022-06-10T10:54:22.574303Z","shell.execute_reply":"2022-06-10T10:54:22.585226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"# implementation of a weighted average between two tensors (model outputs)\n# https://stackoverflow.com/questions/62877879/implementing-late-fusion-in-keras\nclass WeightedAverage(nn.Module):\n    \n    def __init__(self, n_output):\n        super().__init__()\n        self.n_output = n_output\n        w = torch.Tensor(1, self.n_output)\n        nn.init.xavier_normal_(w)\n        w = w.unsqueeze(0)\n        self.w = nn.Parameter(w)\n        \n    def forward(self, inputs):\n        \n        # inputs is a list of tensor of shape [(n_batch, n_feat), ..., (n_batch, n_feat)]\n        # expand last dim of each input passed [(n_batch, n_feat, 1), ..., (n_batch, n_feat, 1)]\n        inputs = [torch.unsqueeze(i, -1) for i in inputs] \n        inputs = torch.cat(inputs, dim=-1)  # (n_batch, n_feat, n_inputs)\n        weights = nn.Softmax(dim=-1)(self.w) # (1,1,n_inputs)\n        # weights sum up to one on last dim\n        \n        return torch.sum(weights * inputs, dim=-1)\n    \nclass NNet(nn.Module):\n    \n    def __init__(self, cfg):\n        super(NNet, self).__init__()\n        \n        self.emb = nn.ModuleList([nn.Embedding(x,y) for x,y in cfg.cat_embs])\n        self.w_avg = WeightedAverage(n_output=2)\n        self.mish_activation = nn.Mish()\n        \n        no_of_embs = sum([y for x, y in cfg.cat_embs])\n        \n        self.no_of_embs = no_of_embs\n        self.no_of_num = len(NUM_FEATURES)\n        self.dense_cat = nn.Linear(self.no_of_embs, 100)\n        \n        self.batch_norm1 = nn.BatchNorm1d(self.no_of_num)\n        self.dropout1 = nn.Dropout(0.2)\n        self.dense1 = nn.Linear(self.no_of_num, 100)\n        \n        self.batch_norm2 = nn.BatchNorm1d(100)\n        self.dense2 = nn.Linear(100, 32)\n        \n        self.batch_norm3 = nn.BatchNorm1d(32)\n        self.dense3 = nn.Linear(32, 16)\n        \n        self.batch_norm4 = nn.BatchNorm1d(16)\n        self.dense4 = nn.Linear(16, 1)\n        \n    def forward(self, cat, num):\n        ## cat data part\n        x_cat = [emb_layer(cat[:,i]) for i,emb_layer in enumerate(self.emb)]\n        x_cat = torch.cat(x_cat,1)\n        x_cat = self.dropout1(x_cat)\n        x_cat = self.dense_cat(x_cat)\n        \n        ##num data part\n        x_num = self.batch_norm1(num)\n        x_num = self.dropout1(x_num)\n        x_num = self.mish_activation(self.dense1(x_num))\n        \n        ##concat\n        x = [x_cat, x_num]\n        x = self.w_avg(x)\n        \n        ##rest of NN\n        x = self.batch_norm2(x)\n        x = self.mish_activation(self.dense2(x))\n        \n        x = self.batch_norm3(x)\n        x = self.mish_activation(self.dense3(x))\n        \n        \n        x = self.batch_norm4(x)\n        x = self.dense4(x)\n        \n        return x\n    \nclass SmoothBCEwLogits(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean', smoothing=0.0):\n        super().__init__(weight=weight, reduction=reduction)\n        self.smoothing = smoothing\n        self.weight = weight\n        self.reduction = reduction\n\n    @staticmethod\n    def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):\n        assert 0 <= smoothing < 1\n\n        with torch.no_grad():\n            targets = targets * (1.0 - smoothing) + 0.5 * smoothing\n            \n        return targets\n\n    def forward(self, inputs, targets):\n        targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),\n            self.smoothing)\n        loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:22.589035Z","iopub.execute_input":"2022-06-10T10:54:22.589846Z","iopub.status.idle":"2022-06-10T10:54:22.669426Z","shell.execute_reply.started":"2022-06-10T10:54:22.589801Z","shell.execute_reply":"2022-06-10T10:54:22.658948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    if CFG.apex:\n        scaler = GradScaler()\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, dct in enumerate(train_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        num = dct['num_data'].to(device)\n        cat = dct['cat_data'].to(device)\n        labels = dct['y'].to(device)\n        batch_size = labels.size(0)\n        if CFG.apex:\n            with autocast():\n                y_preds = model(cat, num)\n                loss = criterion(y_preds.squeeze(1), labels)\n        else:\n            y_preds = model(cat, num)\n            loss = criterion(y_preds.squeeze(1), labels)\n        # record loss\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        if CFG.apex:\n            scaler.scale(loss).backward()\n        else:\n            loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            if CFG.apex:\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f} '\n                  'LR: {lr:.6f}  '\n                  .format(epoch+1, step, len(train_loader), \n                          remain=timeSince(start, float(step+1)/len(train_loader)),\n                          loss=losses,\n                          grad_norm=grad_norm,\n                          lr=scheduler.get_last_lr()[0]))\n        wandb.log({f\"[fold{fold}] loss\": losses.val,\n                   f\"[fold{fold}] lr\": scheduler.get_last_lr()[0]})\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, dct in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        num = dct['num_data'].to(device)\n        cat = dct['cat_data'].to(device)\n        labels = dct['y'].to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(cat, num)\n        preds.append(y_preds.sigmoid().to('cpu').numpy())\n        loss = criterion(y_preds.squeeze(1), labels)\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(step, len(valid_loader),\n                          loss=losses,\n                          remain=timeSince(start, float(step+1)/len(valid_loader))))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:22.672563Z","iopub.execute_input":"2022-06-10T10:54:22.673783Z","iopub.status.idle":"2022-06-10T10:54:22.741099Z","shell.execute_reply.started":"2022-06-10T10:54:22.673692Z","shell.execute_reply":"2022-06-10T10:54:22.739748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train loop","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(folds, fold):\n    \n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['folds'] != fold].index\n    val_idx = folds[folds['folds'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    valid_labels = valid_folds[\"target\"].values\n\n    train_dataset = AmexDataset(train_folds)\n    valid_dataset = AmexDataset(valid_folds)\n\n    train_loader = DataLoader(train_dataset,\n                              batch_size=CFG.batch_size, \n                              shuffle=True, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=False, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    # ====================================================\n    # scheduler \n    # ====================================================\n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, **CFG.reduce_params)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, **CFG.cosanneal_params)\n        elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, **CFG.cosanneal_res_params)\n        elif CFG.scheduler=='OneCycleLR':\n            scheduler = OneCycleLR(optimizer, **CFG.onecycle_params)\n        return scheduler\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    model = NNet(CFG)\n    model.to(device)\n\n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\n    scheduler = get_scheduler(optimizer)\n\n    # ====================================================\n    # loop\n    # ====================================================\n    valid_loss = nn.BCEWithLogitsLoss()\n    train_loss = SmoothBCEwLogits(smoothing=0.0001)\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG.epochs):\n        \n        start_time = time.time()\n        \n        \n          \n        avg_loss = train_fn(fold, train_loader, model, train_loss, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, valid_loss, device)\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        # scoring\n        score = get_score(valid_labels, preds)\n\n        elapsed = time.time() - start_time\n        \n            \n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}')\n        wandb.log({f\"[fold{fold}] epoch\": epoch+1, \n                   f\"[fold{fold}] avg_train_loss\": avg_loss, \n                   f\"[fold{fold}] avg_val_loss\": avg_val_loss,\n                   f\"[fold{fold}] score\": score})\n\n        if score >= best_score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds_score': preds},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score.pth')\n            \n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Loss: {best_loss:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds_loss': preds},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_loss.pth')\n        \n        \n    valid_folds[\"preds_score\"] = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_score.pth', \n                                      map_location=torch.device('cpu'))['preds_score']\n    valid_folds[\"preds_loss\"] = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best_loss.pth', \n                                      map_location=torch.device('cpu'))['preds_loss']\n   \n\n    return valid_folds","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:22.747874Z","iopub.execute_input":"2022-06-10T10:54:22.752264Z","iopub.status.idle":"2022-06-10T10:54:22.801522Z","shell.execute_reply.started":"2022-06-10T10:54:22.752204Z","shell.execute_reply":"2022-06-10T10:54:22.799302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef main():\n\n    \"\"\"\n    Prepare: 1.train \n    \"\"\"\n\n    def get_result(result_df):\n        preds_score = result_df['preds_score'].values\n        preds_loss = result_df['preds_loss'].values\n        labels = result_df[\"target\"].values\n        score = get_score(labels, preds_score)\n        score_loss = get_score(labels, preds_loss)\n        LOGGER.info(f'Score with best score weights: {score:<.4f}')\n        LOGGER.info(f'Score with best loss weights: {score_loss:<.4f}')\n    \n    if CFG.train:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.nfolds):\n            if fold in CFG.trn_folds:\n                _oof_df = train_loop(train, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)\n        \n    wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:22.808370Z","iopub.execute_input":"2022-06-10T10:54:22.811944Z","iopub.status.idle":"2022-06-10T10:54:22.830878Z","shell.execute_reply.started":"2022-06-10T10:54:22.811900Z","shell.execute_reply":"2022-06-10T10:54:22.829039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    main()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T10:54:22.840372Z","iopub.execute_input":"2022-06-10T10:54:22.841657Z"},"trusted":true},"execution_count":null,"outputs":[]}]}