{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# If the training notebook is useful please upvote !!!\n\n### This notebook is based on the notebooks made by zzy.\n\nPlease upvote the LB:0.667 original notebooks:\n\nhttps://www.kaggle.com/code/zzy990106/nfl-2-5d-cnn-baseline-inference\n\nalso can use to LB:0.671, 2.5D CNN Baseline（More TTA trick）\n\nhttps://www.kaggle.com/code/royalacecat/lb-0-671-2-5d-cnn-baseline-more-tta-trick","metadata":{"execution":{"iopub.status.busy":"2023-01-27T15:34:35.699098Z","iopub.execute_input":"2023-01-27T15:34:35.699592Z","iopub.status.idle":"2023-01-27T15:34:35.708483Z","shell.execute_reply.started":"2023-01-27T15:34:35.699558Z","shell.execute_reply":"2023-01-27T15:34:35.706487Z"}}},{"cell_type":"code","source":"import os\nimport sys\nimport glob\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport gc\nimport cv2\nfrom tqdm import tqdm\nimport time\nfrom functools import lru_cache\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom timm.scheduler import CosineLRScheduler\nsys.path.append('../input/timm-0-6-9/pytorch-image-models-master')","metadata":{"execution":{"iopub.status.busy":"2023-01-27T15:33:19.648679Z","iopub.execute_input":"2023-01-27T15:33:19.649616Z","iopub.status.idle":"2023-01-27T15:33:25.879742Z","shell.execute_reply.started":"2023-01-27T15:33:19.649495Z","shell.execute_reply":"2023-01-27T15:33:25.878343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = {\n    'seed': 42,\n    'model': 'resnet50',\n    'img_size': 256,\n    'epochs': 10,\n    'train_bs': 8, \n    'valid_bs': 4,\n    'lr': 1e-3, \n    'weight_decay': 1e-6,\n    'num_workers': 8,\n    'max_grad_norm' : 1000,\n    'epochs_warmup' : 1.0\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(CFG['seed'])\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_contact_id(df):\n    \"\"\"\n    Splits out contact_id into seperate columns.\n    \"\"\"\n    df[\"game_play\"] = df[\"contact_id\"].str[:12]\n    df[\"step\"] = df[\"contact_id\"].str.split(\"_\").str[-3].astype(\"int\")\n    df[\"nfl_player_id_1\"] = df[\"contact_id\"].str.split(\"_\").str[-2]\n    df[\"nfl_player_id_2\"] = df[\"contact_id\"].str.split(\"_\").str[-1]\n    return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = expand_contact_id(pd.read_csv(\"../input/nfl-player-contact-detection/train_labels.csv\"))\ntrain_tracking = pd.read_csv(\"../input/nfl-player-contact-detection/train_player_tracking.csv\")\ntrain_helmets = pd.read_csv(\"../input/nfl-player-contact-detection/train_baseline_helmets.csv\")\ntrain_video_metadata = pd.read_csv(\"../input/nfl-player-contact-detection/train_video_metadata.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p ../train/frames\n\nfor video in tqdm(train_helmets.video.unique()):\n    if 'Endzone2' not in video:\n        !ffmpeg -i ../input/nfl-player-contact-detection/train/{video} -q:v 2 -f image2 ../train/frames/{video}_%04d.jpg -hide_banner -loglevel error","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_features(df, tr_tracking, merge_col=\"step\", use_cols=[\"x_position\", \"y_position\"]):\n    output_cols = []\n    df_combo = (\n        df.astype({\"nfl_player_id_1\": \"str\"})\n        .merge(\n            tr_tracking.astype({\"nfl_player_id\": \"str\"})[\n                [\"game_play\", merge_col, \"nfl_player_id\",] + use_cols\n            ],\n            left_on=[\"game_play\", merge_col, \"nfl_player_id_1\"],\n            right_on=[\"game_play\", merge_col, \"nfl_player_id\"],\n            how=\"left\",\n        )\n        .rename(columns={c: c+\"_1\" for c in use_cols})\n        .drop(\"nfl_player_id\", axis=1)\n        .merge(\n            tr_tracking.astype({\"nfl_player_id\": \"str\"})[\n                [\"game_play\", merge_col, \"nfl_player_id\"] + use_cols\n            ],\n            left_on=[\"game_play\", merge_col, \"nfl_player_id_2\"],\n            right_on=[\"game_play\", merge_col, \"nfl_player_id\"],\n            how=\"left\",\n        )\n        .drop(\"nfl_player_id\", axis=1)\n        .rename(columns={c: c+\"_2\" for c in use_cols})\n        .sort_values([\"game_play\", merge_col, \"nfl_player_id_1\", \"nfl_player_id_2\"])\n        .reset_index(drop=True)\n    )\n    output_cols += [c+\"_1\" for c in use_cols]\n    output_cols += [c+\"_2\" for c in use_cols]\n    \n    if (\"x_position\" in use_cols) & (\"y_position\" in use_cols):\n        index = df_combo['x_position_2'].notnull()\n        \n        distance_arr = np.full(len(index), np.nan)\n        tmp_distance_arr = np.sqrt(\n            np.square(df_combo.loc[index, \"x_position_1\"] - df_combo.loc[index, \"x_position_2\"])\n            + np.square(df_combo.loc[index, \"y_position_1\"]- df_combo.loc[index, \"y_position_2\"])\n        )\n        \n        distance_arr[index] = tmp_distance_arr\n        df_combo['distance'] = distance_arr\n        output_cols += [\"distance\"]\n        \n    df_combo['G_flug'] = (df_combo['nfl_player_id_2']==\"G\")\n    output_cols += [\"G_flug\"]\n    return df_combo, output_cols\n\n\nuse_cols = [\n    'x_position', 'y_position', 'speed', 'distance',\n    'direction', 'orientation', 'acceleration', 'sa'\n]\n\ntrain, feature_cols = create_features(labels, train_tracking, use_cols=use_cols)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_filtered = train.query('not distance>2').reset_index(drop=True)\ntrain_filtered['frame'] = (train_filtered['step']/10*59.94+5*59.94).astype('int')+1\ntrain_filtered.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"del train, labels, train_tracking\ngc.collect()","metadata":{}},{"cell_type":"code","source":"train_aug = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    A.RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n    A.Normalize(mean=[0.], std=[1.]),\n    ToTensorV2()\n])\n\nvalid_aug = A.Compose([\n    A.Normalize(mean=[0.], std=[1.]),\n    ToTensorV2()\n])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"video2helmets = {}\ntrain_helmets_new = train_helmets.set_index('video')\nfor video in tqdm(train_helmets.video.unique()):\n    video2helmets[video] = train_helmets_new.loc[video].reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"del train_helmets, train_helmets_new\ngc.collect()","metadata":{}},{"cell_type":"code","source":"video2frames = {}\n\nfor game_play in tqdm(train_video_metadata.game_play.unique()):\n    for view in ['Endzone', 'Sideline']:\n        video = game_play + f'_{view}.mp4'\n        video2frames[video] = max(list(map(lambda x:int(x.split('_')[-1].split('.')[0]), \\\n                                           glob.glob(f'../train/frames/{video}*'))))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, df, aug=train_aug, mode='train'):\n        self.df = df\n        self.frame = df.frame.values\n        self.feature = df[feature_cols].fillna(-1).values\n        self.players = df[['nfl_player_id_1','nfl_player_id_2']].values\n        self.game_play = df.game_play.values\n        self.aug = aug\n        self.mode = mode\n        \n    def __len__(self):\n        return len(self.df)\n    \n    # @lru_cache(1024)\n    # def read_img(self, path):\n    #     return cv2.imread(path, 0)\n   \n    def __getitem__(self, idx):   \n        window = 24\n        frame = self.frame[idx]\n        \n        if self.mode == 'train':\n            frame = frame + random.randint(-6, 6)\n\n        players = []\n        for p in self.players[idx]:\n            if p == 'G':\n                players.append(p)\n            else:\n                players.append(int(p))\n        \n        imgs = []\n        for view in ['Endzone', 'Sideline']:\n            video = self.game_play[idx] + f'_{view}.mp4'\n\n            tmp = video2helmets[video]\n#             tmp = tmp.query('@frame-@window<=frame<=@frame+@window')\n            tmp[tmp['frame'].between(frame-window, frame+window)]\n            tmp = tmp[tmp.nfl_player_id.isin(players)]#.sort_values(['nfl_player_id', 'frame'])\n            tmp_frames = tmp.frame.values\n            tmp = tmp.groupby('frame')[['left','width','top','height']].mean()\n#0.002s\n\n            bboxes = []\n            for f in range(frame-window, frame+window+1, 1):\n                if f in tmp_frames:\n                    x, w, y, h = tmp.loc[f][['left','width','top','height']]\n                    bboxes.append([x, w, y, h])\n                else:\n                    bboxes.append([np.nan, np.nan, np.nan, np.nan])\n            bboxes = pd.DataFrame(bboxes).interpolate(limit_direction='both').values\n            bboxes = bboxes[::4]\n\n            if bboxes.sum() > 0:\n                flag = 1\n            else:\n                flag = 0\n#0.03s\n                    \n            for i, f in enumerate(range(frame-window, frame+window+1, 4)):\n                img_new = np.zeros((256, 256), dtype=np.float32)\n\n                if flag == 1 and f <= video2frames[video]:\n                    img = cv2.imread(f'../train/frames/{video}_{f:04d}.jpg', 0)\n\n                    x, w, y, h = bboxes[i]\n\n                    img = img[int(y+h/2)-128:int(y+h/2)+128,int(x+w/2)-128:int(x+w/2)+128].copy()\n                    img_new[:img.shape[0], :img.shape[1]] = img\n                    \n                imgs.append(img_new)\n#0.06s\n                \n        feature = np.float32(self.feature[idx])\n\n        img = np.array(imgs).transpose(1, 2, 0)    \n        img = self.aug(image=img)[\"image\"]\n        label = np.float32(self.df.contact.values[idx])\n\n        return img, feature, label","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.backbone = timm.create_model(CFG['model'], pretrained=True, num_classes=500, in_chans=13)\n        self.mlp = nn.Sequential(\n            nn.Linear(18, 64),\n            nn.LayerNorm(64),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n        )\n        self.fc = nn.Linear(64+500*2, 1)\n\n    def forward(self, img, feature):\n        b, c, h, w = img.shape\n        img = img.reshape(b*2, c//2, h, w)\n        img = self.backbone(img).reshape(b, -1)\n        feature = self.mlp(feature)\n        y = self.fc(torch.cat([img, feature], dim=1))\n        return y","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model()\nmodel.to(device)\nmodel.train()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\ncriterion = nn.BCEWithLogitsLoss()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, loader_val, *, compute_score=True, pbar=None):\n    \"\"\"\n    Predict and compute loss and score\n    \"\"\"\n    tb = time.time()\n    in_training = model.training\n    model.eval()\n\n    loss_sum = 0.0\n    n_sum = 0\n    y_all = []\n    y_pred_all = []\n\n    if pbar is not None:\n        pbar = tqdm(desc='Predict', nrows=78, total=pbar)\n        \n    total= len(loader_val)\n\n    for ibatch,(img, feature, label) in tqdm(enumerate(loader_val),total = total):\n        # img, feature, label = [x.to(device) for x in batch]\n        img = img.to(device)\n        feature = feature.to(device)\n        n = label.size(0)\n        label = label.to(device)\n\n        with torch.no_grad():\n            y_pred = model(img, feature)\n        loss = criterion(y_pred.view(-1), label)\n\n        n_sum += n\n        loss_sum += n * loss.item()\n        \n        if pbar is not None:\n            pbar.update(len(img))\n        \n        del loss, img, label\n        gc.collect()\n\n    loss_val = loss_sum / n_sum\n\n\n    ret = {'loss': loss_val,\n           'time': time.time() - tb}\n    \n    model.train(in_training) \n    gc.collect()\n    return ret","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set,valid_set = train_test_split(train_filtered,test_size=0.05, random_state=42,stratify = train_filtered['contact'])\ntrain_set = MyDataset(train_set, train_aug, 'train')\ntrain_loader = DataLoader(train_set, batch_size=CFG['train_bs'], shuffle=True, num_workers=12, pin_memory=True,drop_last=True)\nvalid_set = MyDataset(valid_set, valid_aug, 'test')\nvalid_loader = DataLoader(valid_set, batch_size=CFG['valid_bs'], shuffle=False, num_workers=12, pin_memory=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.AdamW(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\nnbatch = len(train_loader)\nwarmup = CFG['epochs_warmup'] * nbatch\nnsteps = CFG['epochs'] * nbatch ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scheduler = CosineLRScheduler(optimizer,warmup_t=warmup, warmup_lr_init=0.0, warmup_prefix=True,t_initial=(nsteps - warmup), lr_min=1e-6)                ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_val = 0.0\ntb = time.time()\nbest_cv = 0\nbest_loss = 1e10\nfor iepoch in range(CFG['epochs']):\n    print('Epoch:', iepoch+1)\n    loss_sum = 0.0\n    n_sum = 0\n    total = len(train_loader)\n\n    # Train\n    for ibatch,(img, feature, label) in tqdm(enumerate(train_loader),total = total):\n        img = img.to(device)\n        feature = feature.to(device)\n        n = label.size(0)\n        label = label.to(device)\n        \n\n        optimizer.zero_grad()\n        y_pred = model(img, feature).squeeze(-1)\n        loss = criterion(y_pred, label)\n        loss_train = loss.item()\n        loss_sum += n * loss_train\n        n_sum += n\n\n        loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),CFG['max_grad_norm'])\n\n        optimizer.step()\n        scheduler.step(iepoch * nbatch + ibatch + 1)\n        \n    val = evaluate(model, valid_loader)\n    time_val += val['time']\n    loss_train = loss_sum / n_sum\n    dt = (time.time() - tb) / 60\n    print('Epoch: %d Train Loss: %.4f Test Loss: %.4f Time: %.2f min' %\n          (iepoch + 1, loss_train, val['loss'],dt))\n    if val['loss'] < best_loss:\n        best_loss = val['loss']\n        # Save model\n        ofilename = 'best_model.pytorch'\n        torch.save(model.state_dict(), ofilename)\n        print(ofilename, 'written')\n    del val\n    gc.collect()\n\ndt = time.time() - tb\nprint(' %.2f min total, %.2f min val' % (dt / 60, time_val / 60))\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}