{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Using The Zoo solution https://www.kaggle.com/c/nfl-big-data-bowl-2020/discussion/119400 of NFL big data bowl 2020 here is an attempt to predict punt return yards when the ball reach to receiver. \n\nMost of the codes are forked from Takuoko's kernel: https://www.kaggle.com/takuok/1st-place-reproduction-10feats-dev","metadata":{}},{"cell_type":"markdown","source":"## Feature","metadata":{}},{"cell_type":"code","source":"def reorient(df, flip_left=False):\n    \n    \n    df = df.drop_duplicates(subset=['GameKey',\"PlayID\",'GSISID'])\n\n    return_team_role = ['PDL1','PDL2','PDL3','PDL4','PDL5','PDL6','PDM','PDR1','PDR2','PDR3','PDR4','PDR5', 'PDR6','PFB','PLL','PLL1','PLL2','PLL3','PLM','PLR','PLR1','PLR2','PLR3','VL','VLi','VLo','VR','VRi','VRo','PR']\n\n    df['IsReturner'] = df.Role == 'PR'\n    df['IsReturnTeam'] = df.Role.isin(return_team_role)\n\n    punt_team_data = df[df.IsReturnTeam == False].groupby(['GameKey','PlayID'])['x'].mean().reset_index()\n    punt_team_data.columns = ['GameKey','PlayID','punt_team_x']\n    returner_team_data = df[df.Role == 'PR'].groupby(['GameKey','PlayID'])['x'].mean().reset_index()\n    returner_team_data.columns = ['GameKey','PlayID','returner_x']\n    play_data = pd.merge(punt_team_data,returner_team_data,how='inner')\n    play_data['ToLeft'] = play_data.returner_x < play_data.punt_team_x\n    df = pd.merge(df,play_data[['GameKey','PlayID','ToLeft']],on=['GameKey','PlayID'])\n    \n    df.loc[df.ToLeft, 'x'] = 120 - df.loc[df.ToLeft, 'x']\n    df.loc[df.ToLeft, 'y'] = 160 / 3 - df.loc[df.ToLeft, 'y']\n    df.loc[df.ToLeft, 'o'] = np.mod(180 + df.loc[df.ToLeft, 'o'], 360)\n    df['dir'] = 90 - df.dir\n    df.loc[df.ToLeft, 'dir'] = np.mod(180 + df.loc[df.ToLeft, 'dir'], 360)\n    df.loc[df.IsReturnTeam, 'dir'] = df.loc[df.IsReturnTeam, 'dir'].fillna(0).values\n    df.loc[~df.IsReturnTeam, 'dir'] = df.loc[~df.IsReturnTeam, 'dir'].fillna(180).values\n    \n    if flip_left:\n        tmp = df[df['IsReturner']].copy()\n        # df['left'] = df.Y < 160/6\n        tmp['left'] = tmp.dir < 0\n        df = df.merge(tmp[['GameKey','PlayID', 'left']], how='left', on=['GameKey','PlayID'])\n        df['y'] = df.y\n        df.loc[df[\"left\"], 'y'] = 160 / 3 - df.loc[df[\"left\"], 'y']\n        df['dir'] = df.dir\n        df.loc[df[\"left\"], 'dir'] = np.mod(- df.loc[df[\"left\"], 'dir'], 360)\n        df.drop('left', axis=1, inplace=True)\n\n    df[\"S\"] = df[\"dis\"] * 10\n    df['x_dir'] = np.cos((np.pi / 180) * df.dir)\n    df['y_dir'] = np.sin((np.pi / 180) * df.dir)\n    df['x_S'] = df.x_dir * df.S\n    df['y_S'] = df.y_dir * df.S\n    \n    return df\n\n\ndef merge_returnerfeats(df):\n    returner_feats = df[df['Role'] == 'PR'].drop_duplicates()\n    returner_feats = returner_feats[['GameKey',\"PlayID\", \"x\", \"y\", \"x_S\", \"y_S\"]]\n    returner_feats = returner_feats.rename(\n        columns={\"x\": \"Returner_x\", \"y\": \"Returner_y\", \"x_S\": \"Returner_x_S\", \"y_S\": \"Returner_y_S\"})\n    df = df.merge(returner_feats, how=\"left\", on=['GameKey',\"PlayID\"]).drop_duplicates(subset=['GameKey',\"PlayID\",'GSISID'])\n\n    return df\n\ndef scaling(feats, sctype=\"standard\"):\n    v1 = []\n    v2 = []\n    for i in range(feats.shape[1]):\n        feats_ = feats[:, i, :]\n        if sctype == \"standard\":\n            mean_ = np.mean(feats_)\n            std_ = np.std(feats_)\n            feats[:, i, :] -= mean_\n            feats[:, i, :] /= std_\n            v1.append(mean_)\n            v2.append(std_)\n        elif sctype == \"minmax\":\n            max_ = np.max(feats_)\n            min_ = np.min(feats_)\n            feats[:, i, :] = (feats_ - min_) / (max_ - min_)\n            v1.append(max_)\n            v2.append(min_)\n\n    return feats, v1, v2","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{}},{"cell_type":"code","source":"def create_features(df):\n    xysdir_o = df[(df.IsReturnTeam == True) & (df.IsReturner == False)][['x','y','x_S','y_S']].values\n    xysdir_rush = df[df.IsReturner == True][['x','y','x_S','y_S']].values\n    xysdir_d = df[df.IsReturnTeam == False][['x','y','x_S','y_S']].values\n    \n    off_x = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['x'].apply(np.array))\n    def_x = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['x'].apply(np.array))\n    off_y = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['y'].apply(np.array))\n    def_y = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['y'].apply(np.array))\n    off_sx = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['x_S'].apply(np.array))\n    def_sx = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['x_S'].apply(np.array))\n    off_sy = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['y_S'].apply(np.array))\n    def_sy = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['y_S'].apply(np.array))\n    \n    player_vector = []\n    player_vector_aug = []\n    for play in range(len(off_x)):\n        player_feat, player_feat_aug = player_feature(off_x[play],def_x[play],off_y[play],def_y[play],off_sx[play],def_sx[play],\n                                     off_sy[play],def_sy[play],xysdir_rush[play])\n        player_vector.append(player_feat)\n        player_vector_aug.append(player_feat_aug)\n    \n    return np.array(player_vector), np.array(player_vector_aug)\n\n    \ndef player_feature(off_x,def_x,off_y,def_y,off_sx,def_sx,off_sy,def_sy,xysdir_rush):\n    if(len(off_x)<10):\n        off_x = np.pad(off_x,(10-len(off_x),0), 'mean' )\n        off_y = np.pad(off_y,(10-len(off_y),0), 'mean' )\n        off_sx = np.pad(off_sx,(10-len(off_sx),0), 'mean' )\n        off_sy = np.pad(off_sy,(10-len(off_sy),0), 'mean' )\n    if(len(def_x)<11):\n        def_x = np.pad(def_x,(11-len(def_x),0), 'mean' )\n        def_y = np.pad(def_y,(11-len(def_y),0), 'mean' )\n        def_sx = np.pad(def_sx,(11-len(def_sx),0), 'mean' )\n        def_sy = np.pad(def_sy,(11-len(def_sy),0), 'mean' )\n\n    dist_def_off_x = def_x.reshape(-1,1)-off_x.reshape(1,-1)\n    dist_def_off_sx = def_sx.reshape(-1,1)-off_sx.reshape(1,-1)\n    dist_def_off_y = def_y.reshape(-1,1)-off_y.reshape(1,-1)\n    dist_def_off_sy = def_sy.reshape(-1,1)-off_sy.reshape(1,-1)\n    dist_def_rush_x = def_x.reshape(-1,1)-np.repeat(xysdir_rush[0],10).reshape(1,-1)\n    dist_def_rush_y = def_y.reshape(-1,1)-np.repeat(xysdir_rush[1],10).reshape(1,-1)\n    dist_def_rush_sx = def_sx.reshape(-1,1)-np.repeat(xysdir_rush[2],10).reshape(1,-1)\n    dist_def_rush_sy = def_sy.reshape(-1,1)-np.repeat(xysdir_rush[3],10).reshape(1,-1)\n    def_sx = np.repeat(def_sx,10).reshape(11,-1)\n    def_sy = np.repeat(def_sy,10).reshape(11,-1)\n    feats = [dist_def_off_x, dist_def_off_sx, dist_def_off_y, dist_def_off_sy, dist_def_rush_x, dist_def_rush_y,\n            dist_def_rush_sx, dist_def_rush_sy, def_sx, def_sy]\n    feats_aug = [dist_def_off_x, dist_def_off_sx, -1*dist_def_off_y, -1*dist_def_off_sy, dist_def_rush_x, \n                 -1*dist_def_rush_y, dist_def_rush_sx, -1*dist_def_rush_sy, def_sx, -1*def_sy]\n    \n    return np.stack(feats), np.stack(feats_aug)\n\n# def create_features_nearest(df):\n#     xysdir_o = df[(df.IsReturnTeam == True) & (df.IsReturner == False)][['x','y','x_S','y_S']].values\n#     xysdir_rush = df[df.IsReturner == True][['x','y','x_S','y_S']].values\n#     xysdir_d = df[df.IsReturnTeam == False][['x','y','x_S','y_S']].values\n    \n#     off_x = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['x'].apply(np.array))\n#     def_x = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['x'].apply(np.array))\n#     off_y = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['y'].apply(np.array))\n#     def_y = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['y'].apply(np.array))\n#     off_sx = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['x_S'].apply(np.array))\n#     def_sx = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['x_S'].apply(np.array))\n#     off_sy = np.array(df[(df.IsReturnTeam == True) & (df.IsReturner == False)].groupby(['GameKey','PlayID'])['y_S'].apply(np.array))\n#     def_sy = np.array(df[(df.IsReturnTeam == False) ].groupby(['GameKey','PlayID'])['y_S'].apply(np.array))\n    \n#     player_vector = []\n#     player_vector_aug = []\n#     for play in range(len(off_x)):\n#         player_feat, player_feat_aug = player_feature_nearest(off_x[play],def_x[play],off_y[play],def_y[play],off_sx[play],def_sx[play],\n#                                      off_sy[play],def_sy[play],xysdir_rush[play])\n#         player_vector.append(player_feat)\n#         player_vector_aug.append(player_feat_aug)\n    \n#     return np.array(player_vector), np.array(player_vector_aug)\n\n    \n# def player_feature_nearest(off_x,def_x,off_y,def_y,off_sx,def_sx,off_sy,def_sy,xysdir_rush):\n#     if(len(off_x)<10):\n#         off_x = np.pad(off_x,(10-len(off_x),0), 'mean' )\n#         off_y = np.pad(off_y,(10-len(off_y),0), 'mean' )\n#         off_sx = np.pad(off_sx,(10-len(off_sx),0), 'mean' )\n#         off_sy = np.pad(off_sy,(10-len(off_sy),0), 'mean' )\n#     if(len(def_x)<11):\n#         def_x = np.pad(def_x,(11-len(def_x),0), 'mean' )\n#         def_y = np.pad(def_y,(11-len(def_y),0), 'mean' )\n#         def_sx = np.pad(def_sx,(11-len(def_sx),0), 'mean' )\n#         def_sy = np.pad(def_sy,(11-len(def_sy),0), 'mean' )\n\n#     dist_def_rush_x = (def_x.reshape(-1,1)-xysdir_rush[0]).reshape(1,-1)\n#     dist_def_rush_y = (def_y.reshape(-1,1)-xysdir_rush[1]).reshape(1,-1)\n#     dist_def_rush_sx = (def_sx.reshape(-1,1)-xysdir_rush[2]).reshape(1,-1)\n#     dist_def_rush_sy = (def_sy.reshape(-1,1)-xysdir_rush[3]).reshape(1,-1)\n\n#     feats = [ dist_def_rush_x, dist_def_rush_y, dist_def_rush_sx, dist_def_rush_sy]\n#     feats_aug = [ dist_def_rush_x, -1*dist_def_rush_y, dist_def_rush_sx, -1*dist_def_rush_sy]\n\n    \n#     return np.stack(feats), np.stack(feats_aug)\n\n\n# def get_def_speed(df):\n#     df_cp = df[~df.IsOnOffense].copy()\n#     speed = 10*df_cp[\"dis\"].T.values\n#     speed = speed.reshape(-1, 1, 1, 11) \n#     speed = np.repeat(speed, 10, axis=2)\n\n#     return speed\n\n\n# def get_dist(df, col1, col2, type=\"defence\"):\n#     if type == \"defence\":\n#         df_cp = df[~df.IsOnOffense].copy()\n#     elif type == \"offence\":\n#         df_cp = df[df.IsOnOffense].copy()\n#     dist = np.linalg.norm(df_cp[col1].values - df_cp[col2].values, axis=1)\n#     dist = dist.T\n#     dist = dist.reshape(-1, 1, 1, 11)\n#     dist = np.repeat(dist, 10, axis=2)\n\n#     return dist\n\n\n\n# def dist_def_off(df, n_train, cols):\n#     off_x = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['x'].apply(np.array))\n#     def_x = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['x'].apply(np.array))\n#     off_y = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['y'].apply(np.array))\n#     def_y = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['y'].apply(np.array))\n#     off_xs = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['x_S'].apply(np.array))\n#     def_xs = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['x_S'].apply(np.array))\n#     off_ys = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['y_S'].apply(np.array))\n#     def_ys = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['y_S'].apply(np.array))\n#     feats = []\n#     for play in range(len(off_x)):\n#         dist_x = off_x[play].reshape(-1, 1) - def_x[play].reshape(1, -1)\n#         dist_y = off_y[play].reshape(-1, 1) - def_y[play].reshape(1, -1)\n#         dist = np.concatenate([dist_x[:, :, np.newaxis], dist_y[:, :, np.newaxis]], axis=2)\n#         dist_xy = np.linalg.norm(dist.astype(np.float64), axis=2)\n#         dist_xs = off_xs[play].reshape(-1, 1) - def_xs[play].reshape(1, -1)\n#         dist_ys = off_ys[play].reshape(-1, 1) - def_ys[play].reshape(1, -1)\n#         dist = np.concatenate([dist_xs[:, :, np.newaxis], dist_ys[:, :, np.newaxis]], axis=2)\n#         dist_xys = np.linalg.norm(dist.astype(np.float64), axis=2)\n#         feats.append(np.concatenate([dist_xy[np.newaxis, :], dist_xys[np.newaxis, :]], axis=0))\n#     return np.array(feats)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## metrics","metadata":{}},{"cell_type":"code","source":"#metrics\nimport numpy as np\n\n\ndef crps(y_loss_val, y_pred):\n    #y_true = np.clip(np.cumsum(y_loss_val, axis=1), 0, 1)\n    y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)\n    y_pred[:, :99-30] = 0.0\n    y_pred[:, 50+99:] = 1.0\n    val_s = ((y_true - y_pred) ** 2).sum(axis=1).sum(axis=0) / (199 * y_loss_val.shape[0])\n    crps = np.round(val_s, 6)\n    \n    return crps","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"\nfrom torch import nn\n\n\nclass Flatten(nn.Module):\n    def forward(self, input):\n        return input.view(input.size(0), -1)\n\n\nclass CnnModel(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(10, 128, kernel_size=1, stride=1, bias=False),\n            nn.CELU(inplace=True),\n            nn.Conv2d(128, 160, kernel_size=1, stride=1, bias=False),\n            nn.CELU(inplace=True),\n            nn.Conv2d(160, 128, kernel_size=1, stride=1, bias=False),\n            nn.CELU(inplace=True)\n        )\n        self.pool1 = nn.AdaptiveAvgPool2d((1, 11))\n\n        self.conv2 = nn.Sequential(\n            nn.BatchNorm2d(128),\n            nn.Conv2d(128, 160, kernel_size=(1, 1), stride=1, bias=False),\n            nn.CELU(inplace=True),\n            nn.BatchNorm2d(160),\n            nn.Conv2d(160, 96, kernel_size=(1, 1), stride=1, bias=False),\n            nn.CELU(inplace=True),\n            nn.BatchNorm2d(96),\n            nn.Conv2d(96, 96, kernel_size=(1, 1), stride=1, bias=False),\n            nn.CELU(inplace=True),\n            nn.BatchNorm2d(96),\n        )\n        self.pool2 = nn.AdaptiveAvgPool2d((1, 1))\n\n        self.last_linear = nn.Sequential(\n            Flatten(),\n            nn.Linear(96, 256),\n            nn.LayerNorm(256),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.3),\n            nn.Linear(256, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.pool1(x)\n        x = self.conv2(x)\n        x = self.pool2(x)\n        x = self.last_linear(x)\n\n        return x","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## utils","metadata":{}},{"cell_type":"code","source":"#utils\nimport os\nimport random\n\nimport numpy as np\nimport torch\n\n\ndef seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## logger","metadata":{}},{"cell_type":"code","source":"\nimport logging\nimport sys\n\nLOGGER = logging.getLogger()\nFORMATTER = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n\n\ndef setup_logger(out_file=None, stderr=True, stderr_level=logging.INFO, file_level=logging.DEBUG):\n    LOGGER.handlers = []\n    LOGGER.setLevel(min(stderr_level, file_level))\n\n    if stderr:\n        handler = logging.StreamHandler(sys.stderr)\n        handler.setFormatter(FORMATTER)\n        handler.setLevel(stderr_level)\n        LOGGER.addHandler(handler)\n\n    if out_file is not None:\n        handler = logging.FileHandler(out_file)\n        handler.setFormatter(FORMATTER)\n        handler.setLevel(file_level)\n        LOGGER.addHandler(handler)\n\n    LOGGER.info(\"logger set up\")\n    return LOGGER","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## trainer","metadata":{}},{"cell_type":"code","source":"\nimport gc\n\nimport numpy as np\nimport torch\n\n\ndef train_one_epoch(model, train_loader, criterion, optimizer, device, \n                    steps_upd_logging=500, accumulation_steps=1, scheduler=None):\n    model.train()\n\n    total_loss = 0.0\n    for step, (x, targets) in enumerate(train_loader):\n        #x= x.to(device)\n        #targets = targets.to(device)\n        optimizer.zero_grad()\n\n        logits = model(x)\n        #_, targets = targets.max(dim=1)\n        loss = criterion(logits, targets)\n        loss.backward()\n\n        if (step + 1) % accumulation_steps == 0:  # Wait for several backward steps\n            optimizer.step()  # Now we can do an optimizer step\n\n        total_loss += loss.item()\n        \n        if scheduler is not None:\n            scheduler.step()\n\n        if (step + 1) % steps_upd_logging == 0:\n            LOGGER.info('Train loss on step {} was {}'.format(step + 1, round(total_loss / (step + 1), 5)))\n\n\n    return total_loss / (step + 1)\n\ndef validate(model, val_loader, criterion, device):\n    model.eval()\n\n    val_loss = 0.0\n    true_ans_list = []\n    preds_cat = []\n    for step, (x, targets) in enumerate(val_loader):\n        #x= x.to(device)\n        #targets = targets.to(device)\n\n        logits = model(x)\n        #_, targets = targets.max(dim=1)\n        loss = criterion(logits, targets)\n        val_loss += loss.item()\n\n        targets = targets.float().detach().numpy()\n        logits = torch.softmax(logits, 1).float().detach().numpy()\n        true_ans_list.append(targets)\n        preds_cat.append(logits)\n\n        del x, targets, logits\n        gc.collect()\n\n    all_true_ans = np.concatenate(true_ans_list, axis=0)\n    all_preds = np.concatenate(preds_cat, axis=0)\n\n    return all_preds, all_true_ans, val_loss / (step + 1)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loss","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\nclass CRPSLoss(nn.Module):\n    def __init__(self, n_class=199):\n        super().__init__()\n        self.n_class = n_class\n        self.mse = torch.nn.MSELoss()\n\n    def forward(self, y_pred, y_true):\n        y_pred = torch.softmax(y_pred, 1)\n        y_pred = torch.clamp(torch.cumsum(y_pred, 1), 0, 1)\n        #crps = torch.sum(torch.sum((y_true - y_pred) ** 2, 1), 0) / (self.n_class * y_true.shape[0])\n        crps = self.mse(y_pred, y_true)\n        return crps","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## optimizer","metadata":{}},{"cell_type":"code","source":"from torch.optim import Optimizer\n\n\nclass OneCycleLR:\n    \"\"\" Sets the learing rate of each parameter group by the one cycle learning rate policy\n    proposed in https://arxiv.org/pdf/1708.07120.pdf. \n    It is recommended that you set the max_lr to be the learning rate that achieves \n    the lowest loss in the learning rate range test, and set min_lr to be 1/10 th of max_lr.\n    So, the learning rate changes like min_lr -> max_lr -> min_lr -> final_lr, \n    where final_lr = min_lr * reduce_factor.\n    Note: Currently only supports one parameter group.\n    Args:\n        optimizer:             (Optimizer) against which we apply this scheduler\n        num_steps:             (int) of total number of steps/iterations\n        lr_range:              (tuple) of min and max values of learning rate\n        momentum_range:        (tuple) of min and max values of momentum\n        annihilation_frac:     (float), fracion of steps to annihilate the learning rate\n        reduce_factor:         (float), denotes the factor by which we annihilate the learning rate at the end\n        last_step:             (int), denotes the last step. Set to -1 to start training from the beginning\n    Example:\n        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n        >>> scheduler = OneCycleLR(optimizer, num_steps=num_steps, lr_range=(0.1, 1.))\n        >>> for epoch in range(epochs):\n        >>>     for step in train_dataloader:\n        >>>         train(...)\n        >>>         scheduler.step()\n    Useful resources:\n        https://towardsdatascience.com/finding-good-learning-rate-and-the-one-cycle-policy-7159fe1db5d6\n        https://medium.com/vitalify-asia/whats-up-with-deep-learning-optimizers-since-adam-5c1d862b9db0\n    \"\"\"\n\n    def __init__(self,\n                 optimizer: Optimizer,\n                 num_steps: int,\n                 lr_range: tuple = (0.1, 1.),\n                 momentum_range: tuple = (0.85, 0.95),\n                 annihilation_frac: float = 0.1,\n                 reduce_factor: float = 0.01,\n                 last_step: int = -1):\n        # Sanity check\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        self.num_steps = num_steps\n\n        self.min_lr, self.max_lr = lr_range[0], lr_range[1]\n        assert self.min_lr < self.max_lr, \\\n            \"Argument lr_range must be (min_lr, max_lr), where min_lr < max_lr\"\n\n        self.min_momentum, self.max_momentum = momentum_range[0], momentum_range[1]\n        assert self.min_momentum < self.max_momentum, \\\n            \"Argument momentum_range must be (min_momentum, max_momentum), where min_momentum < max_momentum\"\n\n        self.num_cycle_steps = int(num_steps * (1. - annihilation_frac))  # Total number of steps in the cycle\n        self.final_lr = self.min_lr * reduce_factor\n\n        self.last_step = last_step\n\n        if self.last_step == -1:\n            self.step()\n\n    def state_dict(self):\n        \"\"\"Returns the state of the scheduler as a :class:`dict`.\n        It contains an entry for every variable in self.__dict__ which\n        is not the optimizer. (Borrowed from _LRScheduler class in torch.optim.lr_scheduler.py)\n        \"\"\"\n        return {key: value for key, value in self.__dict__.items() if key != 'optimizer'}\n\n    def load_state_dict(self, state_dict):\n        \"\"\"Loads the schedulers state. (Borrowed from _LRScheduler class in torch.optim.lr_scheduler.py)\n        Arguments:\n            state_dict (dict): scheduler state. Should be an object returned\n                from a call to :meth:`state_dict`.\n        \"\"\"\n        self.__dict__.update(state_dict)\n\n    def get_lr(self):\n        return self.optimizer.param_groups[0]['lr']\n\n    def get_momentum(self):\n        return self.optimizer.param_groups[0]['momentum']\n\n    def step(self):\n        \"\"\"Conducts one step of learning rate and momentum update\n        \"\"\"\n        current_step = self.last_step + 1\n        self.last_step = current_step\n\n        if current_step <= self.num_cycle_steps // 2:\n            # Scale up phase\n            scale = current_step / (self.num_cycle_steps // 2)\n            lr = self.min_lr + (self.max_lr - self.min_lr) * scale\n            momentum = self.max_momentum - (self.max_momentum - self.min_momentum) * scale\n        elif current_step <= self.num_cycle_steps:\n            # Scale down phase\n            scale = (current_step - self.num_cycle_steps // 2) / (self.num_cycle_steps - self.num_cycle_steps // 2)\n            lr = self.max_lr - (self.max_lr - self.min_lr) * scale\n            momentum = self.min_momentum + (self.max_momentum - self.min_momentum) * scale\n        elif current_step <= self.num_steps:\n            # Annihilation phase: only change lr\n            scale = (current_step - self.num_cycle_steps) / (self.num_steps - self.num_cycle_steps)\n            lr = self.min_lr - (self.min_lr - self.final_lr) * scale\n            momentum = None\n        else:\n            # Exceeded given num_steps: do nothing\n            return\n\n        self.optimizer.param_groups[0]['lr'] = lr\n        if momentum:\n            self.optimizer.param_groups[0]['momentum'] = momentum","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport gc\nimport sys\nimport time\n\nimport json\nimport pandas as pd\nimport numpy as np\nfrom contextlib import contextmanager\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom glob import glob\n\n#from trainer import train_one_epoch, validate\n\n# ===============\n# Constants\n# ===============\nNGS_path = glob('/kaggle/input/NFL-Punt-Analytics-Competition/NGS/*')\nDATA_DIR = \"../input/nfl-big-data-bowl-2020\"\nTRAIN_PATH = os.path.join(DATA_DIR, \"train.csv\")\nLOGGER_PATH = \"log.txt\"\nTARGET_COLUMNS = 'return_yards'\nN_CLASSES = 199\n\n# ===============\n# Settings\n# ===============\nSEED = np.random.randint(100000)\ndevice = \"cuda\"\nN_SPLITS = 5\nBATCH_SIZE = 64\nTTA = True\nEXP_ID = \"exp1\"\nepochs = 50\nEXP_ID = \"exp1_reproduce\"\n\nsetup_logger(out_file=LOGGER_PATH)\nseed_torch(SEED)\nLOGGER.info(\"seed={}\".format(SEED))\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    LOGGER.info('[{}] done in {} s'.format(name, round(time.time() - t0, 2)))\n\n\nwith timer('load data'):\n    play_information_data = pd.read_csv('/kaggle/input/nfl-punt-analytic-with-suggested-rule-change/punt_play_data.csv')\n    player_role_data = pd.read_csv('/kaggle/input/NFL-Punt-Analytics-Competition/play_player_role_data.csv')\n    train = pd.DataFrame({})\n    for path in NGS_path:\n        data = pd.read_csv(path)\n        data = pd.merge(data[data.Event == 'punt_received'],play_information_data[['GameKey' ,'PlayID' ,'return_yards']])\n        data = pd.merge(data,player_role_data)\n        train = pd.concat([train,data[data.Event == 'punt_received']])\n        del data\n    train = train.dropna(subset=['return_yards'])\n\n\n    \n    #season = train[\"Season_Year\"][::22].values\n\n\nwith timer('create features'):\n    train = reorient(train, flip_left=True)\n    train = merge_returnerfeats(train).sort_values('x')\n\n    game_id = train.groupby(['GameKey' ,'PlayID'])['x'].first().reset_index()['GameKey'].values\n    y_mae = train.groupby(['GameKey','PlayID'])[TARGET_COLUMNS].first().reset_index()[TARGET_COLUMNS].values\n    y_mae = np.where(y_mae < -30, -30, y_mae)\n    y_mae = np.where(y_mae > 50, 50, y_mae)\n    y_crps = np.zeros((y_mae.shape[0], 199))\n    for idx, target in enumerate(list(y_mae)):\n        y_crps[idx][99 + int(target):] = 1\n\n    n_train = len(train.groupby(['GameKey','PlayID'])) \n\n    n_df = len(train)\n    x, x_aug = create_features(train)\n    x = np.concatenate([x, x_aug], axis=0)\n\n    x, sc_mean, sc_std = scaling(x)\n    x_aug = x[n_train:]\n    x = x[:n_train]\n    #LOGGER.info(len(x), len(x_aug))\n\nwith timer('split data'):\n#     x_2017, y_crps_2017, y_mae_2017 = x[season==2017], y_crps[season==2017], y_mae[season==2017]\n    x_usage, y_crps_usage, y_mae_usage = x, y_crps, y_mae\n#     x_aug_2017 = x_aug[season==2017]\n    x_aug_usage = x_aug\n    folds = GroupKFold(n_splits=N_SPLITS).split(y_mae_usage, y_mae_usage, groups=game_id)\n\nwith timer('train'):\n    scores = []\n    for n_fold, (train_idx, val_idx) in enumerate(folds):\n        with timer('create model'):\n            x_train, y_train, y_train_mae = x_usage[train_idx], y_crps_usage[train_idx], y_mae_usage[train_idx]\n            x_val, y_val, y_val_mae = x_usage[val_idx], y_crps_usage[val_idx], y_mae_usage[val_idx]\n            x_aug_train, x_aug_val = x_aug_usage[train_idx], x_aug_usage[val_idx]\n            \n            # add 2017 data\n#             x_train = np.concatenate([x_train, x_2017, x_aug_2017, x_aug_train], axis=0)\n#             y_train = np.concatenate([y_train, y_crps_2017, y_crps_2017, y_train], axis=0)\n\n            train_dataset = TensorDataset(torch.tensor(x_train, dtype=torch.float32), torch.tensor(y_train, dtype=torch.float32))\n            train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=0, pin_memory=True)\n            val_dataset = TensorDataset(torch.tensor(x_val, dtype=torch.float32), torch.tensor(y_val, dtype=torch.float32))\n            val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0, pin_memory=True)\n            val_dataset_aug = TensorDataset(torch.tensor(x_aug_val, dtype=torch.float32), torch.tensor(y_val, dtype=torch.float32))\n            val_loader_aug = DataLoader(val_dataset_aug, batch_size=BATCH_SIZE, shuffle=False, num_workers=0, pin_memory=True)\n            del train_dataset, val_dataset\n            gc.collect()\n\n            model = CnnModel(num_classes=N_CLASSES)\n            #model.to(device)\n\n            num_steps = len(x_train) // BATCH_SIZE\n            #criterion = torch.nn.CrossEntropyLoss()\n            criterion = CRPSLoss()\n            optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n            #scheduler = OneCycleLR(optimizer, num_steps=num_steps, lr_range=(0.0005, 0.001))\n            scheduler = None\n\n        with timer('train fold{}'.format(n_fold)):\n            best_score = 999\n            best_epoch = 0\n            y_pred = np.zeros_like(y_crps)\n            for epoch in range(1, epochs + 1):\n                seed_torch(SEED + epoch)\n\n                LOGGER.info(\"Starting {} epoch...\".format(epoch))\n                tr_loss = train_one_epoch(model, train_loader, criterion, optimizer, device, scheduler=scheduler)\n                LOGGER.info('Mean train loss: {}'.format(round(tr_loss, 5)))\n\n                val_pred, y_true, val_loss = validate(model, val_loader, criterion, device)\n                if TTA:\n                    val_pred_aug, _, val_loss_aug = validate(model, val_loader_aug, criterion, device)\n                    LOGGER.info('valid loss: {} valid loss aug: {}'.format(round(val_loss, 5), round(val_loss_aug, 5)))\n                    val_loss = (val_loss + val_loss_aug) / 2\n                    val_pred = (val_pred + val_pred_aug) / 2\n                score = crps(y_val, val_pred)\n                LOGGER.info('Mean valid loss: {} score: {}'.format(round(val_loss, 5), round(score, 5)))\n                if score < best_score:\n                    best_score = score\n                    best_epoch = epoch\n                    torch.save(model.state_dict(), '{}_fold{}.pth'.format(EXP_ID, n_fold))\n                    y_pred[val_idx] = val_pred\n            \n            scores.append(best_score)\n            LOGGER.info(\"best score={} on epoch={} fold={}\".format(best_score, best_epoch, n_fold))\n    LOGGER.info(\"score avg={}, score fold0={}, score fold1={}, score fold2={}, score fold3={}, score fold4={}\".format(\n        np.mean(scores), scores[0], scores[1], scores[2], scores[3], scores[4]))","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Output\n\nUsing last fold of training loop for visualization ","metadata":{}},{"cell_type":"code","source":"print(model(torch.tensor(x_val, dtype=torch.float32)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_preds = [torch.softmax(model(torch.tensor(x_val, dtype=torch.float32)), dim=1).float().detach().numpy() ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_yards = []\nfor i in range(len(y_preds[0])):\n    predict_yards.append(np.sum(y_preds[0][i]*np.arange(-99,100)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nsns.regplot(predict_yards,y_val_mae)\nplt.xlabel('Predicted Return Yards')\nplt.ylabel('Real Return Yards')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that it's hard to predict punt return yards given the position of players when the the returner get the ball","metadata":{}},{"cell_type":"code","source":"test_x_punt = train[(train.PlayID == 1335) & (train.IsReturnTeam == True)]['x']\ntest_x_return = train[(train.PlayID == 1335) & (train.IsReturnTeam == False)]['x']\ntest_y_punt = train[(train.PlayID == 1335) & (train.IsReturnTeam == True)]['y']\ntest_y_return = train[(train.PlayID == 1335) & (train.IsReturnTeam == False)]['y']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualization of a random play","metadata":{}},{"cell_type":"code","source":"plt.scatter(test_x_punt,test_y_punt)\nplt.scatter(test_x_return,test_y_return)\nplt.legend(('Punt_team','Return team'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally output of train data if someone want to verify the result","metadata":{}},{"cell_type":"code","source":"train.to_csv('train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}