{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"remaining task\n- preprocess S (done)  \n- treat 2017 data (done)  \n- set same channel size (done)  \n- crps loss (done but not work well)   \n- augumentation and TTA   \n- scheduler (done but not work well)  \n- test part    ","metadata":{}},{"cell_type":"markdown","source":"## feature","metadata":{}},{"cell_type":"code","source":"#features\nimport numpy as np\nfrom numba import jit\n\n\ndef create_features(df):\n    xysdir_o = df[(df.IsOnOffense == True) & (df.IsRusher == False)][['X','Y','X_S','Y_S']].values\n    xysdir_rush = df[df.IsRusher == True][['X','Y','X_S','Y_S']].values\n    xysdir_d = df[df.IsOnOffense == False][['X','Y','X_S','Y_S']].values\n    \n    off_x = np.array(df[(df.IsOnOffense == True) & (df.IsRusher == False)].groupby('PlayId')['X'].apply(np.array))\n    def_x = np.array(df[(df.IsOnOffense == False) ].groupby('PlayId')['X'].apply(np.array))\n    off_y = np.array(df[(df.IsOnOffense == True) & (df.IsRusher == False)].groupby('PlayId')['Y'].apply(np.array))\n    def_y = np.array(df[(df.IsOnOffense == False) ].groupby('PlayId')['Y'].apply(np.array))\n    off_sx = np.array(df[(df.IsOnOffense == True) & (df.IsRusher == False)].groupby('PlayId')['X_S'].apply(np.array))\n    def_sx = np.array(df[(df.IsOnOffense == False) ].groupby('PlayId')['X_S'].apply(np.array))\n    off_sy = np.array(df[(df.IsOnOffense == True) & (df.IsRusher == False)].groupby('PlayId')['Y_S'].apply(np.array))\n    def_sy = np.array(df[(df.IsOnOffense == False) ].groupby('PlayId')['Y_S'].apply(np.array))\n    \n    player_vector = []\n    for play in range(len(off_x)):\n        player_feat = player_feature(off_x[play],def_x[play],off_y[play],def_y[play],off_sx[play],def_sx[play],\n                                     off_sy[play],def_sy[play],xysdir_rush[play])\n        player_vector.append(player_feat)\n    \n    return np.array(player_vector)\n\n    \ndef player_feature(off_x,def_x,off_y,def_y,off_sx,def_sx,off_sy,def_sy,xysdir_rush):\n    if(len(off_x<10)):\n        off_x = np.pad(off_x,(10-len(off_x),0), 'mean' )\n        off_y = np.pad(off_y,(10-len(off_y),0), 'mean' )\n        off_sx = np.pad(off_sx,(10-len(off_sx),0), 'mean' )\n        off_sy = np.pad(off_sy,(10-len(off_sy),0), 'mean' )\n    if(len(def_x<11)):\n        def_x = np.pad(def_x,(11-len(def_x),0), 'mean' )\n        def_y = np.pad(def_y,(11-len(def_y),0), 'mean' )\n        def_sx = np.pad(def_sx,(11-len(def_sx),0), 'mean' )\n        def_sy = np.pad(def_sy,(11-len(def_sy),0), 'mean' )\n\n    dist_def_off_x = def_x.reshape(-1,1)-off_x.reshape(1,-1)\n    dist_def_off_sx = def_sx.reshape(-1,1)-off_sx.reshape(1,-1)\n    dist_def_off_y = def_y.reshape(-1,1)-off_y.reshape(1,-1)\n    dist_def_off_sy = def_sy.reshape(-1,1)-off_sy.reshape(1,-1)\n    dist_def_rush_x = def_x.reshape(-1,1)-np.repeat(xysdir_rush[0],10).reshape(1,-1)\n    dist_def_rush_y = def_y.reshape(-1,1)-np.repeat(xysdir_rush[1],10).reshape(1,-1)\n    dist_def_rush_sx = def_sx.reshape(-1,1)-np.repeat(xysdir_rush[2],10).reshape(1,-1)\n    dist_def_rush_sy = def_sy.reshape(-1,1)-np.repeat(xysdir_rush[3],10).reshape(1,-1)\n    def_sx = np.repeat(def_sx,10).reshape(11,-1)\n    def_sy = np.repeat(def_sy,10).reshape(11,-1)\n    feats = [dist_def_off_x, dist_def_off_sx, dist_def_off_y, dist_def_off_sy, dist_def_rush_x, dist_def_rush_y,\n            dist_def_rush_sx, dist_def_rush_sy, def_sx, def_sy]\n    \n    return np.stack(feats)\n\n\ndef get_def_speed(df):\n    df_cp = df[~df.IsOnOffense].copy()\n    speed = df_cp[\"S\"].T.values\n    speed = speed.reshape(-1, 1, 1, 11) \n    speed = np.repeat(speed, 10, axis=2)\n\n    return speed\n\n\ndef get_dist(df, col1, col2, type=\"defence\"):\n    if type == \"defence\":\n        df_cp = df[~df.IsOnOffense].copy()\n    elif type == \"offence\":\n        df_cp = df[df.IsOnOffense].copy()\n    dist = np.linalg.norm(df_cp[col1].values - df_cp[col2].values, axis=1)\n    dist = dist.T\n    dist = dist.reshape(-1, 1, 1, 11)\n    dist = np.repeat(dist, 10, axis=2)\n\n    return dist\n\n\n\ndef dist_def_off(df, n_train, cols):\n    off_x = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['X'].apply(np.array))\n    def_x = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['X'].apply(np.array))\n    off_y = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['Y'].apply(np.array))\n    def_y = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['Y_S'].apply(np.array))\n    off_xs = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['X_S'].apply(np.array))\n    def_xs = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['X_S'].apply(np.array))\n    off_ys = np.array(df[(df.IsOnOffense) & (~train.IsRusher)].groupby('PlayId')['Y_S'].apply(np.array))\n    def_ys = np.array(df[(~df.IsOnOffense) ].groupby('PlayId')['Y_S'].apply(np.array))\n    feats = []\n    for play in range(len(off_x)):\n        dist_x = off_x[play].reshape(-1, 1) - def_x[play].reshape(1, -1)\n        dist_y = off_y[play].reshape(-1, 1) - def_y[play].reshape(1, -1)\n        dist = np.concatenate([dist_x[:, :, np.newaxis], dist_y[:, :, np.newaxis]], axis=2)\n        dist_xy = np.linalg.norm(dist.astype(np.float64), axis=2)\n        dist_xs = off_xs[play].reshape(-1, 1) - def_xs[play].reshape(1, -1)\n        dist_ys = off_ys[play].reshape(-1, 1) - def_ys[play].reshape(1, -1)\n        dist = np.concatenate([dist_xs[:, :, np.newaxis], dist_ys[:, :, np.newaxis]], axis=2)\n        dist_xys = np.linalg.norm(dist.astype(np.float64), axis=2)\n        feats.append(np.concatenate([dist_xy[np.newaxis, :], dist_xys[np.newaxis, :]], axis=0))\n    return np.array(feats)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:12.936111Z","iopub.execute_input":"2022-07-28T17:37:12.936500Z","iopub.status.idle":"2022-07-28T17:37:13.634815Z","shell.execute_reply.started":"2022-07-28T17:37:12.936398Z","shell.execute_reply":"2022-07-28T17:37:13.633727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## preprocess","metadata":{}},{"cell_type":"code","source":"#preprocess\nimport numpy as np\n\n\ndef reorient(df, flip_left, aug=False):\n    df['ToLeft'] = df.PlayDirection == \"left\"\n    \n    df.loc[df.VisitorTeamAbbr == \"ARI\", 'VisitorTeamAbbr'] = \"ARZ\"\n    df.loc[df.HomeTeamAbbr == \"ARI\", 'HomeTeamAbbr'] = \"ARZ\"\n\n    df.loc[df.VisitorTeamAbbr == \"BAL\", 'VisitorTeamAbbr'] = \"BLT\"\n    df.loc[df.HomeTeamAbbr == \"BAL\", 'HomeTeamAbbr'] = \"BLT\"\n\n    df.loc[df.VisitorTeamAbbr == \"CLE\", 'VisitorTeamAbbr'] = \"CLV\"\n    df.loc[df.HomeTeamAbbr == \"CLE\", 'HomeTeamAbbr'] = \"CLV\"\n\n    df.loc[df.VisitorTeamAbbr == \"HOU\", 'VisitorTeamAbbr'] = \"HST\"\n    df.loc[df.HomeTeamAbbr == \"HOU\", 'HomeTeamAbbr'] = \"HST\"\n\n    df['TeamOnOffense'] = \"home\"\n    df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = \"away\"\n    df['IsOnOffense'] = df.Team == df.TeamOnOffense  # Is player on offense?\n    df['YardLine_std'] = 100 - df.YardLine\n    df.loc[df.FieldPosition.fillna('') == df.PossessionTeam, 'YardLine_std'] = \\\n        df.loc[df.FieldPosition.fillna('') == df.PossessionTeam, 'YardLine']\n    df.loc[df.ToLeft, 'X'] = 120 - df.loc[df.ToLeft, 'X']\n    df.loc[df.ToLeft, 'Y'] = 160 / 3 - df.loc[df.ToLeft, 'Y']\n    df.loc[df.ToLeft, 'Orientation'] = np.mod(180 + df.loc[df.ToLeft, 'Orientation'], 360)\n    df['Dir'] = 90 - df.Dir\n    df.loc[df.ToLeft, 'Dir'] = np.mod(180 + df.loc[df.ToLeft, 'Dir'], 360)\n    df.loc[df.IsOnOffense, 'Dir'] = df.loc[df.IsOnOffense, 'Dir'].fillna(0).values\n    df.loc[~df.IsOnOffense, 'Dir'] = df.loc[~df.IsOnOffense, 'Dir'].fillna(180).values\n\n    df['IsRusher'] = df['NflId'] == df['NflIdRusher']\n    if flip_left:\n        tmp = df[df['IsRusher']].copy()\n        # df['left'] = df.Y < 160/6\n        tmp['left'] = tmp.Dir < 0\n        df = df.merge(tmp[['PlayId', 'left']], how='left', on='PlayId')\n        df['Y'] = df.Y\n        df.loc[df[\"left\"], 'Y'] = 160 / 3 - df.loc[df[\"left\"], 'Y']\n        df['Dir'] = df.Dir\n        df.loc[df[\"left\"], 'Dir'] = np.mod(- df.loc[df[\"left\"], 'Dir'], 360)\n        df.drop('left', axis=1, inplace=True)\n\n    df[\"S\"] = df[\"Dis\"] * 10\n    df['X_dir'] = np.cos((np.pi / 180) * df.Dir)\n    df['Y_dir'] = np.sin((np.pi / 180) * df.Dir)\n    df['X_S'] = df.X_dir * df.S\n    df['Y_S'] = df.Y_dir * df.S\n    df['X_A'] = df.X_dir * df.A\n    df['Y_A'] = df.Y_dir * df.A\n    #df.loc[df['Season'] == 2017, 'S'] = (df['S'][df['Season'] == 2017] - 2.4355) / 1.2930 * 1.4551 + 2.7570\n    df['time_step'] = 0.0\n    df = df.sort_values(by=['PlayId', 'IsOnOffense', 'IsRusher', 'Y']).reset_index(drop=True)\n    \n    if aug:\n        df_aug = df.copy()\n        df_aug[\"Y\"] = 53.3 - df_aug[\"Y\"]\n        df = df.append(df_aug).reset_index()\n    \n    return df\n\n\ndef merge_rusherfeats(df):\n    rusher_feats = df[df['NflId'] == df['NflIdRusher']].drop_duplicates()\n    rusher_feats = rusher_feats[[\"PlayId\", \"X\", \"Y\", \"X_S\", \"Y_S\"]]\n    rusher_feats = rusher_feats.rename(\n        columns={\"X\": \"Rusher_X\", \"Y\": \"Rusher_Y\", \"X_S\": \"Rusher_X_S\", \"Y_S\": \"Rusher_Y_S\"})\n    df = df.merge(rusher_feats, how=\"left\", on=\"PlayId\")\n\n    return df\n\ndef scaling(feats, sctype=\"standard\"):\n    v1 = []\n    v2 = []\n    for i in range(feats.shape[1]):\n        feats_ = feats[:, i, :]\n        if sctype == \"standard\":\n            mean_ = np.mean(feats_)\n            std_ = np.std(feats_)\n            feats[:, i, :] -= mean_\n            feats[:, i, :] /= std_\n            v1.append(mean_)\n            v2.append(std_)\n        elif sctype == \"minmax\":\n            max_ = np.max(feats_)\n            min_ = np.min(feats_)\n            feats[:, i, :] = (feats_ - min_) / (max_ - min_)\n            v1.append(max_)\n            v2.append(min_)\n\n    return feats, v1, v2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:13.641447Z","iopub.execute_input":"2022-07-28T17:37:13.643875Z","iopub.status.idle":"2022-07-28T17:37:13.684080Z","shell.execute_reply.started":"2022-07-28T17:37:13.643836Z","shell.execute_reply":"2022-07-28T17:37:13.683127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## metrics","metadata":{}},{"cell_type":"code","source":"#metrics\nimport numpy as np\n\n\ndef crps(y_loss_val, y_pred):\n    y_true = np.clip(np.cumsum(y_loss_val, axis=1), 0, 1)\n    y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)\n    y_pred[:, :99-30] = 0.0\n    y_pred[:, 50+99:] = 1.0\n    val_s = ((y_true - y_pred) ** 2).sum(axis=1).sum(axis=0) / (199 * y_loss_val.shape[0])\n    crps = np.round(val_s, 6)\n    \n    return crps","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:13.689114Z","iopub.execute_input":"2022-07-28T17:37:13.691728Z","iopub.status.idle":"2022-07-28T17:37:13.699664Z","shell.execute_reply.started":"2022-07-28T17:37:13.691688Z","shell.execute_reply":"2022-07-28T17:37:13.698223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nnum_feature = 10\n\n\nclass Flatten(nn.Module):\n    def forward(self, input):\n        return input.view(input.size(0), -1)\n\n\nclass CnnModel(nn.Module):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(num_feature, 128, kernel_size=1, stride=1, bias=False),\n            nn.Conv2d(128, 160, kernel_size=1, stride=1, bias=False),\n            nn.Conv2d(160, 128, kernel_size=1, stride=1, bias=False),\n        )\n        self.pool1 = nn.AdaptiveAvgPool2d((1, 10))\n\n        self.conv2 = nn.Sequential(\n            nn.Conv2d(128, 160, kernel_size=(1, 1), stride=1, bias=False),\n            nn.Conv2d(160, 96, kernel_size=(1, 1), stride=1, bias=False),\n            nn.Conv2d(96, 96, kernel_size=(1, 1), stride=1, bias=False),\n            nn.BatchNorm2d(96),\n        )\n        self.pool2 = nn.AdaptiveAvgPool2d((1, 1))\n\n        self.last_linear = nn.Sequential(\n            Flatten(),\n            nn.Linear(96, 256),\n            nn.Linear(256, 256),\n            nn.Linear(256, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.pool1(x)\n        x = self.conv2(x)\n        x = self.pool2(x)\n        x = self.last_linear(x)\n\n        return x","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:13.704623Z","iopub.execute_input":"2022-07-28T17:37:13.706628Z","iopub.status.idle":"2022-07-28T17:37:15.664789Z","shell.execute_reply.started":"2022-07-28T17:37:13.706590Z","shell.execute_reply":"2022-07-28T17:37:15.663214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## utils","metadata":{}},{"cell_type":"code","source":"#utils\nimport os\nimport random\n\nimport numpy as np\nimport torch\n\n\ndef seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:15.667123Z","iopub.execute_input":"2022-07-28T17:37:15.668309Z","iopub.status.idle":"2022-07-28T17:37:15.684364Z","shell.execute_reply.started":"2022-07-28T17:37:15.668233Z","shell.execute_reply":"2022-07-28T17:37:15.682652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## logger","metadata":{}},{"cell_type":"code","source":"#logger\nimport logging\nimport sys\n\nLOGGER = logging.getLogger()\nFORMATTER = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n\n\ndef setup_logger(out_file=None, stderr=True, stderr_level=logging.INFO, file_level=logging.DEBUG):\n    LOGGER.handlers = []\n    LOGGER.setLevel(min(stderr_level, file_level))\n\n    if stderr:\n        handler = logging.StreamHandler(sys.stderr)\n        handler.setFormatter(FORMATTER)\n        handler.setLevel(stderr_level)\n        LOGGER.addHandler(handler)\n\n    if out_file is not None:\n        handler = logging.FileHandler(out_file)\n        handler.setFormatter(FORMATTER)\n        handler.setLevel(file_level)\n        LOGGER.addHandler(handler)\n\n    LOGGER.info(\"logger set up\")\n    return LOGGER","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:15.687737Z","iopub.execute_input":"2022-07-28T17:37:15.688542Z","iopub.status.idle":"2022-07-28T17:37:15.700531Z","shell.execute_reply.started":"2022-07-28T17:37:15.688501Z","shell.execute_reply":"2022-07-28T17:37:15.699373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## trainer","metadata":{}},{"cell_type":"code","source":"#trainer\nimport gc\n\nimport numpy as np\nimport torch\n\n\ndef train_one_epoch(model, train_loader, criterion, optimizer, device, \n                    steps_upd_logging=500, accumulation_steps=1, scheduler=None):\n    model.train()\n\n    total_loss = 0.0\n    for step, (x, targets) in enumerate(train_loader):\n        x= x.to(device)\n        targets = targets.to(device)\n        optimizer.zero_grad()\n\n        logits = model(x)\n        _, targets = targets.max(dim=1)\n        loss = criterion(logits, targets)\n        loss.backward()\n\n        if (step + 1) % accumulation_steps == 0:  # Wait for several backward steps\n            optimizer.step()  # Now we can do an optimizer step\n\n        total_loss += loss.item()\n        \n        if scheduler is not None:\n            scheduler.step()\n\n        if (step + 1) % steps_upd_logging == 0:\n            LOGGER.info('Train loss on step {} was {}'.format(step + 1, round(total_loss / (step + 1), 5)))\n\n\n    return total_loss / (step + 1)\n\ndef validate(model, val_loader, criterion, device):\n    model.eval()\n\n    val_loss = 0.0\n    true_ans_list = []\n    preds_cat = []\n    for step, (x, targets) in enumerate(val_loader):\n        x= x.to(device)\n        targets = targets.to(device)\n\n        logits = model(x)\n        _, targets = targets.max(dim=1)\n        loss = criterion(logits, targets)\n        val_loss += loss.item()\n\n        targets = targets.float().detach().cpu().numpy()\n        logits = torch.softmax(logits, 1).float().detach().cpu().numpy()\n        true_ans_list.append(targets)\n        preds_cat.append(logits)\n\n        del x, targets, logits\n        gc.collect()\n\n    all_true_ans = np.concatenate(true_ans_list, axis=0)\n    all_preds = np.concatenate(preds_cat, axis=0)\n\n    return all_preds, all_true_ans, val_loss / (step + 1)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:15.702394Z","iopub.execute_input":"2022-07-28T17:37:15.702803Z","iopub.status.idle":"2022-07-28T17:37:15.724168Z","shell.execute_reply.started":"2022-07-28T17:37:15.702765Z","shell.execute_reply":"2022-07-28T17:37:15.723268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loss","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\nclass CRPSLoss(nn.Module):\n    def __init__(self, n_class=199):\n        super().__init__()\n        self.n_class = n_class\n\n    def forward(self, y_pred, y_true):\n        y_true = torch.clamp(torch.cumsum(y_true, 1), 0, 1)\n        y_pred = torch.clamp(torch.cumsum(y_pred, 1), 0, 1)\n        crps = torch.sum(torch.sum((y_true - y_pred) ** 2, 1), 0) / (self.n_class * y_true.shape[0])\n#         crps = torch.mean((y_true - y_pred) ** 2)\n        return crps","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:15.727146Z","iopub.execute_input":"2022-07-28T17:37:15.727412Z","iopub.status.idle":"2022-07-28T17:37:15.737357Z","shell.execute_reply.started":"2022-07-28T17:37:15.727388Z","shell.execute_reply":"2022-07-28T17:37:15.735977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## optimizer","metadata":{}},{"cell_type":"code","source":"from torch.optim import Optimizer\n\n\nclass OneCycleLR:\n    \"\"\" Sets the learing rate of each parameter group by the one cycle learning rate policy\n    proposed in https://arxiv.org/pdf/1708.07120.pdf. \n    It is recommended that you set the max_lr to be the learning rate that achieves \n    the lowest loss in the learning rate range test, and set min_lr to be 1/10 th of max_lr.\n    So, the learning rate changes like min_lr -> max_lr -> min_lr -> final_lr, \n    where final_lr = min_lr * reduce_factor.\n    Note: Currently only supports one parameter group.\n    Args:\n        optimizer:             (Optimizer) against which we apply this scheduler\n        num_steps:             (int) of total number of steps/iterations\n        lr_range:              (tuple) of min and max values of learning rate\n        momentum_range:        (tuple) of min and max values of momentum\n        annihilation_frac:     (float), fracion of steps to annihilate the learning rate\n        reduce_factor:         (float), denotes the factor by which we annihilate the learning rate at the end\n        last_step:             (int), denotes the last step. Set to -1 to start training from the beginning\n    Example:\n        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n        >>> scheduler = OneCycleLR(optimizer, num_steps=num_steps, lr_range=(0.1, 1.))\n        >>> for epoch in range(epochs):\n        >>>     for step in train_dataloader:\n        >>>         train(...)\n        >>>         scheduler.step()\n    Useful resources:\n        https://towardsdatascience.com/finding-good-learning-rate-and-the-one-cycle-policy-7159fe1db5d6\n        https://medium.com/vitalify-asia/whats-up-with-deep-learning-optimizers-since-adam-5c1d862b9db0\n    \"\"\"\n\n    def __init__(self,\n                 optimizer: Optimizer,\n                 num_steps: int,\n                 lr_range: tuple = ( 0.0005 , 0.001 ),\n                 momentum_range: tuple = (0.85, 0.95),\n                 annihilation_frac: float = 0.1,\n                 reduce_factor: float = 0.01,\n                 last_step: int = -1):\n        # Sanity check\n        if not isinstance(optimizer, Optimizer):\n            raise TypeError('{} is not an Optimizer'.format(type(optimizer).__name__))\n        self.optimizer = optimizer\n\n        self.num_steps = num_steps\n\n        self.min_lr, self.max_lr = lr_range[0], lr_range[1]\n        assert self.min_lr < self.max_lr, \\\n            \"Argument lr_range must be (min_lr, max_lr), where min_lr < max_lr\"\n\n        self.min_momentum, self.max_momentum = momentum_range[0], momentum_range[1]\n        assert self.min_momentum < self.max_momentum, \\\n            \"Argument momentum_range must be (min_momentum, max_momentum), where min_momentum < max_momentum\"\n\n        self.num_cycle_steps = int(num_steps * (1. - annihilation_frac))  # Total number of steps in the cycle\n        self.final_lr = self.min_lr * reduce_factor\n\n        self.last_step = last_step\n\n        if self.last_step == -1:\n            self.step()\n\n    def state_dict(self):\n        \"\"\"Returns the state of the scheduler as a :class:`dict`.\n        It contains an entry for every variable in self.__dict__ which\n        is not the optimizer. (Borrowed from _LRScheduler class in torch.optim.lr_scheduler.py)\n        \"\"\"\n        return {key: value for key, value in self.__dict__.items() if key != 'optimizer'}\n\n    def load_state_dict(self, state_dict):\n        \"\"\"Loads the schedulers state. (Borrowed from _LRScheduler class in torch.optim.lr_scheduler.py)\n        Arguments:\n            state_dict (dict): scheduler state. Should be an object returned\n                from a call to :meth:`state_dict`.\n        \"\"\"\n        self.__dict__.update(state_dict)\n\n    def get_lr(self):\n        return self.optimizer.param_groups[0]['lr']\n\n    def get_momentum(self):\n        return self.optimizer.param_groups[0]['momentum']\n\n    def step(self):\n        \"\"\"Conducts one step of learning rate and momentum update\n        \"\"\"\n        current_step = self.last_step + 1\n        self.last_step = current_step\n\n        if current_step <= self.num_cycle_steps // 2:\n            # Scale up phase\n            scale = current_step / (self.num_cycle_steps // 2)\n            lr = self.min_lr + (self.max_lr - self.min_lr) * scale\n            momentum = self.max_momentum - (self.max_momentum - self.min_momentum) * scale\n        elif current_step <= self.num_cycle_steps:\n            # Scale down phase\n            scale = (current_step - self.num_cycle_steps // 2) / (self.num_cycle_steps - self.num_cycle_steps // 2)\n            lr = self.max_lr - (self.max_lr - self.min_lr) * scale\n            momentum = self.min_momentum + (self.max_momentum - self.min_momentum) * scale\n        elif current_step <= self.num_steps:\n            # Annihilation phase: only change lr\n            scale = (current_step - self.num_cycle_steps) / (self.num_steps - self.num_cycle_steps)\n            lr = self.min_lr - (self.min_lr - self.final_lr) * scale\n            momentum = None\n        else:\n            # Exceeded given num_steps: do nothing\n            return\n\n        self.optimizer.param_groups[0]['lr'] = lr\n        if momentum:\n            self.optimizer.param_groups[0]['momentum'] = momentum","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-28T17:37:15.739613Z","iopub.execute_input":"2022-07-28T17:37:15.740722Z","iopub.status.idle":"2022-07-28T17:37:15.769493Z","shell.execute_reply.started":"2022-07-28T17:37:15.740684Z","shell.execute_reply":"2022-07-28T17:37:15.767377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport time\n\nimport json\nimport pandas as pd\nimport numpy as np\nfrom contextlib import contextmanager\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom tqdm.auto import tqdm\n\n#from trainer import train_one_epoch, validate\n\n# ===============\n# Constants\n# ===============\nDATA_DIR = \"../input/nfl-big-data-bowl-2020-final-stage-2-dataset\"\nTRAIN_PATH = os.path.join(DATA_DIR, \"train.csv\")\nTEST_PATH = os.path.join(DATA_DIR, \"test.csv\")\nSOLUTION_PATH = os.path.join(DATA_DIR, \"solution.csv\")\nLOGGER_PATH = \"log.txt\"\nTARGET_COLUMNS = 'Yards'\nN_CLASSES = 199\n\n# ===============\n# Settings\n# ===============\nSEED = 69160\ndevice = \"cuda\"\nN_SPLITS = 5\nBATCH_SIZE = 64\nAUG = False\n\nepochs = 50\nEXP_ID = \"exp2_10feat\"\n\nsetup_logger(out_file=LOGGER_PATH)\nseed_torch(SEED)\nLOGGER.info(\"seed={}\".format(SEED))\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    LOGGER.info('[{}] done in {} s'.format(name, round(time.time() - t0, 2)))\n\n\nwith timer('load data'):\n    train = pd.read_csv(TRAIN_PATH, dtype={'WindSpeed': 'object'})\n    test = pd.read_csv(TEST_PATH, dtype={'WindSpeed': 'object'})\n    solution = pd.read_csv(SOLUTION_PATH)\n    test = pd.merge(test,solution)\n    train = pd.concat([train,test])\n#     train = train[train['Season'] != 2017]\n    game_id = train[\"GameId\"][::22].values\n    season = train[\"Season\"][::22].values\n    y_mae = train[TARGET_COLUMNS][::22].values\n    y_mae = np.where(y_mae < -16, -16, y_mae)\n    y_mae = np.where(y_mae > 50, 50, y_mae)\n    y_crps = np.zeros((y_mae.shape[0], 199))\n    for idx, target in enumerate(list(y_mae)):\n        y_crps[idx][99 + target] = 1\n\n    n_train = len(train) // 22\n    n_train_2017 = len(train[train.Season == 2017]) // 22\n    n_df = len(train)\n\nwith timer('create features'):\n    train = reorient(train, flip_left=True, aug=AUG)\n    train = merge_rusherfeats(train)\n\n    #x_def_speed = get_def_speed(train)\n    #x_def_rusher_dist = get_dist(train, [\"X\", \"Y\"], [\"Rusher_X\", \"Rusher_Y\"], \"defence\")\n    #x_def_rusher_speeddist = get_dist(train, [\"X_S\", \"Y_S\"], [\"Rusher_X_S\", \"Rusher_Y_S\"], \"defence\")\n    #x_def_off_dist = dist_def_off(train, n_train*2 if AUG else n_train, [[\"X\", \"Y\"], [\"X_S\", \"Y_S\"]])\n    #x = np.concatenate([x_def_speed, x_def_rusher_dist, x_def_rusher_speeddist, x_def_off_dist], axis=1)\n    x = create_features(train)\n    x, sc_mean, sc_std = scaling(x)\n\nwith timer('split data'):\n    if AUG:\n        x_aug = x[n_df:]\n        x = x[:n_df]\n        x_aug_2017 = x_aug[season==2017]\n        x_aug_usage = x_aug[season!=2017]\n    x_2017, y_crps_2017, y_mae_2017 = x[season==2017], y_crps[season==2017], y_mae[season==2017]\n    x_usage, y_crps_usage, y_mae_usage = x[season!=2017], y_crps[season!=2017], y_mae[season!=2017]\n    folds = GroupKFold(n_splits=N_SPLITS).split(y_mae_usage, y_mae_usage, groups=game_id[season!=2017])\n\nwith timer('train'):\n    scores = []\n    for n_fold, (train_idx, val_idx) in enumerate(folds):\n        with timer('create model'):\n            x_train, y_train = x_usage[train_idx], y_crps_usage[train_idx]\n            x_val, y_val, y_val_mae = x_usage[val_idx], y_crps_usage[val_idx], y_mae_usage[val_idx]\n            \n            # add 2017 data\n            x_train = np.concatenate([x_train, x_2017], axis=0)\n            y_train = np.concatenate([y_train, y_crps_2017], axis=0)\n            if AUG:\n                x_train = np.concatenate([x_train, x_aug_usage[train_idx], x_aug_2017], axis=0)\n                y_train = np.concatenate([y_train, y_train], axis=0)\n                val_dataset_aug = TensorDataset(torch.tensor(x_aug_usage[val_idx], dtype=torch.float32), torch.tensor(y_val, dtype=torch.float32))\n                val_loader_aug = DataLoader(val_dataset_aug, batch_size=BATCH_SIZE, shuffle=False, num_workers=0, pin_memory=True)\n\n            train_dataset = TensorDataset(torch.tensor(x_train, dtype=torch.float32), torch.tensor(y_train, dtype=torch.float32))\n            train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, pin_memory=True)\n            val_dataset = TensorDataset(torch.tensor(x_val, dtype=torch.float32), torch.tensor(y_val, dtype=torch.float32))\n            val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, pin_memory=True)\n            del train_dataset, val_dataset\n            gc.collect()\n\n            model = CnnModel(num_classes=N_CLASSES)\n            model.to(device)\n\n            num_steps = len(x_train) // BATCH_SIZE\n            criterion = torch.nn.CrossEntropyLoss()\n#             criterion = CRPSLoss()\n            optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n#             scheduler = OneCycleLR(optimizer, num_steps=num_steps, lr_range=(5e-4, 1e-3))\n            scheduler = None\n\n        with timer('train fold{}'.format(n_fold)):\n            best_score = 999\n            best_epoch = 0\n            y_pred = np.zeros_like(y_crps)\n            for epoch in tqdm(range(1, epochs + 1)):\n                seed_torch(SEED + epoch)\n\n                LOGGER.info(\"Starting {} epoch...\".format(epoch))\n                tr_loss = train_one_epoch(model, train_loader, criterion, optimizer, device, scheduler=scheduler)\n                LOGGER.info('Mean train loss: {}'.format(round(tr_loss, 5)))\n\n                val_pred, y_true, val_loss = validate(model, val_loader, criterion, device)\n                if AUG:\n                    val_pred_aug, _, val_loss_aug = validate(model, val_loader_aug, criterion, device)\n                    val_loss = [val_loss, val_loss_aug]\n                    val_pred = val_pred + val_pred_aug\n                score = crps(y_val, val_pred)\n                LOGGER.info('Mean valid loss: {} score: {}'.format(round(val_loss, 5), round(score, 5)))\n                if score < best_score:\n                    best_score = score\n                    best_epoch = epoch\n                    torch.save(model.state_dict(), '{}_fold{}.pth'.format(EXP_ID, n_fold))\n                    y_pred[val_idx] = val_pred\n            \n            scores.append(best_score)\n            LOGGER.info(\"best score={} on epoch={} fold={}\".format(best_score, best_epoch, n_fold))\n    LOGGER.info(\"score avg={}, score fold0={}, score fold1={}, score fold2={}, score fold3={}, score fold4={}\".format(\n        np.mean(scores), scores[0], scores[1], scores[2], scores[3], scores[4]))","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-28T17:41:01.011638Z","iopub.execute_input":"2022-07-28T17:41:01.011985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-28T17:38:14.387070Z","iopub.status.idle":"2022-07-28T17:38:14.389870Z","shell.execute_reply.started":"2022-07-28T17:38:14.389599Z","shell.execute_reply":"2022-07-28T17:38:14.389626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## Test part haven't developed yet.\n# from kaggle.competitions import nflrush\n\n\n# def scaling_test(feats, means, stds):\n#     for i in range(feats.shape[1]):\n#         feats_ = feats[:, i, :]\n#         mean_ = means[i]\n#         std_ = stds[i]\n#         feats[:, i, :] -= mean_\n#         feats[:, i, :] /= std_\n\n#     return feats\n\n\n# def post_process(y_pred, yardline_train, time_steps):\n#     upper = 100 - yardline_train\n#     lower = - yardline_train\n#     for i in range(y_pred.shape[0]):\n#         y_pred[i, 99 + upper[i]:] = 1\n#         y_pred[i, :99 + lower[i]] = 0\n#         y_pred[i] = nondecreasing(y_pred[i].ravel())\n#     if 1:\n#         n_steps = len(time_steps)\n#         len_y_pred = len(y_pred) // n_steps\n#         y_pred = np.mean([y_pred[len_y_pred * i: len_y_pred * (i+1)] for i in range(n_steps)], axis=0) \n#     return y_pred\n\n\n# if False:\n#     env = nflrush.make_env()\n#     iter_test = env.iter_test()\n\n#     models = []\n#     for i in range(N_SPLITS):\n#         model = CnnModel(num_classes=N_CLASSES)\n#         model.load_state_dict(torch.load('{}_fold{}.pth'.format(EXP_ID, n_fold)))\n#         models.append(model)\n\n#     for test_df, sample_prediction_df in iter_test:\n#         n_test = len(test_df) // 22\n#         test_df = reorient(test_df, flip_left=True)\n#         test_df = merge_rusherfeats(test_df)\n\n#         x_def_speed = get_def_speed(test_df)\n#         x_def_rusher_dist = get_dist(test_df, [\"X\", \"Y\"], [\"Rusher_X\", \"Rusher_Y\"], \"defence\")\n#         x_def_rusher_speeddist = get_dist(test_df, [\"X_S\", \"Y_S\"], [\"Rusher_X_S\", \"Rusher_Y_S\"], \"defence\")\n#         x_def_off_dist = dist_def_off(test_df, n_test, [\"X\", \"Y\", \"X_S\", \"Y_S\"])\n#         x = np.concatenate([x_def_speed, x_def_rusher_dist, x_def_rusher_speeddist, x_def_off_dist], axis=1)\n#         x = scaling_test(x, sc_mean, sc_std)\n\n#         y_preds = [torch.softmax(model(torch.tensor(x, dtype=torch.float32)), dim=1).float().detach().numpy() for model in models]\n#         y_pred = np.mean(y_preds, axis=0)\n#         y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)\n#         #y_pred = post_process(y_pred, yardline, time_steps)\n#         sample_prediction_df.iloc[0, :] = y_pred.reshape(-1)\n#         env.predict(sample_prediction_df)\n#     env.write_submission_file()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-28T17:38:14.393842Z","iopub.status.idle":"2022-07-28T17:38:14.394506Z","shell.execute_reply.started":"2022-07-28T17:38:14.394237Z","shell.execute_reply":"2022-07-28T17:38:14.394261Z"},"trusted":true},"execution_count":null,"outputs":[]}]}