{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### CREDITS\n* [baseline average 1.47](https://www.kaggle.com/mlconsult/baseline-average-1-47)\n* [BaseLine Model: Player Mean or Median ?](https://www.kaggle.com/ulrich07/baseline-model-player-mean-or-median)\n* [Fork - MLB baseline avergage 1.47](https://www.kaggle.com/junichih/mlb-baseline-median-1-45) \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-16T15:46:24.94492Z","iopub.execute_input":"2021-06-16T15:46:24.945502Z","iopub.status.idle":"2021-06-16T15:46:25.761799Z","shell.execute_reply.started":"2021-06-16T15:46:24.945463Z","shell.execute_reply":"2021-06-16T15:46:25.760806Z"}}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/iterativestratification')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport pandas as pd\nimport numpy as np\nfrom datetime import timedelta\nfrom tqdm import tqdm\nimport gc\nfrom functools import reduce\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:49.110061Z","iopub.execute_input":"2021-06-24T01:54:49.110430Z","iopub.status.idle":"2021-06-24T01:54:49.976505Z","shell.execute_reply.started":"2021-06-24T01:54:49.110351Z","shell.execute_reply":"2021-06-24T01:54:49.975680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_df(df, col, bool_in=False):\n    tp = df.loc[ ~df[col].isnull() ,[col]].copy()\n    df.drop(col, axis=1, inplace=True)\n    \n    tp[col] = tp[col].str.replace(\"null\",'\"\"')\n    if bool_in:\n        tp[col] = tp[col].str.replace(\"false\",'\"False\"')\n        tp[col] = tp[col].str.replace(\"true\",'\"True\"')\n    tp[col] = tp[col].apply(lambda x: eval(x) )\n    a = tp[col].sum()\n    gc.collect()\n    return pd.DataFrame(a)\n#===============","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:49.980058Z","iopub.execute_input":"2021-06-24T01:54:49.980319Z","iopub.status.idle":"2021-06-24T01:54:49.989138Z","shell.execute_reply.started":"2021-06-24T01:54:49.980293Z","shell.execute_reply":"2021-06-24T01:54:49.988337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = \"../input/mlb-player-digital-engagement-forecasting\"","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:49.992769Z","iopub.execute_input":"2021-06-24T01:54:49.993009Z","iopub.status.idle":"2021-06-24T01:54:49.998720Z","shell.execute_reply.started":"2021-06-24T01:54:49.992986Z","shell.execute_reply":"2021-06-24T01:54:49.997910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## UTILITY FUNCTIONS","metadata":{}},{"cell_type":"code","source":"#=======================#\ndef flatten(df, col):\n    du = (df.pivot(index=\"playerId\", columns=\"EvalDate\", \n               values=col).add_prefix(f\"{col}_\").\n      rename_axis(None, axis=1).reset_index())\n    return du\n#============================#\ndef reducer(left, right):\n    return left.merge(right, on=\"playerId\")\n#========================","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:50.000617Z","iopub.execute_input":"2021-06-24T01:54:50.001035Z","iopub.status.idle":"2021-06-24T01:54:50.008948Z","shell.execute_reply.started":"2021-06-24T01:54:50.001001Z","shell.execute_reply":"2021-06-24T01:54:50.008000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TGTCOLS = [\"target1\",\"target2\",\"target3\",\"target4\"]\ndef train_lag(df, lag=1):\n    dp = df[[\"playerId\",\"EvalDate\"]+TGTCOLS].copy()\n    dp[\"EvalDate\"]  =dp[\"EvalDate\"] + timedelta(days=lag) \n    df = df.merge(dp, on=[\"playerId\", \"EvalDate\"], suffixes=[\"\",f\"_{lag}\"], how=\"left\")\n    return df\n#=================================\ndef test_lag(sub):\n    sub[\"playerId\"] = sub[\"date_playerId\"].apply(lambda s: int(  s.split(\"_\")[1]  ) )\n    assert sub.date.nunique() == 1\n    dte = sub[\"date\"].unique()[0]\n    \n    eval_dt = pd.to_datetime(dte, format=\"%Y%m%d\")\n    dtes = [eval_dt + timedelta(days=-k) for k in LAGS]\n    mp_dtes = {eval_dt + timedelta(days=-k):k for k in LAGS}\n    \n    sl = LAST.loc[LAST.EvalDate.between(dtes[-1], dtes[0]), [\"EvalDate\",\"playerId\"]+TGTCOLS].copy()\n    sl[\"EvalDate\"] = sl[\"EvalDate\"].map(mp_dtes)\n    du = [flatten(sl, col) for col in TGTCOLS]\n    du = reduce(reducer, du)\n    return du, eval_dt\n    #\n#===============","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:50.011718Z","iopub.execute_input":"2021-06-24T01:54:50.012015Z","iopub.status.idle":"2021-06-24T01:54:50.021938Z","shell.execute_reply.started":"2021-06-24T01:54:50.011986Z","shell.execute_reply":"2021-06-24T01:54:50.020840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#tr = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ntr = pd.read_csv(\"../input/mlb-data/target.csv\")\nprint(tr.shape)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:50.023076Z","iopub.execute_input":"2021-06-24T01:54:50.023412Z","iopub.status.idle":"2021-06-24T01:54:53.832103Z","shell.execute_reply.started":"2021-06-24T01:54:50.023361Z","shell.execute_reply":"2021-06-24T01:54:53.831252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr[\"EvalDate\"] = pd.to_datetime(tr[\"EvalDate\"])\ntr[\"EvalDate\"] = tr[\"EvalDate\"] + timedelta(days=-1)\ntr[\"EvalYear\"] = tr[\"EvalDate\"].dt.year","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:53.833406Z","iopub.execute_input":"2021-06-24T01:54:53.833947Z","iopub.status.idle":"2021-06-24T01:54:54.442544Z","shell.execute_reply.started":"2021-06-24T01:54:53.833911Z","shell.execute_reply":"2021-06-24T01:54:54.441670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MED_DF = tr.groupby([\"playerId\",\"EvalYear\"])[TGTCOLS].median().reset_index()\nMEDCOLS = [\"tgt1_med\",\"tgt2_med\", \"tgt3_med\", \"tgt4_med\"]\nMED_DF.columns = [\"playerId\",\"EvalYear\"] + MEDCOLS","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:54.445123Z","iopub.execute_input":"2021-06-24T01:54:54.445475Z","iopub.status.idle":"2021-06-24T01:54:54.836741Z","shell.execute_reply.started":"2021-06-24T01:54:54.445439Z","shell.execute_reply":"2021-06-24T01:54:54.835489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MED_DF.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:54.838760Z","iopub.execute_input":"2021-06-24T01:54:54.839122Z","iopub.status.idle":"2021-06-24T01:54:54.852997Z","shell.execute_reply.started":"2021-06-24T01:54:54.839086Z","shell.execute_reply":"2021-06-24T01:54:54.851967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LAG = 17\nLAGS = list(range(1, MAX_LAG + 1))\nFECOLS = [f\"{col}_{lag}\" for lag in reversed(LAGS) for col in TGTCOLS]","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:54.854484Z","iopub.execute_input":"2021-06-24T01:54:54.854865Z","iopub.status.idle":"2021-06-24T01:54:54.860820Z","shell.execute_reply.started":"2021-06-24T01:54:54.854829Z","shell.execute_reply":"2021-06-24T01:54:54.859608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAGS","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:54.862318Z","iopub.execute_input":"2021-06-24T01:54:54.862783Z","iopub.status.idle":"2021-06-24T01:54:54.871873Z","shell.execute_reply.started":"2021-06-24T01:54:54.862749Z","shell.execute_reply":"2021-06-24T01:54:54.870828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor lag in tqdm(LAGS):\n    tr = train_lag(tr, lag=lag)\n    gc.collect()\n#===========\ntr = tr.sort_values(by=[\"playerId\", \"EvalDate\"])\nprint(tr.shape)\ntr = tr.dropna()\nprint(tr.shape)\ntr = tr.merge(MED_DF, on=[\"playerId\",\"EvalYear\"])\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:54:54.873164Z","iopub.execute_input":"2021-06-24T01:54:54.873498Z","iopub.status.idle":"2021-06-24T01:55:38.601258Z","shell.execute_reply.started":"2021-06-24T01:54:54.873465Z","shell.execute_reply":"2021-06-24T01:55:38.600541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:38.602403Z","iopub.execute_input":"2021-06-24T01:55:38.602735Z","iopub.status.idle":"2021-06-24T01:55:38.627550Z","shell.execute_reply.started":"2021-06-24T01:55:38.602700Z","shell.execute_reply":"2021-06-24T01:55:38.626630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = tr[FECOLS+MEDCOLS].values\ny = tr[TGTCOLS].values\ncl = tr[\"playerId\"].values","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:38.628753Z","iopub.execute_input":"2021-06-24T01:55:38.629084Z","iopub.status.idle":"2021-06-24T01:55:40.060721Z","shell.execute_reply.started":"2021-06-24T01:55:38.629050Z","shell.execute_reply":"2021-06-24T01:55:40.059854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:40.061980Z","iopub.execute_input":"2021-06-24T01:55:40.062333Z","iopub.status.idle":"2021-06-24T01:55:40.068015Z","shell.execute_reply.started":"2021-06-24T01:55:40.062299Z","shell.execute_reply":"2021-06-24T01:55:40.067066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Neural Net Inference","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random\nimport os\nimport copy\nimport seaborn as sns\nimport gc\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import QuantileTransformer\nfrom sklearn.decomposition import PCA\nfrom datetime import timedelta\nfrom tqdm import tqdm\nimport gc\nfrom functools import reduce\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.autograd import Variable\nimport pickle\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\nDEVICE = ('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\nseed_everything(seed=42)\n\n\n############## Models ######################################################################\n\nclass simple_MLP_block(nn.Module):\n    def __init__(self, input_dim, keep_prob, out_dim):\n        super(simple_MLP_block, self).__init__()\n        self.keep_prob = keep_prob\n        self.batch_norm = nn.BatchNorm1d(input_dim)\n        if keep_prob!=0:\n            self.dropout = nn.Dropout(keep_prob)\n        self.dense = nn.Linear(input_dim, out_dim)\n\n    def forward(self, x):\n        x = self.batch_norm(x)\n        if self.keep_prob!=0:\n            x = self.dropout(x)\n        x = self.dense(x)\n\n        return x\n\nclass Simple_MLP_Model(nn.Module):  # <-- Update\n    def __init__(self, input_dim, hidden_dim, out_dim=4):\n        super(Simple_MLP_Model, self).__init__()\n\n        self.block1 = simple_MLP_block(input_dim, 0, hidden_dim)\n        self.block2 = simple_MLP_block(hidden_dim, 0.2, int(hidden_dim))\n        self.block3 = simple_MLP_block(int(hidden_dim), 0.2, out_dim)\n\n    def forward(self, x):\n\n        x = F.leaky_relu(self.block1(x))\n        x = F.leaky_relu(self.block2(x))\n        out = self.block3(x)\n\n        return out\n\n\nclass ResidualBlock(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim, downsample=None):\n        super(ResidualBlock, self).__init__()\n        self.bn1 = nn.BatchNorm1d(input_dim)\n        #self.dropout1 = nn.Dropout(input_dim/2000)\n        self.fc1 = nn.Linear(input_dim, hidden_dim)\n        self.relu1 = nn.LeakyReLU()\n\n        self.bn2 = nn.BatchNorm1d(hidden_dim)\n        self.dropout2 = nn.Dropout(0.1)\n        self.fc2 = nn.Linear(hidden_dim, output_dim)\n        self.downsample = downsample\n\n    def forward(self, x):\n        residual = x\n        out = self.bn1(x)\n        #out = self.dropout1(out)\n        out = self.fc1(out)\n        out = self.relu1(out)\n        out1 = self.bn2(out)\n        out1 = self.dropout2(out1)\n        out1 = self.fc2(out1)\n        if self.downsample:\n            residual = self.downsample(x)\n        out1 += residual\n\n        return out1, out\n\nclass ResNet(nn.Module):\n\n    def __init__(self, input, hidden_dim, block, num_classes=4):\n        super(ResNet, self).__init__()\n\n        self.layer1 = self.make_layer(block, input, hidden_dim, int(hidden_dim))\n        self.layer2 = self.make_layer(block, int(hidden_dim), int(hidden_dim), num_classes)\n\n    def make_layer(self, block, input_dim, hidden_dim, output_dim):\n        downsample = None\n        if (input_dim != output_dim):\n            downsample = nn.Sequential(\n                nn.BatchNorm1d(input_dim),\n                nn.Dropout(0.1),\n                nn.Linear(input_dim, output_dim))\n\n        layer=block(input_dim, hidden_dim, output_dim, downsample)\n\n        return layer\n\n    def forward(self, x):\n        out, _ = self.layer1(x)\n        out, _ = self.layer2(F.leaky_relu(out))\n\n        return out\n\n\n####################### Final Ensemble Model ######################################\n\nclass Ensemble_MoaModel(nn.Module):\n\n    def __init__(self, input_dims, number_of_dims, hidden_dims, model_name, out_dim=206):\n        super(Ensemble_MoaModel, self).__init__()\n\n        self.models = torch.nn.ModuleList()\n        self.input_dims = input_dims\n        self.model_name = model_name\n\n        for i in range(len(hidden_dims)):\n            if self.model_name=='Simple_MLP':\n                self.models.append(Simple_MLP_Model(input_dim=number_of_dims[i],\n                                                    hidden_dim=hidden_dims[i], out_dim=out_dim))\n\n            elif self.model_name==\"ResNet\":\n                self.models.append(ResNet(input=number_of_dims[i],\n                                          hidden_dim=hidden_dims[i],\n                                          block=ResidualBlock,\n                                          num_classes=out_dim))\n            else:\n                print(\"Please check model name. There is no this model!!!\")\n\n    def forward(self, x):\n        out = []\n\n        for i in range(len(self.input_dims)):\n            temp = self.models[i](x[:, self.input_dims[i]])\n            out.append(temp.unsqueeze(0))\n\n        out = torch.cat(out, dim=0)\n        out = out.permute(1, 0, 2)\n        out = torch.cat([out, torch.mean(out, dim=1).unsqueeze(1)], dim=1)\n\n        return out\n\n############################### Create Pytorch Dataset ########################################\n\nclass MoADataset:\n    def __init__(self, features, targets):\n        self.features = features\n        self.targets = targets\n\n    def __len__(self):\n        return (self.features.shape[0])\n\n    def __getitem__(self, idx):\n        dct = {\n            'x': torch.tensor(self.features[idx, :], dtype=torch.float),\n            'y': torch.tensor(self.targets[idx, :], dtype=torch.float),\n        }\n        return dct\n\nclass TestDataset:\n    def __init__(self, features):\n        self.features = features\n\n    def __len__(self):\n        return (self.features.shape[0])\n\n    def __getitem__(self, idx):\n        dct = {\n            'x': torch.tensor(self.features[idx, :], dtype=torch.float)\n        }\n        return dct\n\n################################# training inference #####################################\ndef train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):\n\n    model.train()\n    final_loss = 0\n\n    for data in tqdm(dataloader):\n        optimizer.zero_grad()\n        inputs, targets = data['x'].to(device), data['y'].to(device)\n\n        outputs = model(inputs)\n        loss = 0\n\n        for i in range(outputs.shape[1]):\n            loss = loss + loss_fn(outputs[:, i, :],\n                                  targets.to(device))\n\n        loss.backward()\n        #torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n        optimizer.step()\n        scheduler.step()\n\n        final_loss += loss.item()\n\n    final_loss /= len(dataloader)\n\n    return final_loss\n\n################################# validation inference #####################################\ndef valid_fn(model, dataloader, device, scheduler=None, loss_fn=None):\n\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    val_y = []\n\n    for data in dataloader:\n        inputs, targets = data['x'].to(device), data['y'].to(device)\n        val_y.append(targets.detach().cpu().numpy())\n\n        outputs = model(inputs)\n\n        if loss_fn==None:\n            valid_preds.append(np.mean(outputs.detach().cpu().numpy(), axis=1))\n            final_loss = 0\n        else:\n            loss = loss_fn(torch.mean(outputs, dim=1), targets.to(device))\n            final_loss += loss.item()\n            valid_preds.append(np.mean(outputs.detach().cpu().numpy(), axis=1))\n\n    final_loss /= len(dataloader)\n    valid_preds = np.concatenate(valid_preds)\n    val_y = np.concatenate(val_y)\n\n    score = mean_absolute_error(val_y, valid_preds)\n\n    if scheduler!=None:\n        scheduler.step(score)\n\n    return final_loss, score, valid_preds\n\n################################# prediction inference #####################################\ndef inference_fn(model, dataloader, device):\n    model.eval()\n    preds = []\n\n    for data in dataloader:\n        inputs = data['x'].to(device)\n\n        with torch.no_grad():\n            outputs = model(inputs)\n\n        preds.append(np.mean(outputs.detach().cpu().numpy(), axis=1))\n\n    preds = np.concatenate(preds)\n\n    return preds\n\n################################# Data preprocess #####################################\ndef make_df(df, col, bool_in=False):\n    tp = df.loc[~df[col].isnull(), [col]].copy()\n    df.drop(col, axis=1, inplace=True)\n\n    tp[col] = tp[col].str.replace(\"null\", '\"\"')\n    if bool_in:\n        tp[col] = tp[col].str.replace(\"false\", '\"False\"')\n        tp[col] = tp[col].str.replace(\"true\", '\"True\"')\n    tp[col] = tp[col].apply(lambda x: eval(x))\n    a = tp[col].sum()\n    gc.collect()\n    return pd.DataFrame(a)\n\ndef flatten(df, col):\n    du = (df.pivot(index=\"playerId\", columns=\"EvalDate\",\n               values=col).add_prefix(f\"{col}_\").\n      rename_axis(None, axis=1).reset_index())\n    return du\n\ndef reducer(left, right):\n    return left.merge(right, on=\"playerId\")\n\ndef train_lag(df, target_cols, lag=1):\n    dp = df[[\"playerId\",\"EvalDate\"]+target_cols].copy()\n    dp[\"EvalDate\"]  =dp[\"EvalDate\"] + timedelta(days=lag)\n    df = df.merge(dp, on=[\"playerId\", \"EvalDate\"], suffixes=[\"\",f\"_{lag}\"], how=\"left\")\n    return df\n\n\n################################# Model Training Phase #####################################\ndef run_training(folds, target, feature_cols, target_cols,\n                 input_dims, number_of_dims, hidden_dims,\n                 model_name, BATCH_SIZE, LEARNING_RATE, WEIGHT_DECAY,\n                 EPOCHS, EARLY_STOP, EARLY_STOPPING_STEPS, fold, seed, MODEL_ROOT):\n\n    seed_everything(seed)\n    folds = folds.reset_index(drop=True)\n    val_idx = folds[folds['kfold'] == fold].index\n\n    train_df = folds[folds['kfold'] != fold].reset_index(drop=True)\n    valid_df = folds[folds['kfold'] == fold].reset_index(drop=True)\n\n    x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values\n    x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values\n\n    train_dataset = MoADataset(x_train, y_train)\n    valid_dataset = MoADataset(x_valid, y_valid)\n    trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    if os.path.isfile(MODEL_ROOT + \"/\"+ f\"FOLD_{fold}_{seed}.pth\"):\n\n        model_new = torch.load(MODEL_ROOT + \"/\"+ f\"FOLD_{fold}_{seed}.pth\")\n        model_new.to(DEVICE)\n        model_new.eval()\n        oof = np.zeros((len(folds), target.shape[1]))\n        valid_loss, valid_score, valid_preds = valid_fn(model_new, validloader, DEVICE, scheduler=None, loss_fn=None)\n        oof[val_idx] = valid_preds\n\n    else:\n\n        model_new = Ensemble_MoaModel(input_dims, number_of_dims, hidden_dims, model_name, out_dim=4)\n        model_new.to(DEVICE)\n\n        optimizer = torch.optim.Adam(model_new.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n        scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,\n                                                  max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))\n\n        # scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=4,\n        #                               factor=0.25, verbose=True)\n\n\n        loss_fn = nn.L1Loss()\n\n        early_stopping_steps = EARLY_STOPPING_STEPS\n        early_step = 0\n\n        oof = np.zeros((len(folds), target.shape[1]))\n        best_loss = np.inf\n\n        for epoch in range(EPOCHS):\n\n            train_loss = train_fn(model_new, optimizer, scheduler, loss_fn, trainloader, DEVICE)\n\n            valid_loss, valid_score, valid_preds = valid_fn(model_new, validloader, DEVICE, scheduler=None, loss_fn=loss_fn)\n            print(f\"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss},  valid_score: {valid_score}\")\n\n            if valid_score < best_loss:\n\n                best_loss = valid_score\n                oof[val_idx] = valid_preds\n                torch.save(model_new, MODEL_ROOT + \"/\"+ f\"FOLD_{fold}_{seed}.pth\")\n\n            elif (EARLY_STOP == True):\n\n                early_step += 1\n                if (early_step >= early_stopping_steps):\n                    break\n\n            del train_loss, valid_loss, valid_score\n            gc.collect()\n\n        model_new = torch.load(MODEL_ROOT + \"/\"+ f\"FOLD_{fold}_{seed}.pth\")\n        model_new.to(DEVICE)\n\n    return oof\n\n\n################################# FOLD SELECTION #####################################\n\ndef fold_selection(folds, target_cols, NFOLDS=5):\n\n    train = folds.copy()\n    skf = StratifiedKFold(n_splits=NFOLDS)\n\n    for f, (t_idx, v_idx) in enumerate(skf.split(X=train, y=train[target_cols])):\n        folds.loc[v_idx, 'kfold'] = int(f)\n\n    folds['kfold'] = folds['kfold'].astype(int)\n\n    return folds\n\n################################# ENSEMBLE MODELS STRUCTURE GENERATION #####################################\n\ndef ENSEMBLE_MODEL_STRUCTURE(number_models, MODEL_ROOT):\n\n    if number_models == 1:\n\n        hidden_dims = [256]\n        number_of_dims = [num_features]\n        input_dims = [np.arange(num_features)]\n\n    elif os.path.isfile(MODEL_ROOT + \"/\" + 'hidden_dims.pkl'):\n        with open(MODEL_ROOT + \"/\" + 'hidden_dims.pkl', 'rb') as handle:\n            hidden_dims = pickle.load(handle)\n\n        with open(MODEL_ROOT + \"/\" + 'number_of_dims.pkl', 'rb') as handle:\n            number_of_dims = pickle.load(handle)\n\n        with open(MODEL_ROOT + \"/\" + 'input_dims.pkl', 'rb') as handle:\n            input_dims = pickle.load(handle)\n\n    else:\n\n        hidden_dims = np.random.randint(128, 384, number_models)\n        number_of_dims = np.random.randint(int(num_features*0.75), num_features, number_models)\n        input_dims = []\n        for i in range(number_models):\n            input_dims.append(np.random.randint(0, num_features, number_of_dims[i]))\n\n        with open(MODEL_ROOT + \"/\" + 'hidden_dims.pkl', 'wb') as handle:\n            pickle.dump(hidden_dims, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n        with open(MODEL_ROOT + \"/\" + 'number_of_dims.pkl', 'wb') as handle:\n            pickle.dump(number_of_dims, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n        with open(MODEL_ROOT + \"/\" + 'input_dims.pkl', 'wb') as handle:\n            pickle.dump(input_dims, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\n    return hidden_dims, number_of_dims, input_dims\n\n##################### Model training phase for each seed ######################\n\ndef run_k_fold(folds, target, feature_cols, target_cols, input_dims, number_of_dims, hidden_dims,\n                 model_name, BATCH_SIZE, LEARNING_RATE, WEIGHT_DECAY,\n                 EPOCHS, EARLY_STOP, EARLY_STOPPING_STEPS, NFOLDS, seed, MODEL_ROOT):\n\n    oof = np.zeros((len(folds), len(target_cols)))\n\n    for fold in range(NFOLDS):\n        oof_ = run_training(folds, target, feature_cols, target_cols,\n                 input_dims, number_of_dims, hidden_dims,\n                 model_name, BATCH_SIZE, LEARNING_RATE, WEIGHT_DECAY,\n                 EPOCHS, EARLY_STOP, EARLY_STOPPING_STEPS, fold, seed, MODEL_ROOT)\n\n        oof += oof_\n\n    return oof\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:40.069501Z","iopub.execute_input":"2021-06-24T01:55:40.069907Z","iopub.status.idle":"2021-06-24T01:55:41.618353Z","shell.execute_reply.started":"2021-06-24T01:55:40.069873Z","shell.execute_reply":"2021-06-24T01:55:41.617504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nets = []\n\nfor kf in range(5):\n    model = torch.load(f'../input/mlb-resnet-ensemble-architecture/FOLD_{kf}_20.pth')\n    nets.append(model)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:41.619732Z","iopub.execute_input":"2021-06-24T01:55:41.620090Z","iopub.status.idle":"2021-06-24T01:55:47.259950Z","shell.execute_reply.started":"2021-06-24T01:55:41.620052Z","shell.execute_reply":"2021-06-24T01:55:47.259137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Historical information to use in prediction time\nbound_dt = pd.to_datetime(\"2021-01-01\")\nLAST = tr.loc[tr.EvalDate>bound_dt].copy()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:47.263027Z","iopub.execute_input":"2021-06-24T01:55:47.263279Z","iopub.status.idle":"2021-06-24T01:55:47.409787Z","shell.execute_reply.started":"2021-06-24T01:55:47.263254Z","shell.execute_reply":"2021-06-24T01:55:47.408859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAST_MED_DF = MED_DF.loc[MED_DF.EvalYear==2021].copy()\nLAST_MED_DF.drop(\"EvalYear\", axis=1, inplace=True)\ndel tr","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:47.411153Z","iopub.execute_input":"2021-06-24T01:55:47.411498Z","iopub.status.idle":"2021-06-24T01:55:47.423104Z","shell.execute_reply.started":"2021-06-24T01:55:47.411464Z","shell.execute_reply":"2021-06-24T01:55:47.422118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAST.shape, LAST_MED_DF.shape, MED_DF.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:47.424623Z","iopub.execute_input":"2021-06-24T01:55:47.424997Z","iopub.status.idle":"2021-06-24T01:55:47.436817Z","shell.execute_reply.started":"2021-06-24T01:55:47.424959Z","shell.execute_reply":"2021-06-24T01:55:47.436045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#nets[0].summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:47.438163Z","iopub.execute_input":"2021-06-24T01:55:47.438569Z","iopub.status.idle":"2021-06-24T01:55:47.444656Z","shell.execute_reply.started":"2021-06-24T01:55:47.438482Z","shell.execute_reply":"2021-06-24T01:55:47.443776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\"\"\"\nimport mlb\nFE = []; SUB = [];\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\nfor (test_df, sub) in iter_test:\n    # Features computation at Evaluation Date\n    sub = sub.reset_index()\n    sub_fe, eval_dt = test_lag(sub)\n    sub_fe = sub_fe.merge(LAST_MED_DF, on=\"playerId\", how=\"left\")\n    sub_fe = sub_fe.fillna(0.)\n    \n    _preds = 0.\n    for reg in nets:\n        _preds += reg(torch.tensor(sub_fe[FECOLS + MEDCOLS].values, dtype=torch.float).to(DEVICE)) / 5\n    \n    _preds = np.mean(_preds.detach().cpu().numpy(), axis=1)\n    sub_fe[TGTCOLS] = np.clip(_preds, 0, 100)\n    sub.drop([\"date\"]+TGTCOLS, axis=1, inplace=True)\n    sub = sub.merge(sub_fe[[\"playerId\"]+TGTCOLS], on=\"playerId\", how=\"left\")\n    sub.drop(\"playerId\", axis=1, inplace=True)\n    sub = sub.fillna(0.)\n    # Submit\n    env.predict(sub)\n    # Update Available information\n    sub_fe[\"EvalDate\"] = eval_dt\n    #sub_fe.drop(MEDCOLS, axis=1, inplace=True)\n    LAST = LAST.append(sub_fe)\n    LAST = LAST.drop_duplicates(subset=[\"EvalDate\",\"playerId\"], keep=\"last\")\n#\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:47.445872Z","iopub.execute_input":"2021-06-24T01:55:47.446250Z","iopub.status.idle":"2021-06-24T01:55:52.186213Z","shell.execute_reply.started":"2021-06-24T01:55:47.446214Z","shell.execute_reply":"2021-06-24T01:55:52.185383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:52.188964Z","iopub.execute_input":"2021-06-24T01:55:52.189215Z","iopub.status.idle":"2021-06-24T01:55:52.201228Z","shell.execute_reply.started":"2021-06-24T01:55:52.189189Z","shell.execute_reply":"2021-06-24T01:55:52.200118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LAST.shape, sub_fe.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:52.202935Z","iopub.execute_input":"2021-06-24T01:55:52.203316Z","iopub.status.idle":"2021-06-24T01:55:52.212974Z","shell.execute_reply.started":"2021-06-24T01:55:52.203275Z","shell.execute_reply":"2021-06-24T01:55:52.211980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_tr[\"dte\"] = pd.to_datetime(df_tr[\"date\"], format='%Y%m%d')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T01:55:52.214505Z","iopub.execute_input":"2021-06-24T01:55:52.214916Z","iopub.status.idle":"2021-06-24T01:55:52.220715Z","shell.execute_reply.started":"2021-06-24T01:55:52.214877Z","shell.execute_reply":"2021-06-24T01:55:52.219608Z"},"trusted":true},"execution_count":null,"outputs":[]}]}