{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ❗️❗️❗️DISCLAIMER: This notebook is a copy of [great notebook from @paddykb](https://www.kaggle.com/code/paddykb/tps-2022-10-fastai) with multistart NN training and test dataset augmentation (TTA). If you like this kernel, you MUST upvote @paddykb's kernel beforehand and don't forget to upvote this one too ❗️❗️❗️","metadata":{}},{"cell_type":"code","source":"import random\nimport numpy as np\nimport pandas as pd\nimport gc\nfrom pathlib import Path\nfrom fastai.tabular.all import *\nimport fastai.losses as loss","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-14T20:48:29.780431Z","iopub.execute_input":"2022-10-14T20:48:29.780995Z","iopub.status.idle":"2022-10-14T20:48:32.944081Z","shell.execute_reply.started":"2022-10-14T20:48:29.780915Z","shell.execute_reply":"2022-10-14T20:48:32.935697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Introduction\n\nIn this notebook I'm building a NN using fastai. The \"value add\" is the augmentation of the training batches with random permutations from the 144 possible:\n* Six ways to order team A players.\n* Six ways to order team B players\n* flip teams A and B\n* horizontal reflection\n\nI've made no attempt to tune the model or try different architectures. If you do, please make your notebook public so I can learn from your efforts.\n\nOn the shoulders of giants:\n* @pietromaldini1 [data augmentation discussion](https://www.kaggle.com/competitions/tabular-playground-series-oct-2022/discussion/357577)\n* @hsuyab [Fast loading & High Compression with Feather](https://www.kaggle.com/code/hsuyab/fast-loading-high-compression-with-feather)\n* @slawekbiel [preloading data into the GPU](https://www.kaggle.com/code/slawekbiel/fast-fastai-training)\n* @spyrow [Mirroring the board](https://www.kaggle.com/code/spyrow/playground-oct-2022-lgbmclassifier?scriptVersionId=107206095)\n\n**todo**: *Since we have event order in the train set, it would be interesting to model successive positions. I.e. use the current positions as input and the next position of players and ball as the output. (strip off the last layer and use it as an embedding). Take a closer look at borrowing this from @ryancaldwell [Predict next frame](https://www.kaggle.com/code/ryancaldwell/cnn-predict-next-frame)*\n\n\nWhat's new in v3?\n* shuffle the training data between epochs\n* feature engineering ideas @samuelcortinhas, @pietromaldini1 and others. (euclidean distance to the goal, distance of players from the ball, and speeds)\n* still looking at @ryancaldwell [Predict next frame](https://www.kaggle.com/code/ryancaldwell/cnn-predict-next-frame)*\n* Increased the number of training epochs.","metadata":{}},{"cell_type":"code","source":"features = [\n    'ball_pos_x', 'ball_pos_y','ball_pos_z', 'ball_vel_x', 'ball_vel_y', 'ball_vel_z', \n    'p0_pos_x', 'p0_pos_y', 'p0_pos_z', 'p0_vel_x', 'p0_vel_y', 'p0_vel_z', 'p0_boost', 'p0_na',\n    'p1_pos_x', 'p1_pos_y', 'p1_pos_z', 'p1_vel_x', 'p1_vel_y', 'p1_vel_z', 'p1_boost', 'p1_na',\n    'p2_pos_x', 'p2_pos_y', 'p2_pos_z', 'p2_vel_x', 'p2_vel_y', 'p2_vel_z', 'p2_boost', 'p2_na',\n    'p3_pos_x', 'p3_pos_y', 'p3_pos_z', 'p3_vel_x', 'p3_vel_y', 'p3_vel_z', 'p3_boost', 'p3_na',\n    'p4_pos_x', 'p4_pos_y', 'p4_pos_z', 'p4_vel_x', 'p4_vel_y', 'p4_vel_z', 'p4_boost', 'p4_na',\n    'p5_pos_x', 'p5_pos_y', 'p5_pos_z', 'p5_vel_x', 'p5_vel_y', 'p5_vel_z', 'p5_boost', 'p5_na',\n    'boost0_timer', 'boost1_timer', \n    'boost2_timer', 'boost3_timer',\n    'boost4_timer', 'boost5_timer']\n\nfeatures_x_pos = [pos for pos, feature in enumerate(features) if feature.endswith('_x')]\nfeatures_y_pos = [pos for pos, feature in enumerate(features) if feature.endswith('_y')]\n\ntargets = [\n    'team_A_scoring_within_10sec',\n    'team_B_scoring_within_10sec']","metadata":{"execution":{"iopub.status.busy":"2022-10-14T20:49:54.986509Z","iopub.execute_input":"2022-10-14T20:49:54.987204Z","iopub.status.idle":"2022-10-14T20:49:55.001977Z","shell.execute_reply.started":"2022-10-14T20:49:54.987155Z","shell.execute_reply":"2022-10-14T20:49:55.000958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"%%time\nDEBUG = False\ninput_path = Path('../input/fast-loading-high-compression-with-feather/feather_data')\n\ndef fe(x):\n    # indicators for respawns...\n    x['p0_na'] = x['p0_pos_x'].isna().astype('int8')\n    x['p1_na'] = x['p1_pos_x'].isna().astype('int8')\n    x['p2_na'] = x['p2_pos_x'].isna().astype('int8')\n    x['p3_na'] = x['p3_pos_x'].isna().astype('int8')\n    x['p4_na'] = x['p4_pos_x'].isna().astype('int8')\n    x['p5_na'] = x['p5_pos_x'].isna().astype('int8')\n    for feature in features:\n        if feature.endswith('_na'):\n            continue\n        # this is just scaling the features to something reasonable\n        # it might make sense to apply a transformation to the z-dimension.\n        if feature.endswith('_x'):\n            x[feature] = (x[feature] / 82).fillna(0).astype('float16')\n        if feature.endswith('_y'):\n            x[feature] = (x[feature] / 120).fillna(0).astype('float16')\n        if feature.endswith('_z'):\n            x[feature] = (x[feature] / 40).fillna(0).astype('float16')\n        if feature.endswith('_boost'):\n            x[feature] = (x[feature] / 100).fillna(0).astype('float16')\n        if feature.endswith('_timer'):\n            x[feature] = (-x[feature] / 100).astype('float16')\n    return x\n\ndef read_train():\n    dfs = []\n    for i in range(10):\n        dfs.append(fe(pd.read_feather(input_path / f'train_{i}_compressed.ftr')))\n    result = pd.concat(dfs)\n    if DEBUG:\n        result = result.sample(frac=0.05)\n    return result\n\ndef read_test():\n    return fe(pd.read_feather(input_path / 'test_compressed.ftr'))\n\ndf_train = read_train()\ngc.collect()\n\nprint(f'Train Rows = {len(df_train):,}  ' \n      f'Memory Usage = {df_train.memory_usage(deep=True).sum() / (1024 * 1024):4.1f} Mb'\n     '\\n')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T20:49:56.083388Z","iopub.execute_input":"2022-10-14T20:49:56.083846Z","iopub.status.idle":"2022-10-14T20:50:59.572978Z","shell.execute_reply.started":"2022-10-14T20:49:56.083807Z","shell.execute_reply":"2022-10-14T20:50:59.571882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Move Training Data to the GPU","metadata":{}},{"cell_type":"code","source":"# split train & validation\n\ngame_nums = df_train['game_num'].unique()\ntrain_game_nums = random.sample(list(game_nums), int(len(game_nums) * 0.80))\n\ntrain_feature_tensor = torch.tensor(\n    df_train.query(\"game_num in @train_game_nums\")[features].to_numpy())\ntrain_target_tensor  = torch.tensor(\n    df_train.query(\"game_num in @train_game_nums\")[targets].to_numpy())\nvalid_feature_tensor = torch.tensor(\n    df_train.query(\"game_num not in @train_game_nums\")[features].to_numpy())\nvalid_target_tensor  = torch.tensor(\n    df_train.query(\"game_num not in @train_game_nums\")[targets].to_numpy())\n\ngc.collect()\n\nif torch.cuda.is_available():\n    train_feature_tensor = train_feature_tensor.cuda()\n    train_target_tensor  = train_target_tensor.cuda()\n    valid_feature_tensor = valid_feature_tensor.cuda()\n    valid_target_tensor  = valid_target_tensor.cuda()\n\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T20:50:59.575147Z","iopub.execute_input":"2022-10-14T20:50:59.575853Z","iopub.status.idle":"2022-10-14T20:51:50.583015Z","shell.execute_reply.started":"2022-10-14T20:50:59.575816Z","shell.execute_reply":"2022-10-14T20:51:50.582043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some light feature engineering on each batch\n\ndef fe_goal_distance(ball):\n    # might need to rescale the vectors...\n    dist_a = ((ball[:,0:1] - 0) ** 2 + (ball[:,1:2] - 1) ** 2 + (ball[:,2:3] - 0) ** 2) ** 0.5 / 2\n    dist_b = ((ball[:,0:1] - 0) ** 2 + (ball[:,1:2] + 1) ** 2 + (ball[:,2:3] - 0) ** 2) ** 0.5 / 2 \n    return dist_a, dist_b\n    \ndef fe_dist_ball_player(ball, player):\n    # might need to rescale the vectors...\n    dist = ((\n        (ball[:,0:1] - player[:,0:1]) ** 2 + \n        (ball[:,1:2] - player[:,1:2]) ** 2 + \n        (ball[:,2:3] - player[:,2:3]) ** 2) ** 0.5) / 12\n    return dist\n\ndef fe_speed_of_thing(thing):\n    # might need to rescale the vectors...\n    return (thing[:, 3:4] ** 2 + thing[:, 4:5] ** 2 + thing[:, 5:6] ** 2) ** 0.5\n\ndef augment_fe(empty, X, Y):\n    ball = X[:, :6]\n    p0 = X[:,  6:14]\n    p1 = X[:, 14:22]\n    p2 = X[:, 22:30]\n    p3 = X[:, 30:38]\n    p4 = X[:, 38:46]\n    p5 = X[:, 46:54]\n    boosts = X[:, 54:]\n    \n    ## distance to goal\n    goal_a, goal_b = fe_goal_distance(ball)\n    \n    ## distance to ball\n    p0d = fe_dist_ball_player(ball, p0)\n    p1d = fe_dist_ball_player(ball, p1)\n    p2d = fe_dist_ball_player(ball, p2)\n    p3d = fe_dist_ball_player(ball, p3)\n    p4d = fe_dist_ball_player(ball, p4)\n    p5d = fe_dist_ball_player(ball, p5)\n    \n    ## speeds\n    ball_s = fe_speed_of_thing(ball)\n    p0s = fe_speed_of_thing(p0)\n    p1s = fe_speed_of_thing(p1)\n    p2s = fe_speed_of_thing(p2)\n    p3s = fe_speed_of_thing(p3)\n    p4s = fe_speed_of_thing(p4)\n    p5s = fe_speed_of_thing(p5)\n    \n    new_X = torch.cat([\n        ball, p0, p1, p2, p3, p4, p5, boosts,\n        # 15 new features:\n        goal_a, goal_b,\n        ball_s,\n        p0d, p1d, p2d, p3d, p4d, p5d,\n        p0s, p1s, p2s, p3s, p4s, p5s\n    ], dim=1)\n    \n    return empty, new_X, Y","metadata":{"execution":{"iopub.status.busy":"2022-10-14T20:51:50.587454Z","iopub.execute_input":"2022-10-14T20:51:50.589774Z","iopub.status.idle":"2022-10-14T20:51:50.608926Z","shell.execute_reply.started":"2022-10-14T20:51:50.589735Z","shell.execute_reply":"2022-10-14T20:51:50.607939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augment_mirror(empty, X, Y):\n    # mirror the match\n    # interchange player 1 and 2\n    positions = X[:,:54]\n    positions[:, features_x_pos] = -positions[:, features_x_pos]\n    positions[:, features_y_pos] = -positions[:, features_y_pos]\n    \n    ball = positions[:, :6]\n    p0 = positions[:,  6:14]\n    p1 = positions[:, 14:22]\n    p2 = positions[:, 22:30]\n    p3 = positions[:, 30:38]\n    p4 = positions[:, 38:46]\n    p5 = positions[:, 46:54]\n    \n    players = torch.cat([p3, p4, p5, p0, p1, p2], dim=1)\n    # mirror\n    boosts = X[:, [59, 58, 57, 56, 55, 54]]\n    \n    flip_X = torch.cat([ball, players, boosts], dim=1)\n    flip_Y = Y[:, :, [1,0]]\n    \n    return empty, flip_X, flip_Y\n\ndef augment_flip_x(empty, X, Y):\n    # mirror the match in the Y-axis\n    positions = X[:,:54]\n    positions[:, features_x_pos] = -positions[:, features_x_pos]\n    boosts = X[:, [55, 54, 57, 56, 59, 58]]\n    \n    flip_X = torch.cat([positions, boosts], dim=1)\n    \n    return empty, flip_X, Y\n\ndef augment_shuffle(empty, X, Y):\n    # randomly order players (within teams)\n    ball = X[:, :6]\n    p0 = X[:,  6:14]\n    p1 = X[:, 14:22]\n    p2 = X[:, 22:30]\n    p3 = X[:, 30:38]\n    p4 = X[:, 38:46]\n    p5 = X[:, 46:54]\n    boosts = X[:, 54:]\n    \n    # shuffle player positions\n    pA = torch.cat(random.sample([p0, p1, p2], 3), dim=1)\n    pB = torch.cat(random.sample([p3, p4, p5], 3), dim=1)\n    \n    # shuffled feats\n    shuffled_X = torch.cat([ball, pA, pB, boosts], dim=1)\n    \n    return empty, shuffled_X, Y\n\n# This bespoke dataset classes are based on the ideas of @slawekbiel \nclass BespokeDataset:\n    def __init__(self, feature_tensor, targets, augment=False, \n                 augment_coef_mirror = 0.5, augment_coef_flipx = 0.5):\n        store_attr()\n        self.n_inp = 2\n        self.augment_coef_mirror = augment_coef_mirror\n        self.augment_coef_flipx = augment_coef_flipx\n        \n    def __getitem__(self, idx):\n        # convert float16 -> float32 during the minibatch\n        # and apply any augmentation\n        batch = torch.empty(0), self.feature_tensor[idx].float(), self.targets[idx, None]\n        if self.augment:\n            # shuffle player positions.\n            batch = augment_shuffle(*batch)\n            if random.random() > self.augment_coef_mirror:\n                batch = augment_mirror(*batch)\n            if random.random() > self.augment_coef_flipx:\n                batch = augment_flip_x(*batch)\n        # Apply feature engineering here \n        # (as we'll only use a little memory)\n        batch = augment_fe(*batch)\n        return batch\n    \n    def __len__(self):\n        return len(self.feature_tensor)\n    \nclass BespokeDL(DataLoader):\n    def __iter__(self):\n        if self.shuffle:\n            self.__idxs = torch.tensor(np.random.permutation(range(0,self.n)))\n        else:\n            self.__idxs = torch.tensor(range(0,self.n))\n        for batch_start in range(0, self.n, self.bs):\n            if batch_start + self.bs > self.n and self.drop_last:\n                return \n            indices = self.__idxs[batch_start:batch_start+self.bs]\n            yield self.dataset[indices]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T20:51:50.614136Z","iopub.execute_input":"2022-10-14T20:51:50.616584Z","iopub.status.idle":"2022-10-14T20:51:50.643197Z","shell.execute_reply.started":"2022-10-14T20:51:50.616545Z","shell.execute_reply":"2022-10-14T20:51:50.642328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit multiple times using all training data","metadata":{}},{"cell_type":"code","source":"# refit on full data - ignore the validation... it is meaningless\nds_train = BespokeDataset(\n    torch.cat([train_feature_tensor, valid_feature_tensor]), \n    torch.cat([train_target_tensor, valid_target_tensor]), \n    augment=True, augment_coef_mirror = 0.5, augment_coef_flipx = 0.5)\nds_val   = BespokeDataset(valid_feature_tensor, valid_target_tensor, augment=True)\ndls = DataLoaders.from_dsets(ds_train, ds_val, bs=4096, dl_type=BespokeDL, num_workers=0, shuffle=True)\n\nLEARNERS = []\n\nfor i in range(5):\n    model = TabularModel(\n        emb_szs={}, n_cont=len(features) + 15, \n        ps=0.3, out_sz=len(targets), act_cls=Mish(inplace=True),\n        layers=[2048, 2048, 1024, 1024, 64], y_range=(0,1))\n    if torch.cuda.is_available():\n        model = model.cuda()\n\n    learn = Learner(dls, model, loss_func=loss.BCELossFlat())\n\n    # It is still not overfitting...\n    fit = learn.fit(5 if DEBUG else 40, 1e-3)\n    \n    LEARNERS.append(learn)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-10-14T20:57:22.491612Z","iopub.execute_input":"2022-10-14T20:57:22.492045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Submission","metadata":{}},{"cell_type":"code","source":"df_test = read_test()\ngc.collect()\n\nPREDS = []\n\nfor it, learn in enumerate(LEARNERS):\n    print('START LEARNER {}'.format(it))\n    # Usual predict\n    ds_test = BespokeDataset(torch.tensor(df_test[features].to_numpy()), torch.zeros(len(df_test), 2))\n    test_dl = learn.dls.test_dl(ds_test)\n    test_dl.shuffle = False  \n    preds, _ = learn.get_preds(dl=test_dl)\n    PREDS.append(preds.numpy() * 10) # weight usual prediction\n    \n    # TTA - we DON'T use mirror augmentation because it changes the targets order (augment_coef_mirror = 1.0)\n    ds_test_1 = BespokeDataset(torch.tensor(df_test[features].to_numpy()), torch.zeros(len(df_test), 2), \n                             augment = True, augment_coef_mirror = 1.0, augment_coef_flipx = 0.5)\n    test_dl_1 = learn.dls.test_dl(ds_test_1)\n    test_dl_1.shuffle = False  \n    for i in range(5):\n        preds_1, _ = learn.get_preds(dl=test_dl_1)\n        PREDS.append(preds_1.numpy())\n        \n    # TTA2 - we use ONLY mirror augmentation because it changes the targets order (augment_coef_mirror = -1.0)\n    ds_test_2 = BespokeDataset(torch.tensor(df_test[features].to_numpy()), torch.zeros(len(df_test), 2), \n                             augment = True, augment_coef_mirror = -1.0, augment_coef_flipx = 0.5)\n    test_dl_2 = learn.dls.test_dl(ds_test_2)\n    test_dl_2.shuffle = False  \n    for i in range(5):\n        preds_2, _ = learn.get_preds(dl=test_dl_2)\n        PREDS.append(preds_2.numpy()[:, ::-1]) # revert the order back","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-10-13T22:54:15.716024Z","iopub.execute_input":"2022-10-13T22:54:15.716568Z","iopub.status.idle":"2022-10-13T22:54:48.036579Z","shell.execute_reply.started":"2022-10-13T22:54:15.716513Z","shell.execute_reply":"2022-10-13T22:54:48.035465Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/tabular-playground-series-oct-2022/sample_submission.csv')\nsubmission.iloc[:, 1:] = np.array(PREDS).sum(axis = 0) / (20 * len(LEARNERS))\nsubmission.to_csv('model_fastai_multistart_tta.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T22:55:15.325538Z","iopub.execute_input":"2022-10-13T22:55:15.326150Z","iopub.status.idle":"2022-10-13T22:55:19.839238Z","shell.execute_reply.started":"2022-10-13T22:55:15.326107Z","shell.execute_reply":"2022-10-13T22:55:19.838130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ❗️❗️❗️DISCLAIMER: This notebook is a copy of [great notebook from @paddykb](https://www.kaggle.com/code/paddykb/tps-2022-10-fastai) with multistart NN training and test dataset augmentation (TTA). If you like this kernel, you MUST upvote @paddykb's kernel beforehand and don't forget to upvote this one too ❗️❗️❗️","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}