{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Imports\nimport math\nimport pandas as pd\nimport numpy as np\nimport sys\nimport timm\nimport gc\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn import preprocessing\nimport torch\nfrom torch import nn\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom tqdm import tqdm\nimport glob\nimport random\nfrom torch.utils.data import Dataset, DataLoader\nimport cv2\nimport os\nimport matplotlib.pyplot as plt\nimport torch.optim as optim\nfrom sklearn.model_selection import train_test_split\nimport time\nfrom sklearn.metrics import matthews_corrcoef\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-17T16:34:35.728227Z","iopub.execute_input":"2023-07-17T16:34:35.728635Z","iopub.status.idle":"2023-07-17T16:34:35.738308Z","shell.execute_reply.started":"2023-07-17T16:34:35.728600Z","shell.execute_reply":"2023-07-17T16:34:35.737150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Imports\n#!pip install torcheval\n#import torcheval\n#!pip install ffmpeg-python\n\nimport pandas as pd\nimport numpy as np\nimport sys\nimport gc\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn import preprocessing\n\nfrom torch.autograd import Variable \nfrom tqdm import tqdm\nimport glob\nimport random\nimport cv2\nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport time\n#import torcheval\nimport timm\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\n\n#from torcheval.metrics import BinaryAccuracy\n\nfrom torch import nn\nfrom sklearn.metrics import matthews_corrcoef\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.compose import ColumnTransformer\n\nfrom timm.scheduler import CosineLRScheduler\n","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:34:36.022429Z","iopub.execute_input":"2023-07-17T16:34:36.023420Z","iopub.status.idle":"2023-07-17T16:34:36.061290Z","shell.execute_reply.started":"2023-07-17T16:34:36.023380Z","shell.execute_reply":"2023-07-17T16:34:36.060233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:34:38.793761Z","iopub.execute_input":"2023-07-17T16:34:38.794201Z","iopub.status.idle":"2023-07-17T16:34:38.898583Z","shell.execute_reply.started":"2023-07-17T16:34:38.794163Z","shell.execute_reply":"2023-07-17T16:34:38.897238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder = \"/kaggle/input/nfl-player-contact-detection/\"","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:34:39.122819Z","iopub.execute_input":"2023-07-17T16:34:39.123216Z","iopub.status.idle":"2023-07-17T16:34:39.128859Z","shell.execute_reply.started":"2023-07-17T16:34:39.123179Z","shell.execute_reply":"2023-07-17T16:34:39.127115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expand_contact_id(df):\n    \"\"\"\n    Splits out contact_id into seperate columns.\n    \"\"\"\n    df[\"game_play\"] = df[\"contact_id\"].str[:12]\n    df[\"step\"] = df[\"contact_id\"].str.split(\"_\").str[-3].astype(\"int\")\n    df[\"nfl_player_id_1\"] = df[\"contact_id\"].str.split(\"_\").str[-2]\n    df[\"nfl_player_id_2\"] = df[\"contact_id\"].str.split(\"_\").str[-1]\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:34:40.265056Z","iopub.execute_input":"2023-07-17T16:34:40.265849Z","iopub.status.idle":"2023-07-17T16:34:40.273365Z","shell.execute_reply.started":"2023-07-17T16:34:40.265810Z","shell.execute_reply":"2023-07-17T16:34:40.271948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels = expand_contact_id(pd.read_csv(folder+\"/sample_submission.csv\")[[\"contact_id\",\"contact\"]])\ntest_tracking = pd.read_csv(folder+\"/test_player_tracking.csv\")\ntest_helmets = pd.read_csv(folder+\"/test_baseline_helmets.csv\")\ntest_video_metadata = pd.read_csv(folder+\"/test_video_metadata.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:34:45.366635Z","iopub.execute_input":"2023-07-17T16:34:45.367035Z","iopub.status.idle":"2023-07-17T16:34:46.115538Z","shell.execute_reply.started":"2023-07-17T16:34:45.366999Z","shell.execute_reply":"2023-07-17T16:34:46.114372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /kaggle/test/frames\n\nfor video in tqdm(test_helmets.video.unique()):\n    if 'Endzone2' not in video:\n        !ffmpeg -i /kaggle/input/nfl-player-contact-detection/test/{video} -q:v 2 -f image2 /kaggle/test/frames/{video}_%04d.jpg -hide_banner -loglevel error","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:34:46.617401Z","iopub.execute_input":"2023-07-17T16:34:46.618236Z","iopub.status.idle":"2023-07-17T16:35:42.266524Z","shell.execute_reply.started":"2023-07-17T16:34:46.618196Z","shell.execute_reply":"2023-07-17T16:35:42.265100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def FrameCapture(path, video):\n#        vid = cv2.VideoCapture(path + video)\n#        frame = 0\n#        success = 1\n#        if vid.isOpened():\n#            print(path + video)\n#            success, image = vid.read()\n#            while success: \n#                \n#                cv2.imwrite(\"/kaggle/working/frames/%s_%04d.jpg\" % (video, frame), image, [int(cv2.IMWRITE_JPEG_QUALITY), 10])\n#                frame += 1\n#                success, image = vid.read()\n#                \n#            vid.release()\n        ","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:42.269766Z","iopub.execute_input":"2023-07-17T16:35:42.270201Z","iopub.status.idle":"2023-07-17T16:35:42.281411Z","shell.execute_reply.started":"2023-07-17T16:35:42.270154Z","shell.execute_reply":"2023-07-17T16:35:42.280270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!mkdir -p ../working/frames\n#\n#\n#for video in tqdm(test_helmets.video.unique()):\n#     if 'Endzone2' not in video and 'All29' not in video:\n#         FrameCapture(\"/kaggle/input/nfl-player-contact-detection/test/\", video)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:42.283475Z","iopub.execute_input":"2023-07-17T16:35:42.285179Z","iopub.status.idle":"2023-07-17T16:35:42.463370Z","shell.execute_reply.started":"2023-07-17T16:35:42.285117Z","shell.execute_reply":"2023-07-17T16:35:42.462128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"video2helmets = {}\ntest_helmets_new = test_helmets.set_index('video')\nfor video in tqdm(test_helmets.video.unique()):\n    video2helmets[video] = test_helmets_new.loc[video].reset_index(drop=True)\n    \ndel test_helmets, test_helmets_new\ngc.collect()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:42.467372Z","iopub.execute_input":"2023-07-17T16:35:42.468304Z","iopub.status.idle":"2023-07-17T16:35:42.751508Z","shell.execute_reply.started":"2023-07-17T16:35:42.468263Z","shell.execute_reply":"2023-07-17T16:35:42.750335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"video2frames = {}\n\nfor game_play in tqdm(test_video_metadata.game_play.unique()):\n    for view in ['Endzone', 'Sideline']:\n        video = game_play + f'_{view}.mp4'\n        video2frames[video] = max(list(map(lambda x:int(x.split('_')[-1].split('.')[0]), \\\n                                           glob.glob(f\"/kaggle/test/frames/{video}*\"))))","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:42.753410Z","iopub.execute_input":"2023-07-17T16:35:42.754186Z","iopub.status.idle":"2023-07-17T16:35:42.836824Z","shell.execute_reply.started":"2023-07-17T16:35:42.754147Z","shell.execute_reply":"2023-07-17T16:35:42.835705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef create_features(df, tr_tracking, merge_col=\"step\", use_cols=[\"x_position\", \"y_position\"]):\n    output_cols = []\n    df_combo = (\n        df.astype({\"nfl_player_id_1\": \"str\"})\n        .merge(\n            tr_tracking.astype({\"nfl_player_id\": \"str\"})[\n                [\"game_play\", merge_col, \"nfl_player_id\",] + use_cols\n            ],\n            left_on=[\"game_play\", merge_col, \"nfl_player_id_1\"],\n            right_on=[\"game_play\", merge_col, \"nfl_player_id\"],\n            how=\"left\",\n        )\n        .rename(columns={c: c+\"_1\" for c in use_cols})\n        .drop(\"nfl_player_id\", axis=1)\n        .merge(\n            tr_tracking.astype({\"nfl_player_id\": \"str\"})[\n                [\"game_play\", merge_col, \"nfl_player_id\"] + use_cols\n            ],\n            left_on=[\"game_play\", merge_col, \"nfl_player_id_2\"],\n            right_on=[\"game_play\", merge_col, \"nfl_player_id\"],\n            how=\"left\",\n        )\n        .drop(\"nfl_player_id\", axis=1)\n        .rename(columns={c: c+\"_2\" for c in use_cols})\n        .sort_values([\"game_play\", merge_col, \"nfl_player_id_1\", \"nfl_player_id_2\"])\n        .reset_index(drop=True)\n    )\n    output_cols += [c+\"_1\" for c in use_cols]\n    output_cols += [c+\"_2\" for c in use_cols]\n    \n    if (\"x_position\" in use_cols) & (\"y_position\" in use_cols):\n        index = df_combo['x_position_2'].notnull()\n        \n        distance_arr = np.full(len(index), np.nan)\n        tmp_distance_arr = np.sqrt(\n            np.square(df_combo.loc[index, \"x_position_1\"] - df_combo.loc[index, \"x_position_2\"])\n            + np.square(df_combo.loc[index, \"y_position_1\"]- df_combo.loc[index, \"y_position_2\"])\n        )\n        \n        distance_arr[index] = tmp_distance_arr\n        df_combo['distance'] = distance_arr\n        output_cols += [\"distance\"]\n        \n    df_combo['G_flug'] = (df_combo['nfl_player_id_2']==\"G\")\n    output_cols += [\"G_flug\"]\n    return df_combo, output_cols\n\nuse_cols = [\n    'x_position', 'y_position', 'speed', 'distance',\n    'direction', 'orientation', 'acceleration', 'sa'\n]\n\n\ntest, feature_cols = create_features(test_labels, test_tracking, use_cols=use_cols)\ntest_filtered = test.query('not distance > 2.0').reset_index(drop=True)\ntest_filtered['frame'] = (test_filtered['step']/10*59.94+5*59.94).astype('int')+1","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:42.838766Z","iopub.execute_input":"2023-07-17T16:35:42.839542Z","iopub.status.idle":"2023-07-17T16:35:43.209692Z","shell.execute_reply.started":"2023-07-17T16:35:42.839490Z","shell.execute_reply":"2023-07-17T16:35:43.208532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test, test_labels, test_tracking\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:43.211820Z","iopub.execute_input":"2023-07-17T16:35:43.212595Z","iopub.status.idle":"2023-07-17T16:35:43.439340Z","shell.execute_reply.started":"2023-07-17T16:35:43.212554Z","shell.execute_reply":"2023-07-17T16:35:43.438136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_aug = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    #A.RandomBrightnessContrast(brightness_limit=(-0.05, 0.05), contrast_limit=(-0.05, 0.05), p=0.3),\n    #A.RandomGamma(p=0.5),\n    A.Normalize(mean=[0.], std=[1.]),\n    ToTensorV2()])\n\nvalid_aug = A.Compose([\n    A.Normalize(mean=[0.], std=[1.]),\n    ToTensorV2()\n])","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:43.442906Z","iopub.execute_input":"2023-07-17T16:35:43.443910Z","iopub.status.idle":"2023-07-17T16:35:43.451497Z","shell.execute_reply.started":"2023-07-17T16:35:43.443864Z","shell.execute_reply":"2023-07-17T16:35:43.450328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaled_features = test_filtered.copy()\n\nfeatures = scaled_features[feature_cols]\nscaler = MinMaxScaler().fit(features.values)\nfeatures = scaler.transform(features.values)\nscaled_features[feature_cols] = features\ntest_filtered = scaled_features\n\ndel scaled_features\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:35:43.452922Z","iopub.execute_input":"2023-07-17T16:35:43.453649Z","iopub.status.idle":"2023-07-17T16:35:43.656832Z","shell.execute_reply.started":"2023-07-17T16:35:43.453609Z","shell.execute_reply":"2023-07-17T16:35:43.655586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Want to output imgs, feature, label\n#imgs.shape = 5x6x256x256\n#feature.shape = \n#label.shape = \n\n\nclass MyDataset_LSTM(Dataset):\n    def __init__(self, df, aug=train_aug):\n        self.df = df\n        self.frame = df.frame.values\n        self.feature = df[feature_cols].fillna(-1).values\n        self.players = df[['nfl_player_id_1','nfl_player_id_2']].values\n        self.game_play = df.game_play.values\n        self.aug = aug\n        self.contact_id = df.contact_id.values\n        \n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):  \n        time_size = 12\n        sequence = 5    #number of frames in lstm sequence\n        frame = self.frame[idx]\n        frame_idx = frame\n        \n        \n        if self.aug == train_aug:\n            frame = frame + random.randint(-6, 6)\n        frame_diff = frame - frame_idx\n        players = []\n        \n        #??\n        for p in self.players[idx]:\n            if p == 'G':\n                players.append(p)\n            else:\n                players.append(int(p))\n   \n       \n        imgs_arr = []\n        \n        for f in range(frame-((sequence//2)*time_size), frame+((sequence//2)*time_size)+1, time_size):\n            imgs = []\n\n            for view in ['Endzone', 'Sideline']:\n\n                video = self.game_play[idx] + f'_{view}.mp4'\n                \n                tmp = video2helmets[video]           \n                tmp = tmp[tmp.nfl_player_id.isin(players)]#.sort_values(['nfl_player_id', 'frame'])         \n                tmp = tmp[tmp['frame'] == f]\n                \n                bboxes = []\n                if len(tmp['left']):   \n                    for i in range(len(tmp.index)):\n                        \n                        x = tmp['left'].tolist()[i]\n                        w = tmp['width'].tolist()[i]\n                        y = tmp['top'].tolist()[i]\n                        h = tmp['height'].tolist()[i]\n                        if math.isfinite(x):\n                            bboxes.append([x, w, y, h])\n                        \n                            \n                        \n                        \n                    img_helmet = np.zeros((720,1280), dtype=np.float32)  \n                    if len(tmp) == 2:\n                        \n                        \n                        #df_dist = self.df[(self.df['game_play'] == self.game_play) &\n                        #                   (self.df['nfl_player_id_1'] == str(players[0]))  & (self.df['nfl_player_id_2'] == str(players[1]))]\n                        #closest_frame = min(df_dist.frame.values, key=lambda x:abs(x-(f-frame_diff)))\n                        dist = self.df.distance[idx]\n\n                        if math.isfinite(dist):\n                            color_dist = int(156+(100-(100*dist)))\n                            \n                        else:\n                            color_dist = 100\n                           \n                        color = (color_dist, 0, 0)                        \n                        \n                    else:\n                        color = (100, 0, 0)\n                    \n                    \n                    for i in range(len(tmp)):\n                        \n                        start_point = (bboxes[i][0], bboxes[i][2])\n                        end_point = (bboxes[i][0] +  bboxes[i][1], bboxes[i][2] + bboxes[i][3])\n                        \n\n                        img_helmet = cv2.rectangle(img_helmet, start_point, end_point, color, thickness =-1)\n                    \n                    \n                    img_new = np.zeros((256, 256), dtype=np.float32)         \n                    img = cv2.imread(f\"/kaggle/test/frames/{video}_{f:04d}.jpg\", 0)\n\n                    if len(tmp) == 2:  \n                        x = ((bboxes[0][0] + bboxes[1][0])/2) + ((bboxes[0][1] + bboxes[1][1])/4)\n                        y = ((bboxes[0][2] + bboxes[1][2])/2) + ((bboxes[0][3] + bboxes[1][3])/4)   \n                    else: \n                        x = bboxes[0][0] + (bboxes[0][1]/2)\n                        y = bboxes[0][2] + (bboxes[0][3]/2)\n                              \n                              \n                              \n                              \n                    img = img[int(y)-128:int(y)+128,int(x)-128:int(x)+128].copy()\n                    img_helmet = img_helmet[int(y)-128:int(y)+128,int(x)-128:int(x)+128].copy()\n                    img_new[:img.shape[0], :img.shape[1]] = img    \n  \n                    img_h = np.zeros((256, 256), dtype=np.float32)         \n                    img_h[:img_helmet.shape[0], :img_helmet.shape[1]] = img_helmet               \n                             \n                else:\n                    img_h = np.zeros((256,256), dtype=np.float32) \n                    img_new = np.zeros((256, 256), dtype=np.float32)   \n\n\n                imgs.append(img_new)\n                imgs.append(img_h)\n                \n            imgs_arr.append(imgs)\n       \n                \n        feature = np.float32(self.feature[idx])\n        row=len(imgs_arr)\n        column=len(imgs_arr[0])\n   \n \n        imga = np.array(imgs_arr)\n\n        b, c, h, w = imga.shape\n        imga = imga.reshape(b//b, c*sequence, h, w)\n        imga = np.squeeze(imga)\n        imga = imga.transpose(1,2,0)\n    \n        imga = self.aug(image=imga)[\"image\"]\n        label = np.float32(self.df.contact.values[idx])\n\n        \n        return imga, feature, label\n    \n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:15:07.937182Z","iopub.execute_input":"2023-06-15T13:15:07.938018Z","iopub.status.idle":"2023-06-15T13:15:07.972972Z","shell.execute_reply.started":"2023-06-15T13:15:07.937962Z","shell.execute_reply":"2023-06-15T13:15:07.971693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass MyDataset_LSTM(Dataset):\n    def __init__(self, df, aug=valid_aug):\n        self.df = df\n        self.frame = df.frame.values\n        self.feature = df[feature_cols].fillna(-1).values\n        self.players = df[['nfl_player_id_1','nfl_player_id_2']].values\n        self.game_play = df.game_play.values\n        self.aug = aug\n        self.contact_id = df.contact_id.values\n        \n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):  \n        time_size = 12\n        sequence = 5    #number of frames in lstm sequence\n        frame = self.frame[idx]\n        frame_idx = frame\n        \n        \n        if self.aug == train_aug:\n            print('hej')\n            frame = frame + random.randint(-6, 6)\n        frame_diff = frame - frame_idx\n        players = []\n        \n        #??\n        for p in self.players[idx]:\n            if p == 'G':\n                players.append(p)\n            else:\n                players.append(int(p))\n   \n       \n        imgs_arr = []\n        \n        for f in range(frame-((sequence//2)*time_size), frame+((sequence//2)*time_size)+1, time_size):\n            imgs = []\n\n            for view in ['Endzone', 'Sideline']:\n\n                video = self.game_play[idx] + f'_{view}.mp4'\n                \n                tmp = video2helmets[video]           \n                tmp = tmp[tmp.nfl_player_id.isin(players)]#.sort_values(['nfl_player_id', 'frame'])         \n                tmp = tmp[tmp['frame'] == f]\n                \n                bboxes = []\n                img_size = 0\n                if len(tmp['left']):\n                  \n                    for i in range(len(tmp.index)):\n                        \n                        x = tmp['left'].tolist()[i]\n                        w = tmp['width'].tolist()[i]\n                        y = tmp['top'].tolist()[i]\n                        h = tmp['height'].tolist()[i]\n                        if math.isfinite(x):\n                            bboxes.append([x, w, y, h])\n                            img_size = max([img_size, w, h])\n                            \n                        \n                        \n                    img_helmet = np.zeros((720,1280), dtype=np.float32)  \n                    if len(tmp) == 2:\n                        \n                        \n                        #df_dist = self.df[(self.df['game_play'] == self.game_play) &\n                        #                   (self.df['nfl_player_id_1'] == str(players[0]))  & (self.df['nfl_player_id_2'] == str(players[1]))]\n                        #closest_frame = min(df_dist.frame.values, key=lambda x:abs(x-(f-frame_diff)))\n                        dist = self.df.distance[idx]\n\n                        if math.isfinite(dist):\n                            color_dist = int(156+(100-(100*dist)))\n                            \n                        else:\n                            color_dist = 100\n                           \n                        color = (color_dist, 0, 0)                        \n                        \n                    else:\n                        color = (100, 0, 0)\n                    \n                    \n                    for i in range(len(tmp)):\n                        \n                        start_point = (bboxes[i][0], bboxes[i][2])\n                        end_point = (bboxes[i][0] +  bboxes[i][1], bboxes[i][2] + bboxes[i][3])\n                        \n\n                        img_helmet = cv2.rectangle(img_helmet, start_point, end_point, color, thickness =-1)\n                    \n                    \n                    img_new = np.zeros((256, 256), dtype=np.float32)         \n                    img = cv2.imread(f\"/kaggle/test/frames/{video}_{f:04d}.jpg\", 0)\n                    \n                    if len(tmp) == 2:  \n                        x = ((bboxes[0][0] + bboxes[1][0])/2) + ((bboxes[0][1] + bboxes[1][1])/4)\n                        y = ((bboxes[0][2] + bboxes[1][2])/2) + ((bboxes[0][3] + bboxes[1][3])/4)   \n                    else: \n                        x = bboxes[0][0] + (bboxes[0][1]/2)\n                        y = bboxes[0][2] + (bboxes[0][3]/2)\n                   \n                    if x < img_size*4:\n                            x = img_size * 4\n                    if y < img_size * 4:\n                            y = img_size * 4\n                 \n                    if img.size == 0:      \n                        img_h = np.zeros((256,256), dtype=np.float32) \n                        img_new = np.zeros((256, 256), dtype=np.float32) \n                        \n                    else:\n                        \n                        \n                        img = img[int(y)-img_size*4:int(y)+img_size*4,int(x)-img_size*4:int(x)+img_size*4].copy()\n                       \n                        \n                        img = cv2.resize(img, dsize=(256, 256), interpolation=cv2.INTER_LINEAR)\n                        \n                        img_new[:img.shape[0], :img.shape[1]] = img \n                        \n                        #img = img[int(y)-128:int(y)+128,int(x)-128:int(x)+128].copy()\n                        img_helmet = img_helmet[int(y)-img_size*4:int(y)+img_size*4,int(x)-img_size*4:int(x)+img_size*4].copy()\n                        img_helmet = cv2.resize(img_helmet, dsize=(256, 256), interpolation=cv2.INTER_CUBIC)\n\n                      \n                        \n\n\n\n                        img_h = np.zeros((256, 256), dtype=np.float32)         \n                        img_h[:img_helmet.shape[0], :img_helmet.shape[1]] = img_helmet               \n                             \n                else:\n                    \n                    img_h = np.zeros((256,256), dtype=np.float32) \n                    img_new = np.zeros((256, 256), dtype=np.float32)   \n\n\n                imgs.append(img_new)\n                imgs.append(img_h)\n                #plt.imshow(imgs[1])\n                \n                \n                #plt.show()\n                \n                \n            imgs_arr.append(imgs)\n       \n                \n        feature = np.float32(self.feature[idx])\n         \n   \n \n        imga = np.array(imgs_arr)\n\n        b, c, h, w = imga.shape\n        imga = imga.reshape(b//b, c*sequence, h, w)\n        imga = np.squeeze(imga)\n        imga = imga.transpose(1,2,0)\n    \n        imga = self.aug(image=imga)[\"image\"]\n        label = np.float32(self.df.contact.values[idx])\n\n        \n        return imga, feature, label\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:43:16.636685Z","iopub.execute_input":"2023-07-17T16:43:16.637098Z","iopub.status.idle":"2023-07-17T16:43:16.678606Z","shell.execute_reply.started":"2023-07-17T16:43:16.637038Z","shell.execute_reply":"2023-07-17T16:43:16.677478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, feature, label = MyDataset_LSTM(test_filtered, valid_aug)[2000]\nplt.imshow(img.permute(1,2,0)[:,:,0])\nplt.rcParams[\"figure.figsize\"] = 3,3\nplt.savefig('nfl_18.png')\n\nplt.show()\nimg.shape, feature, label\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:43:17.027498Z","iopub.execute_input":"2023-07-17T16:43:17.027867Z","iopub.status.idle":"2023-07-17T16:43:17.388188Z","shell.execute_reply.started":"2023-07-17T16:43:17.027834Z","shell.execute_reply":"2023-07-17T16:43:17.387063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = MyDataset_LSTM(test_filtered, valid_aug)\ntest_loader = DataLoader(test_set, batch_size=16, shuffle=False, num_workers=2, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:43:17.390410Z","iopub.execute_input":"2023-07-17T16:43:17.391344Z","iopub.status.idle":"2023-07-17T16:43:17.401794Z","shell.execute_reply.started":"2023-07-17T16:43:17.391295Z","shell.execute_reply":"2023-07-17T16:43:17.400201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, num_layers = 1, input_size = 256, hidden_size = 64, seq_length = 5):\n        super(Model, self).__init__()\n        self.num_layers = num_layers #number of layers\n        self.input_size = input_size #input size\n        self.hidden_size = hidden_size #hidden state\n        self.seq_length = seq_length #sequence length\n        \n        #efficientnet_b1\n        self.backbone = timm.create_model('resnet50', pretrained=False, num_classes=128, in_chans=2)\n        \n                \n        self.mlp = nn.Sequential(\n            nn.Linear(18, 32),\n            nn.LayerNorm(32),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n         \n        )\n\n        \n        self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,\n                          num_layers=num_layers, batch_first=True) \n        self.fc_lstm = nn.Linear(hidden_size, 128) \n        \n        self.softmax = nn.Softmax()\n        self.fc = nn.Linear(128+32, 1)\n\n    def forward(self, img, x):\n        \n        b, c, h, w = img.shape   \n        img = img.reshape(b*(c//2),c//(c//2), h, w)\n        img = self.backbone(img)\n        img = img.reshape(b,self.seq_length, -1)\n\n\n        h_0 = Variable(torch.zeros(self.num_layers, img.size(0), self.hidden_size)).to(device) #hidden state\n        c_0 = Variable(torch.zeros(self.num_layers, img.size(0), self.hidden_size)).to(device) #internal state\n        # Propagate input through LSTM\n        #self.lstm.flatten_parameters()\n\n\n        output, (hn, cn) = self.lstm(img, (h_0, c_0)) #lstm with input, hidden, and internal state\n        hn = hn.view(-1, self.hidden_size) #reshaping the data for Dense layer next\n        out = self.softmax(hn)\n        out = self.fc_lstm(out)\n        #out = self.softmax(out)\n    \n        feature = self.mlp(x)\n\n        #feature = torch.transpose(feature, 0, 1)\n        y = self.fc(torch.cat([out, feature], dim=1))\n        return y","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:43:17.505543Z","iopub.execute_input":"2023-07-17T16:43:17.505847Z","iopub.status.idle":"2023-07-17T16:43:17.523122Z","shell.execute_reply.started":"2023-07-17T16:43:17.505818Z","shell.execute_reply":"2023-07-17T16:43:17.521780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel = Model()\nmodel= nn.DataParallel(model)\nmodel.to(device)\nmodel.load_state_dict(torch.load('/kaggle/input/nfl-cnn-model/nfl_model (3).pytorch'))\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:43:18.413461Z","iopub.execute_input":"2023-07-17T16:43:18.413948Z","iopub.status.idle":"2023-07-17T16:43:19.904309Z","shell.execute_reply.started":"2023-07-17T16:43:18.413899Z","shell.execute_reply":"2023-07-17T16:43:19.903143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = []\nwith torch.no_grad():\n    \n        for batch,(img, feature, label) in tqdm(enumerate(test_loader),total = len(test_loader)):\n            img = img.to(device)\n            feature = feature.to(device)\n            label = label.to(device)\n            output = model(img, feature).squeeze(-1)\n\n            y_pred.extend(output.sigmoid().cpu().numpy())\n\ny_pred = np.array(y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:43:19.906864Z","iopub.execute_input":"2023-07-17T16:43:19.907268Z","iopub.status.idle":"2023-07-17T16:51:42.474646Z","shell.execute_reply.started":"2023-07-17T16:43:19.907225Z","shell.execute_reply":"2023-07-17T16:51:42.472497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"th = 0.5\n\ntest_filtered['contact'] = (y_pred >= th).astype('int')\n\nsub = pd.read_csv('/kaggle/input/nfl-player-contact-detection/sample_submission.csv')\n\nsub = sub.drop(\"contact\", axis=1).merge(test_filtered[['contact_id', 'contact']], how='left', on='contact_id')\nsub['contact'] = sub['contact'].fillna(0).astype('int')\n\nsub[[\"contact_id\", \"contact\"]].to_csv(\"/kaggle/working/submission.csv\", index=False)\n\nsub.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-07-17T16:52:19.788577Z","iopub.execute_input":"2023-07-17T16:52:19.788995Z","iopub.status.idle":"2023-07-17T16:52:19.999183Z","shell.execute_reply.started":"2023-07-17T16:52:19.788960Z","shell.execute_reply":"2023-07-17T16:52:19.998074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Testa th = 0.5, 0,4, 0,45","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:34:43.857013Z","iopub.execute_input":"2023-06-15T13:34:43.857722Z","iopub.status.idle":"2023-06-15T13:34:43.862374Z","shell.execute_reply.started":"2023-06-15T13:34:43.857677Z","shell.execute_reply":"2023-06-15T13:34:43.861179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = y_pred > 0.4","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:34:43.864063Z","iopub.execute_input":"2023-06-15T13:34:43.864448Z","iopub.status.idle":"2023-06-15T13:34:43.876998Z","shell.execute_reply.started":"2023-06-15T13:34:43.864407Z","shell.execute_reply":"2023-06-15T13:34:43.875964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(x)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:37:06.947415Z","iopub.execute_input":"2023-06-15T13:37:06.948107Z","iopub.status.idle":"2023-06-15T13:37:06.987391Z","shell.execute_reply.started":"2023-06-15T13:37:06.948059Z","shell.execute_reply":"2023-06-15T13:37:06.981146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:37:12.585143Z","iopub.execute_input":"2023-06-15T13:37:12.585974Z","iopub.status.idle":"2023-06-15T13:37:12.593637Z","shell.execute_reply.started":"2023-06-15T13:37:12.585918Z","shell.execute_reply":"2023-06-15T13:37:12.592521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}