{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-09T05:53:56.078621Z","iopub.execute_input":"2023-01-09T05:53:56.079108Z","iopub.status.idle":"2023-01-09T05:53:56.622458Z","shell.execute_reply.started":"2023-01-09T05:53:56.079013Z","shell.execute_reply":"2023-01-09T05:53:56.621170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Objective\n\n##### To extract certain percentage of the game plays to use them for training neural networks.\n##### How to use YoloV7 and detectron2 for players instance segmentation.\n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pylab as plt\nfrom sklearn.metrics import matthews_corrcoef\n\nSEED = 19951205","metadata":{"execution":{"iopub.status.busy":"2023-01-09T05:54:01.637759Z","iopub.execute_input":"2023-01-09T05:54:01.638193Z","iopub.status.idle":"2023-01-09T05:54:02.254728Z","shell.execute_reply.started":"2023-01-09T05:54:01.638158Z","shell.execute_reply":"2023-01-09T05:54:02.253252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Files","metadata":{}},{"cell_type":"code","source":"# Read in data files\nBASE_DIR = \"../input/nfl-player-contact-detection\"\n\n# Labels and sample submission\nlabels = pd.read_csv(f\"{BASE_DIR}/train_labels.csv\", parse_dates=[\"datetime\"])\n\nss = pd.read_csv(f\"{BASE_DIR}/sample_submission.csv\")\n\n# Player tracking data\ntr_tracking = pd.read_csv(\n    f\"{BASE_DIR}/train_player_tracking.csv\", parse_dates=[\"datetime\"]\n)\nte_tracking = pd.read_csv(\n    f\"{BASE_DIR}/test_player_tracking.csv\", parse_dates=[\"datetime\"]\n)\n\n# Baseline helmet detection labels\ntr_helmets = pd.read_csv(f\"{BASE_DIR}/train_baseline_helmets.csv\")\nte_helmets = pd.read_csv(f\"{BASE_DIR}/test_baseline_helmets.csv\")\n\n# Video metadata with start/stop timestamps\ntr_video_metadata = pd.read_csv(\n    \"../input/nfl-player-contact-detection/train_video_metadata.csv\",\n    parse_dates=[\"start_time\", \"end_time\", \"snap_time\"],\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T05:54:52.425255Z","iopub.execute_input":"2023-01-09T05:54:52.425651Z","iopub.status.idle":"2023-01-09T05:55:41.250179Z","shell.execute_reply.started":"2023-01-09T05:54:52.425619Z","shell.execute_reply":"2023-01-09T05:55:41.248880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Files","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\n\nnp.random.seed(SEED)\n\nkf = GroupKFold()\nkf_dict = {}\n\nfor i, (train_index, test_index) in enumerate(kf.split(tr_video_metadata, None, tr_video_metadata['game_key'])):\n    print(f\"Fold {i}:\")\n    kf_dict[i] = {'train_games': list(tr_video_metadata.iloc[train_index].game_key.unique()),\n                  'val_games': list(tr_video_metadata.iloc[test_index].game_key.unique())}","metadata":{"execution":{"iopub.status.busy":"2023-01-09T05:55:41.252251Z","iopub.execute_input":"2023-01-09T05:55:41.252640Z","iopub.status.idle":"2023-01-09T05:55:41.293428Z","shell.execute_reply.started":"2023-01-09T05:55:41.252605Z","shell.execute_reply":"2023-01-09T05:55:41.290121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Should save the validation data games in order to use them further on, during validation in different strategies.","metadata":{}},{"cell_type":"code","source":"import pickle\n\nwith open('kf_dict', 'wb') as f:\n    pickle.dump(kf_dict, f)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:00:19.537880Z","iopub.execute_input":"2023-01-09T06:00:19.538339Z","iopub.status.idle":"2023-01-09T06:00:19.549220Z","shell.execute_reply.started":"2023-01-09T06:00:19.538306Z","shell.execute_reply":"2023-01-09T06:00:19.547375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('kf_dict', 'rb') as f:\n    kf_dict = pickle.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:00:26.878355Z","iopub.execute_input":"2023-01-09T06:00:26.878819Z","iopub.status.idle":"2023-01-09T06:00:26.885288Z","shell.execute_reply.started":"2023-01-09T06:00:26.878783Z","shell.execute_reply":"2023-01-09T06:00:26.883996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extracting the validation set as a starter.","metadata":{}},{"cell_type":"code","source":"import subprocess, os\n\nval_games = kf_dict[0]['val_games']\n\n!mkdir -p validation \n!chmod 777 validation\n\nfor g in val_games:\n    g_paths = !ls /kaggle/input/nfl-player-contact-detection/train/$g*Sideline*\n    for g_path in g_paths:\n        game_play = g_path.split('/')[-1].split('/')[-1][:-4]\n        !mkdir validation/$game_play && chmod 777 validation/$game_play\n        !ffmpeg -i \"$g_path\" \"validation/$game_play/frame-%04d.jpg\"","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:01:06.115580Z","iopub.execute_input":"2023-01-09T06:01:06.116055Z","iopub.status.idle":"2023-01-09T06:08:11.096592Z","shell.execute_reply.started":"2023-01-09T06:01:06.116018Z","shell.execute_reply":"2023-01-09T06:08:11.094553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extractign 10% of train game_plays","metadata":{}},{"cell_type":"code","source":"train_games = kf_dict[0]['train_games']\n\ntrain_games_plays = tr_video_metadata.query('game_key in @train_games').sample(frac=0.1, replace=False, random_state=SEED).game_play","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:08:19.140741Z","iopub.execute_input":"2023-01-09T06:08:19.141201Z","iopub.status.idle":"2023-01-09T06:08:19.159552Z","shell.execute_reply.started":"2023-01-09T06:08:19.141162Z","shell.execute_reply":"2023-01-09T06:08:19.158029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p train\n!chmod 777 train\n\nfor g in train_games_plays:\n    g_paths = !ls /kaggle/input/nfl-player-contact-detection/train/$g*Sideline*\n    for g_path in g_paths:\n        game_play = g_path.split('/')[-1].split('/')[-1][:-4]\n        !mkdir train/$game_play && chmod 777 train/$game_play\n        !ffmpeg -i \"$g_path\" \"train/$game_play/frame-%04d.jpg\"","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:08:22.660294Z","iopub.execute_input":"2023-01-09T06:08:22.660690Z","iopub.status.idle":"2023-01-09T06:13:51.771779Z","shell.execute_reply.started":"2023-01-09T06:08:22.660655Z","shell.execute_reply":"2023-01-09T06:13:51.770525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Players Instance Segmentation","metadata":{}},{"cell_type":"markdown","source":"### Extract one video to test","metadata":{}},{"cell_type":"code","source":"!mkdir -p frames\n!ffmpeg -i /kaggle/input/nfl-player-contact-detection/train/58168_003392_Sideline.mp4 -q:v 2 -f image2 /kaggle/working/frames/frame_%04d.jpg -hide_banner -loglevel error","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:14:05.142420Z","iopub.execute_input":"2023-01-09T06:14:05.142840Z","iopub.status.idle":"2023-01-09T06:14:12.418538Z","shell.execute_reply.started":"2023-01-09T06:14:05.142806Z","shell.execute_reply":"2023-01-09T06:14:12.416848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html\n\n! git clone -b mask https://github.com/WongKinYiu/yolov7.git\n! pip install pyyaml==5.1\n! pip install 'git+https://github.com/facebookresearch/detectron2.git'\n\n%cd yolov7\n! curl -L https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-mask.pt -o yolov7-mask.pt","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:14:16.335851Z","iopub.execute_input":"2023-01-09T06:14:16.337027Z","iopub.status.idle":"2023-01-09T06:21:46.104913Z","shell.execute_reply.started":"2023-01-09T06:14:16.336934Z","shell.execute_reply":"2023-01-09T06:21:46.103130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import torch\nimport cv2\nimport yaml\nfrom torchvision import transforms\nfrom glob import glob\nfrom utils.datasets import letterbox\nfrom utils.general import non_max_suppression_mask_conf\n\nfrom detectron2.modeling.poolers import ROIPooler\nfrom detectron2.structures import Boxes\nfrom detectron2.utils.memory import retry_if_cuda_oom\nfrom detectron2.layers import paste_masks_in_image","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:21:46.110633Z","iopub.execute_input":"2023-01-09T06:21:46.111135Z","iopub.status.idle":"2023-01-09T06:21:47.637318Z","shell.execute_reply.started":"2023-01-09T06:21:46.111092Z","shell.execute_reply":"2023-01-09T06:21:47.635650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames_paths = glob('/kaggle/working/frames/*')\n\ni = 25\nframe = cv2.imread(frames_paths[i])\nframe = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n\nfig = plt.figure(figsize=(12, 6))\nplt.imshow(frame);","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:21:47.638655Z","iopub.execute_input":"2023-01-09T06:21:47.641055Z","iopub.status.idle":"2023-01-09T06:21:48.350886Z","shell.execute_reply.started":"2023-01-09T06:21:47.641000Z","shell.execute_reply":"2023-01-09T06:21:48.349964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model():\n    model = torch.load('yolov7-mask.pt', map_location=device)['model']\n    # Put in inference mode\n    model.eval()\n\n    if torch.cuda.is_available():\n        # half() turns predictions into float16 tensors\n        # which significantly lowers inference time\n        model.half().to(device)\n    return model\n\ndef run_inference(url):\n    image = cv2.imread(url) # shape: (480, 640, 3)\n    # Resize and pad image\n    image = letterbox(image, 640, stride=64, auto=True)[0] # shape: (480, 640, 3)\n    # Apply transforms\n    image = transforms.ToTensor()(image) # torch.Size([3, 480, 640])\n    # Match tensor type (`torch.FloatTensor` -> `torch.HalfTensor`) with model\n    image = image.half().to(device)\n    # Turn image into batch\n    image = image.unsqueeze(0) # torch.Size([1, 3, 480, 640])\n    output = model(image)\n    return output, image\n\n\ndef plot_results(original_image, pred_img, pred_masks_np, nbboxes, pred_cls, pred_conf, plot_labels=True):\n    for one_mask, bbox, cls, conf in zip(pred_masks_np, nbboxes, pred_cls, pred_conf):\n        if conf < 0.25:\n            continue\n        color = [np.random.randint(255), np.random.randint(255), np.random.randint(255)]\n\n        pred_img = pred_img.copy()\n\n        # Apply mask over image in color\n        pred_img[one_mask] = pred_img[one_mask] * 0.5 + np.array(color, dtype=np.uint8) * 0.5\n        # Draw rectangles around all found objects\n        pred_img = cv2.rectangle(pred_img, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, 2)\n\n        if plot_labels:\n            label = '%s %.3f' % (names[int(cls)], conf)\n            t_size = cv2.getTextSize(label, 0, fontScale=0.1, thickness=1)[0]\n            c2 = bbox[0] + t_size[0], bbox[1] - t_size[1] - 3\n            pred_img = cv2.rectangle(pred_img, (bbox[0], bbox[1]), c2, color, -1, cv2.LINE_AA)\n            pred_img = cv2.putText(pred_img, label, (bbox[0], bbox[1] - 2), 0, 0.5, [255, 255, 255], thickness=1, lineType=cv2.LINE_AA)  \n\n    fig, ax = plt.subplots(1, 2, figsize=(pred_img.shape[0]/10, pred_img.shape[1]/10), dpi=150)\n\n    original_image = np.moveaxis(image.cpu().numpy().squeeze(), 0, 2).astype('float32')\n    original_image = cv2.cvtColor(original_image, cv2.COLOR_RGB2BGR)\n\n    ax[0].imshow(original_image)\n    ax[0].axis(\"off\")\n    ax[1].imshow(pred_img)\n    ax[1].axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:29:33.909356Z","iopub.execute_input":"2023-01-09T06:29:33.909858Z","iopub.status.idle":"2023-01-09T06:29:33.928238Z","shell.execute_reply.started":"2023-01-09T06:29:33.909819Z","shell.execute_reply":"2023-01-09T06:29:33.926879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('data/hyp.scratch.mask.yaml') as f:\n    hyp = yaml.load(f, Loader=yaml.FullLoader)\n    \ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = load_model()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:29:35.919812Z","iopub.execute_input":"2023-01-09T06:29:35.920266Z","iopub.status.idle":"2023-01-09T06:29:36.065375Z","shell.execute_reply.started":"2023-01-09T06:29:35.920231Z","shell.execute_reply":"2023-01-09T06:29:36.064196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output, image = run_inference()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:29:36.863344Z","iopub.execute_input":"2023-01-09T06:29:36.863817Z","iopub.status.idle":"2023-01-09T06:29:36.888981Z","shell.execute_reply.started":"2023-01-09T06:29:36.863779Z","shell.execute_reply":"2023-01-09T06:29:36.886799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.keys()","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:29:38.733203Z","iopub.execute_input":"2023-01-09T06:29:38.734131Z","iopub.status.idle":"2023-01-09T06:29:38.754971Z","shell.execute_reply.started":"2023-01-09T06:29:38.734092Z","shell.execute_reply":"2023-01-09T06:29:38.753550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inf_out = output['test']\nattn = output['attn']\nbases = output['bases']\nsem_output = output['sem']\n\nbases = torch.cat([bases, sem_output], dim=1)\nnb, _, height, width = image.shape\nnames = model.names\npooler_scale = model.pooler_scale\n\npooler = ROIPooler(output_size=hyp['mask_resolution'], \n                   scales=(pooler_scale,), \n                   sampling_ratio=1, \n                   pooler_type='ROIAlignV2', \n                   canonical_level=2)\n                   \n# output, output_mask, output_mask_score, output_ac, output_ab\noutput, output_mask, _, _, _ = non_max_suppression_mask_conf(inf_out, \n                                                             attn, \n                                                             bases, \n                                                             pooler, \n                                                             hyp, \n                                                             conf_thres=0.25, \n                                                             iou_thres=0.65, \n                                                             merge=False, \n                                                             mask_iou=None)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:26:15.531864Z","iopub.execute_input":"2023-01-09T06:26:15.532320Z","iopub.status.idle":"2023-01-09T06:26:15.566592Z","shell.execute_reply.started":"2023-01-09T06:26:15.532283Z","shell.execute_reply":"2023-01-09T06:26:15.564351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output[0].shape \noutput_mask[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-01-09T06:26:16.719554Z","iopub.execute_input":"2023-01-09T06:26:16.720221Z","iopub.status.idle":"2023-01-09T06:26:16.750420Z","shell.execute_reply.started":"2023-01-09T06:26:16.720178Z","shell.execute_reply":"2023-01-09T06:26:16.749047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred, pred_masks = output[0], output_mask[0]\nbase = bases[0]\nbboxes = Boxes(pred[:, :4])\n\noriginal_pred_masks = pred_masks.view(-1, \n                                      hyp['mask_resolution'], \n                                      hyp['mask_resolution'])\n\npred_masks = retry_if_cuda_oom(paste_masks_in_image)(original_pred_masks, \n                                                     bboxes, \n                                                     (height, width), \n                                                     threshold=0.5)\n                                                     \n# Detach Tensors from the device, send to the CPU and turn into NumPy arrays\npred_masks_np = pred_masks.detach().cpu().numpy()\npred_cls = pred[:, 5].detach().cpu().numpy()\npred_conf = pred[:, 4].detach().cpu().numpy()\nnimg = image[0].permute(1, 2, 0) * 255\nnimg = nimg.cpu().numpy().astype(np.uint8)\nnimg = cv2.cvtColor(nimg, cv2.COLOR_RGB2BGR)\nnbboxes = bboxes.tensor.detach().cpu().numpy().astype(np.int32)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_pred_masks.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nplot_results(image, nimg, pred_masks_np, nbboxes, pred_cls, pred_conf, plot_labels=False)","metadata":{},"execution_count":null,"outputs":[]}]}