{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\nIn this notebook, a set of the helmet data is mapped wih a frame of a video. <br>\nI picked `58168_003392_Endzone.mp4` as that is the first video that appeared in the helmet dataset.","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport gc\n\nimport cv2\nfrom PIL import Image, ImageDraw\nfrom glob import glob\nfrom base64 import b64encode\nfrom IPython.display import HTML\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torchvision import transforms","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-05T07:17:45.624874Z","iopub.execute_input":"2023-01-05T07:17:45.625314Z","iopub.status.idle":"2023-01-05T07:17:45.633177Z","shell.execute_reply.started":"2023-01-05T07:17:45.625277Z","shell.execute_reply":"2023-01-05T07:17:45.631638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = \"kaggle\"","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.635633Z","iopub.execute_input":"2023-01-05T07:17:45.636807Z","iopub.status.idle":"2023-01-05T07:17:45.645811Z","shell.execute_reply.started":"2023-01-05T07:17:45.636756Z","shell.execute_reply":"2023-01-05T07:17:45.644325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Constant Variables\nif env == \"kaggle\":\n    DATA_DIR = \"/kaggle/input/nfl-player-contact-detection/\"\nelse:\n    # For Google Colab\n    DATA_DIR = \"\"\n\nTRAIN_HELMET = os.path.join(DATA_DIR,\"test_baseline_helmets.csv\")\nSUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.647026Z","iopub.execute_input":"2023-01-05T07:17:45.648049Z","iopub.status.idle":"2023-01-05T07:17:45.660216Z","shell.execute_reply.started":"2023-01-05T07:17:45.648011Z","shell.execute_reply":"2023-01-05T07:17:45.658688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pick up a video from the train folder\nsample_video = \"58168_003392_Endzone.mp4\"\nSAMPLE = os.path.join(DATA_DIR, \"train/\"+sample_video)\nprint(SAMPLE)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.662935Z","iopub.execute_input":"2023-01-05T07:17:45.663343Z","iopub.status.idle":"2023-01-05T07:17:45.673509Z","shell.execute_reply.started":"2023-01-05T07:17:45.663308Z","shell.execute_reply":"2023-01-05T07:17:45.672203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n# Helper Functions\n####################################################################################\n\n# Play a video\n# https://www.kaggle.com/code/asimple/getting-started-eda-player-contact-detection\ndef play(filename: str):\n    video = open(filename,'rb').read()\n    src = f'data:video/mp4;base64,{b64encode(video).decode()}'\n    html = f'<video width=500 controls autoplay loop><source src=\"{src}\" type=\"video/mp4\"></video>'\n\n    return HTML(html)\n\n# Calculate the file size in KB, MB, GB\ndef convert_bytes(size):\n    \"\"\" Convert bytes to KB, or MB or GB\"\"\"\n    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:\n        if size < 1024.0:\n            return \"%3.1f %s\" % (size, x)\n        size /= 1024.0","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.675445Z","iopub.execute_input":"2023-01-05T07:17:45.676678Z","iopub.status.idle":"2023-01-05T07:17:45.689672Z","shell.execute_reply.started":"2023-01-05T07:17:45.676627Z","shell.execute_reply":"2023-01-05T07:17:45.688228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Sample Video","metadata":{}},{"cell_type":"code","source":"play(SAMPLE)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.692763Z","iopub.execute_input":"2023-01-05T07:17:45.693693Z","iopub.status.idle":"2023-01-05T07:17:45.853753Z","shell.execute_reply.started":"2023-01-05T07:17:45.693642Z","shell.execute_reply":"2023-01-05T07:17:45.852376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get metadata of the loaded video using cv2\n# Start capturing the feed\ncap = cv2.VideoCapture(SAMPLE)\n# Find the number of frames\nframe_number = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\nprint(\"Number of frames: \", frame_number)\nfps = cap.get(cv2.CAP_PROP_FPS)\nprint('Frames per second: ', fps)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.856037Z","iopub.execute_input":"2023-01-05T07:17:45.856671Z","iopub.status.idle":"2023-01-05T07:17:45.884756Z","shell.execute_reply.started":"2023-01-05T07:17:45.856603Z","shell.execute_reply":"2023-01-05T07:17:45.883665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This matches with the description of the competition.\nThere are 711 frames overall and 59.94 frames are taken per second.","metadata":{}},{"cell_type":"code","source":"video_length_second = frame_number / fps\nprint('Video lenth: ', video_length_second, 's')","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.886670Z","iopub.execute_input":"2023-01-05T07:17:45.887891Z","iopub.status.idle":"2023-01-05T07:17:45.893986Z","shell.execute_reply.started":"2023-01-05T07:17:45.887840Z","shell.execute_reply":"2023-01-05T07:17:45.892563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a folder where each frame is archived as jpg \ntry: \n    os.mkdir('../frames') \nexcept OSError as error: \n    import shutil\n    shutil.rmtree('../frames')\n    os.mkdir('../frames')","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.896161Z","iopub.execute_input":"2023-01-05T07:17:45.896773Z","iopub.status.idle":"2023-01-05T07:17:45.974648Z","shell.execute_reply.started":"2023-01-05T07:17:45.896690Z","shell.execute_reply":"2023-01-05T07:17:45.973439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, extract each frame using ffmpeg. You may use OpenCV for it but I chose ffmpeg as it was faster.","metadata":{}},{"cell_type":"code","source":"%%time\n# Use ffmpeg command to produce frames from the video\n!ffmpeg -i /kaggle/input/nfl-player-contact-detection/train/58168_003392_Endzone.mp4 -q:v 2 -f image2 ../frames/58168_003392_Endzone_%04d.jpg -hide_banner -loglevel error","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:45.977811Z","iopub.execute_input":"2023-01-05T07:17:45.978420Z","iopub.status.idle":"2023-01-05T07:17:52.864597Z","shell.execute_reply.started":"2023-01-05T07:17:45.978379Z","shell.execute_reply":"2023-01-05T07:17:52.862842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the downloaded frames\nframe_list = os.listdir('../frames/')\nprint(frame_list[:5])","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:52.867040Z","iopub.execute_input":"2023-01-05T07:17:52.868126Z","iopub.status.idle":"2023-01-05T07:17:52.878168Z","shell.execute_reply.started":"2023-01-05T07:17:52.868067Z","shell.execute_reply":"2023-01-05T07:17:52.876781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the max frame number\nmax(frame_list)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:52.880187Z","iopub.execute_input":"2023-01-05T07:17:52.881782Z","iopub.status.idle":"2023-01-05T07:17:52.892440Z","shell.execute_reply.started":"2023-01-05T07:17:52.881691Z","shell.execute_reply":"2023-01-05T07:17:52.891119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There were indeed 711 frames.","metadata":{}},{"cell_type":"markdown","source":"Now, I am going to display the frame images.","metadata":{}},{"cell_type":"code","source":"# Display the first 20 frames in the result\n\n# Specify the figure width and height\nfig = plt.figure(figsize=(25, 20))\n\nfor i in range(1, 21):\n    frame_id = '%04d' % i\n    img = Image.open(f'../frames/58168_003392_Endzone_{frame_id}.jpg')\n    ax = fig.add_subplot(4, 5, i, xticks=[], yticks=[])    \n    plt.imshow(img)\n    plt.title(frame_id)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:52.894138Z","iopub.execute_input":"2023-01-05T07:17:52.894563Z","iopub.status.idle":"2023-01-05T07:17:58.047103Z","shell.execute_reply.started":"2023-01-05T07:17:52.894522Z","shell.execute_reply":"2023-01-05T07:17:58.045788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the first to the last, not much difference is observed as fps is high. In the next cell, I am going to increase the interval for each frame for the 20 images to be displayed.","metadata":{}},{"cell_type":"code","source":"# Display the first 20 frames in the result\n\n# Specify the figure width and height\nfig = plt.figure(figsize=(25, 20))\n\nfor i in range(1, 21):\n    frame_id_int = i * 28\n    frame_id = '%04d' % frame_id_int\n    img = Image.open(f'../frames/58168_003392_Endzone_{frame_id}.jpg')\n    ax = fig.add_subplot(4, 5, i, xticks=[], yticks=[])    \n    plt.imshow(img)\n    plt.title(frame_id)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:17:58.048831Z","iopub.execute_input":"2023-01-05T07:17:58.049196Z","iopub.status.idle":"2023-01-05T07:18:03.474134Z","shell.execute_reply.started":"2023-01-05T07:17:58.049152Z","shell.execute_reply":"2023-01-05T07:18:03.471440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Train Helmet","metadata":{}},{"cell_type":"code","source":"train_helmets_df = pd.read_csv(TRAIN_HELMET)\nprint(f\"Dataframe Size: {convert_bytes(train_helmets_df.size)}\")","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.475743Z","iopub.execute_input":"2023-01-05T07:18:03.476110Z","iopub.status.idle":"2023-01-05T07:18:03.551912Z","shell.execute_reply.started":"2023-01-05T07:18:03.476077Z","shell.execute_reply":"2023-01-05T07:18:03.550470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_helmets_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.553199Z","iopub.execute_input":"2023-01-05T07:18:03.553541Z","iopub.status.idle":"2023-01-05T07:18:03.572083Z","shell.execute_reply.started":"2023-01-05T07:18:03.553511Z","shell.execute_reply":"2023-01-05T07:18:03.570627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print how many videos are available in the train helmet data\ntrain_helmets_df.video.unique()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.574052Z","iopub.execute_input":"2023-01-05T07:18:03.574826Z","iopub.status.idle":"2023-01-05T07:18:03.592070Z","shell.execute_reply.started":"2023-01-05T07:18:03.574575Z","shell.execute_reply":"2023-01-05T07:18:03.590439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a method that extracts the train helmet data for a certain video\ndef get_helmets_df(source_helmets_df, video):\n    return source_helmets_df[source_helmets_df.video == video].copy()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.593780Z","iopub.execute_input":"2023-01-05T07:18:03.594293Z","iopub.status.idle":"2023-01-05T07:18:03.602958Z","shell.execute_reply.started":"2023-01-05T07:18:03.594254Z","shell.execute_reply":"2023-01-05T07:18:03.601827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_helmets_df = get_helmets_df(train_helmets_df, sample_video)\nprint(f\"Dataframe size: {convert_bytes(sample_helmets_df.size)}\")","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.604408Z","iopub.execute_input":"2023-01-05T07:18:03.605486Z","iopub.status.idle":"2023-01-05T07:18:03.624193Z","shell.execute_reply.started":"2023-01-05T07:18:03.605445Z","shell.execute_reply":"2023-01-05T07:18:03.622581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This tells that 82.9KB in 554,6KB of train_helmets contains the data related to the picked sample video.","metadata":{}},{"cell_type":"code","source":"# Check min and max frame number of the sample helmet data\nprint('Min frame no: ', sample_helmets_df.frame.min())\nprint('Max frame no: ', sample_helmets_df.frame.max())","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.626416Z","iopub.execute_input":"2023-01-05T07:18:03.628095Z","iopub.status.idle":"2023-01-05T07:18:03.636173Z","shell.execute_reply.started":"2023-01-05T07:18:03.628022Z","shell.execute_reply":"2023-01-05T07:18:03.634977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The result was in the range of 1 - 711 frames of the video, so the data looks good.","metadata":{}},{"cell_type":"markdown","source":"# Spot Helmets in a Frame\nI pick up frame no `0290` for this visualization work.","metadata":{}},{"cell_type":"code","source":"sample_frame_no = \"0290\"","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.640803Z","iopub.execute_input":"2023-01-05T07:18:03.641252Z","iopub.status.idle":"2023-01-05T07:18:03.647739Z","shell.execute_reply.started":"2023-01-05T07:18:03.641213Z","shell.execute_reply":"2023-01-05T07:18:03.646244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Keep only the helmet data of the select video and frame\ntrain_helmets_sample_df = train_helmets_df[(train_helmets_df.video == sample_video) & (train_helmets_df.frame == int(sample_frame_no))].copy()\ndel train_helmets_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:03.649515Z","iopub.execute_input":"2023-01-05T07:18:03.650280Z","iopub.status.idle":"2023-01-05T07:18:04.057475Z","shell.execute_reply.started":"2023-01-05T07:18:03.650241Z","shell.execute_reply":"2023-01-05T07:18:04.056228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_helmets_sample_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:04.058828Z","iopub.execute_input":"2023-01-05T07:18:04.059698Z","iopub.status.idle":"2023-01-05T07:18:04.066344Z","shell.execute_reply.started":"2023-01-05T07:18:04.059658Z","shell.execute_reply":"2023-01-05T07:18:04.065513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is not too lengthy, so I print all the records of `train_helmets_sample_df` below.","metadata":{}},{"cell_type":"code","source":"display(train_helmets_sample_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:18:04.067696Z","iopub.execute_input":"2023-01-05T07:18:04.068244Z","iopub.status.idle":"2023-01-05T07:18:04.095082Z","shell.execute_reply.started":"2023-01-05T07:18:04.068208Z","shell.execute_reply":"2023-01-05T07:18:04.093458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks there are 20 players in the frame.\nLet's pick up one player's record and identify his helmet in the image.","metadata":{}},{"cell_type":"code","source":"sample_helmet = train_helmets_sample_df.loc[0]\nprint(sample_helmet.left)\nprint(sample_helmet.top)\nprint(sample_helmet.height)\nprint(sample_helmet.width)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:20:39.435021Z","iopub.execute_input":"2023-01-05T07:20:39.435438Z","iopub.status.idle":"2023-01-05T07:20:39.442504Z","shell.execute_reply.started":"2023-01-05T07:20:39.435405Z","shell.execute_reply":"2023-01-05T07:20:39.441489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define a method to return the location of every corner of a rectangle.","metadata":{}},{"cell_type":"code","source":"def locate_rectangle(helmet):\n    left = helmet.left\n    right = helmet.left + helmet.width\n    top = helmet.top\n    bottom = helmet.top + helmet.height\n    return left, top, right, bottom","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:25:49.667001Z","iopub.execute_input":"2023-01-05T07:25:49.667424Z","iopub.status.idle":"2023-01-05T07:25:49.674617Z","shell.execute_reply.started":"2023-01-05T07:25:49.667391Z","shell.execute_reply":"2023-01-05T07:25:49.673081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open(f'../frames/58168_003392_Endzone_0290.jpg')\nplt.figure(figsize=(8, 6), dpi=100)\n\n# Draw rectangle\ndraw = ImageDraw.Draw(img)\ndraw.rectangle((locate_rectangle(sample_helmet)), outline='#FF0000', width=8)\n\nplt.imshow(img)\n\nprint(img.size)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:25:52.987258Z","iopub.execute_input":"2023-01-05T07:25:52.987665Z","iopub.status.idle":"2023-01-05T07:25:53.594617Z","shell.execute_reply.started":"2023-01-05T07:25:52.987632Z","shell.execute_reply":"2023-01-05T07:25:53.593469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As shown above, the helmet of the sample player was spotted.\nNext, I will spot the helmets of all the players in the frame.","metadata":{}},{"cell_type":"code","source":"img = Image.open(f'../frames/58168_003392_Endzone_0290.jpg')\nplt.figure(figsize=(8, 6), dpi=100)\n\n# Draw rectangle\ndraw = ImageDraw.Draw(img)\n\n# Do iteration to draw rectangles\nfor i, row in train_helmets_sample_df.iterrows():\n#     print(f\"Player: {i}\")\n    helmet = train_helmets_sample_df.loc[i]\n    draw.rectangle((locate_rectangle(helmet)), outline='#FF0000', width=8)\n\nplt.imshow(img)\n\nprint(img.size)    ","metadata":{"execution":{"iopub.status.busy":"2023-01-05T07:33:26.356707Z","iopub.execute_input":"2023-01-05T07:33:26.357943Z","iopub.status.idle":"2023-01-05T07:33:26.941745Z","shell.execute_reply.started":"2023-01-05T07:33:26.357888Z","shell.execute_reply":"2023-01-05T07:33:26.940612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thank you. Please upvote if you find this helpful.","metadata":{}}]}