{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ASL Competition: Drawing video and gif animation from landmark files","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## 1. Exploratory Data Analysis","metadata":{"execution":{"iopub.status.busy":"2023-04-15T21:19:37.988963Z","iopub.execute_input":"2023-04-15T21:19:37.989376Z","iopub.status.idle":"2023-04-15T21:19:37.994029Z","shell.execute_reply.started":"2023-04-15T21:19:37.989344Z","shell.execute_reply":"2023-04-15T21:19:37.992979Z"}}},{"cell_type":"code","source":"# Installing libraries\n!pip install opencv-python\n!pip install imageio --quiet\n!pip install 'imageio[pyav]' --quiet\n!pip install 'imageio[ffmpeg]' --quiet\n!pip install tqdm\n!pip install mediapipe\n","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:57:10.787163Z","iopub.execute_input":"2023-04-27T02:57:10.787768Z","iopub.status.idle":"2023-04-27T02:58:19.225943Z","shell.execute_reply.started":"2023-04-27T02:57:10.787713Z","shell.execute_reply":"2023-04-27T02:58:19.224563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Libraries\nimport os\nimport base64\nfrom IPython.display import Video, display\nfrom IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport imageio\nimport mediapipe as mp\n\nBASE_PREFIX = \"../input/asl-signs\"\nMEDIA_PREFIX = \"output/media\"\n\n# creating output directory\nos.makedirs(MEDIA_PREFIX, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:19.228765Z","iopub.execute_input":"2023-04-27T02:58:19.229146Z","iopub.status.idle":"2023-04-27T02:58:19.237244Z","shell.execute_reply.started":"2023-04-27T02:58:19.229106Z","shell.execute_reply":"2023-04-27T02:58:19.235761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploring Metadata","metadata":{}},{"cell_type":"code","source":"!ls ../input/asl-signs/","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:19.239032Z","iopub.execute_input":"2023-04-27T02:58:19.239391Z","iopub.status.idle":"2023-04-27T02:58:20.344188Z","shell.execute_reply.started":"2023-04-27T02:58:19.239333Z","shell.execute_reply":"2023-04-27T02:58:20.342678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{BASE_PREFIX}/train.csv\")\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.345909Z","iopub.execute_input":"2023-04-27T02:58:20.346729Z","iopub.status.idle":"2023-04-27T02:58:20.491870Z","shell.execute_reply.started":"2023-04-27T02:58:20.346680Z","shell.execute_reply":"2023-04-27T02:58:20.490681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#signs to predict \ntrain_df.groupby([\"sign\"]).count() #250 unique signs","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.496037Z","iopub.execute_input":"2023-04-27T02:58:20.496652Z","iopub.status.idle":"2023-04-27T02:58:20.531168Z","shell.execute_reply.started":"2023-04-27T02:58:20.496581Z","shell.execute_reply":"2023-04-27T02:58:20.530008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploring Landmark data\nPicking up one random parquet (landmark file) from the metadata","metadata":{}},{"cell_type":"code","source":"# filtering the row in train_df\ntrain_df.query(\"sequence_id==1012730846\")","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.532465Z","iopub.execute_input":"2023-04-27T02:58:20.532860Z","iopub.status.idle":"2023-04-27T02:58:20.550200Z","shell.execute_reply.started":"2023-04-27T02:58:20.532823Z","shell.execute_reply":"2023-04-27T02:58:20.549062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extracting the 'path' value corresponding to the \"please\" sign\nplease_sign_id = \"1012730846\"\nsample_landmark_path = train_df.query(f\"sequence_id=={please_sign_id}\").iloc[0,0]\nsample_landmark_path","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.552126Z","iopub.execute_input":"2023-04-27T02:58:20.552914Z","iopub.status.idle":"2023-04-27T02:58:20.565759Z","shell.execute_reply.started":"2023-04-27T02:58:20.552862Z","shell.execute_reply":"2023-04-27T02:58:20.564227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loading the sample landmark\nsample_df = pd.read_parquet(f\"{BASE_PREFIX}/{sample_landmark_path}\")\nsample_df","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.567542Z","iopub.execute_input":"2023-04-27T02:58:20.568337Z","iopub.status.idle":"2023-04-27T02:58:20.608379Z","shell.execute_reply.started":"2023-04-27T02:58:20.568286Z","shell.execute_reply":"2023-04-27T02:58:20.607039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Processing parquet file to convert a video, then as a gift","metadata":{}},{"cell_type":"code","source":"def convert_landmarks_to_video(sample_name, sample_df):\n    # Dropping the records with null x and y coordinates (z doesn't matter for drawing)\n    sample_df = sample_df.dropna(subset=[\"x\",\"y\"])\n    # Create a VideoWriter object to write the output video\n    output_video = cv2.VideoWriter(f'{MEDIA_PREFIX}/{sample_name}.avi', cv2.VideoWriter_fourcc(*'MJPG'), 30, (640, 480))\n\n    # Iterate over each frame in the DataFrame\n    for frame_num, frame_data in sample_df.groupby('frame'):\n        # Create a blank image for the current frame\n        input_image = np.zeros((480, 640, 3), dtype=np.uint8)\n        \n        # Iterate over each row in the current frame\n        for i, row in frame_data.iterrows():\n            # Extract the landmark coordinates from the row\n            x, y = int(row['x'] * 640), int(row['y'] * 480)\n\n            # Draw a circle at the landmark location on the image\n            cv2.circle(input_image, (x, y), 5, (0, 255, 0), -1)\n        # Write the output frame to the output video\n        output_video.write(input_image)\n        \n    # Release the video writer and close the file\n    output_video.release()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.610116Z","iopub.execute_input":"2023-04-27T02:58:20.611313Z","iopub.status.idle":"2023-04-27T02:58:20.621948Z","shell.execute_reply.started":"2023-04-27T02:58:20.611255Z","shell.execute_reply":"2023-04-27T02:58:20.620648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_video_to_gif(sample_name):\n    video_path = f'{MEDIA_PREFIX}/{sample_name}.avi'\n    gif_path = f'{MEDIA_PREFIX}/{sample_name}.gif'\n    # Convert video to gif\n    with imageio.get_reader(video_path) as reader, imageio.get_writer(gif_path, mode='I') as writer:\n        for frame in reader:\n            writer.append_data(frame)\n\n    # Display gif in notebook\n    with open(gif_path, 'rb') as f:\n        display(HTML('<img width=400 src=\"data:image/gif;base64,{}\"/>'.format(base64.b64encode(f.read()).decode())))","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.624197Z","iopub.execute_input":"2023-04-27T02:58:20.624805Z","iopub.status.idle":"2023-04-27T02:58:20.634272Z","shell.execute_reply.started":"2023-04-27T02:58:20.624756Z","shell.execute_reply":"2023-04-27T02:58:20.632954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_sign(sign_id):\n    landmark_path = train_df.query(f\"sequence_id=={sign_id}\").iloc[0,0]\n    sign_df = pd.read_parquet(f\"{BASE_PREFIX}/{landmark_path}\")\n    convert_landmarks_to_video(sign_id, sign_df)\n    convert_video_to_gif(sign_id)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.636757Z","iopub.execute_input":"2023-04-27T02:58:20.637123Z","iopub.status.idle":"2023-04-27T02:58:20.650998Z","shell.execute_reply.started":"2023-04-27T02:58:20.637091Z","shell.execute_reply":"2023-04-27T02:58:20.649571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## sign sample: please\nplease_sign_id = '1012730846'\nshow_sign(please_sign_id)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:20.652723Z","iopub.execute_input":"2023-04-27T02:58:20.653100Z","iopub.status.idle":"2023-04-27T02:58:23.764797Z","shell.execute_reply.started":"2023-04-27T02:58:20.653065Z","shell.execute_reply":"2023-04-27T02:58:23.763332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Another example: \"blow\" sign\nblow_sign_id = '1000035562'\nshow_sign(blow_sign_id)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:23.766718Z","iopub.execute_input":"2023-04-27T02:58:23.767157Z","iopub.status.idle":"2023-04-27T02:58:25.588455Z","shell.execute_reply.started":"2023-04-27T02:58:23.767110Z","shell.execute_reply":"2023-04-27T02:58:25.586927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Another example: wait\nid_wait = \"1019555958\"\nshow_sign(id_wait)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:25.592731Z","iopub.execute_input":"2023-04-27T02:58:25.593473Z","iopub.status.idle":"2023-04-27T02:58:29.988543Z","shell.execute_reply.started":"2023-04-27T02:58:25.593416Z","shell.execute_reply":"2023-04-27T02:58:29.986793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"landmark_path = train_df.query(f\"sequence_id=={id_wait}\").iloc[0,0]\nsign_df = pd.read_parquet(f\"{BASE_PREFIX}/{landmark_path}\")\n#sign_df = sign_df.dropna(subset=[\"x\",\"y\",\"z\"])\nsign_df","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:29.990683Z","iopub.execute_input":"2023-04-27T02:58:29.991058Z","iopub.status.idle":"2023-04-27T02:58:30.045071Z","shell.execute_reply.started":"2023-04-27T02:58:29.991016Z","shell.execute_reply":"2023-04-27T02:58:30.043692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign_df.groupby([\"type\"]).count()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:30.046243Z","iopub.execute_input":"2023-04-27T02:58:30.046572Z","iopub.status.idle":"2023-04-27T02:58:30.066741Z","shell.execute_reply.started":"2023-04-27T02:58:30.046541Z","shell.execute_reply":"2023-04-27T02:58:30.065554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sign_df.frame.unique()\nfrom tqdm.notebook import tqdm\n\nN_PARQUETS_TO_TEAD = 20\n\nfor i, row in tqdm(train_df.iterrows(), total=N_PARQUETS_TO_TEAD):\n    df_sign = pd.read_parquet(f\"{BASE_PREFIX}/{row['path']}\")\n    df_sign = df_sign.dropna(subset=['x','y','z'])\n    df_sign_without_face = df_sign.query(\"type != 'face'\")\n    print(f\"{row['sequence_id']} -> {row['sign']} -> frames: {len(df_sign.frame.unique())}-> total:{len(df_sign)} -> without face: {len(df_sign_without_face)}\")\n    if i >= N_PARQUETS_TO_TEAD:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:30.068036Z","iopub.execute_input":"2023-04-27T02:58:30.068765Z","iopub.status.idle":"2023-04-27T02:58:30.546174Z","shell.execute_reply.started":"2023-04-27T02:58:30.068727Z","shell.execute_reply":"2023-04-27T02:58:30.544848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_marks(landmarks: dict):\n    marks = []\n    for el in landmarks:\n        marks.append(el[0])\n        marks.append(el[1])\n    marks.sort()\n    return list(dict.fromkeys(marks)) # removing duplicates","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:30.547447Z","iopub.execute_input":"2023-04-27T02:58:30.547783Z","iopub.status.idle":"2023-04-27T02:58:30.554319Z","shell.execute_reply.started":"2023-04-27T02:58:30.547749Z","shell.execute_reply":"2023-04-27T02:58:30.553118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lips_marks = get_marks(mp.solutions.face_mesh_connections.FACEMESH_LIPS)\nhand_marks = get_marks(mp.solutions.hands.HAND_CONNECTIONS)\npose_marks = get_marks(mp.solutions.pose.POSE_CONNECTIONS)\n\nprint(f'lips: {lips_marks}')\nprint(f'hand: {hand_marks}')\nprint(f'pose: {pose_marks}')","metadata":{"execution":{"iopub.status.busy":"2023-04-27T02:58:30.555418Z","iopub.execute_input":"2023-04-27T02:58:30.555764Z","iopub.status.idle":"2023-04-27T02:58:30.569309Z","shell.execute_reply.started":"2023-04-27T02:58:30.555731Z","shell.execute_reply":"2023-04-27T02:58:30.568354Z"},"trusted":true},"execution_count":null,"outputs":[]}]}