{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport math\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:54.479241Z","iopub.execute_input":"2023-04-27T21:47:54.480000Z","iopub.status.idle":"2023-04-27T21:47:54.485834Z","shell.execute_reply.started":"2023-04-27T21:47:54.479960Z","shell.execute_reply":"2023-04-27T21:47:54.484501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config Parameters","metadata":{}},{"cell_type":"code","source":"SEED = 7122000\nUSE_WHOLE_DATASET = False\n\n\nTOTAL_FRAMES = 50\n\n# A single skeleton (for a single frame) will be of size: \n# total_dim = 3 * (left_hand + right_hand + face + pose), \n# where 3 comes from the fact that each coord is (x,y,z)\nSKELETON_SIZE = {\n    \"left_hand\": 21,\n    \"right_hand\": 21,\n    \"face\": 468,\n    \"pose\": 33,\n}\n","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:54.487580Z","iopub.execute_input":"2023-04-27T21:47:54.488590Z","iopub.status.idle":"2023-04-27T21:47:54.498485Z","shell.execute_reply.started":"2023-04-27T21:47:54.488553Z","shell.execute_reply":"2023-04-27T21:47:54.497240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/asl-signs/train.csv') if USE_WHOLE_DATASET else pd.read_csv('/kaggle/input/asl-signs/train.csv').sample(int(5e3), random_state=SEED)\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:54.539359Z","iopub.execute_input":"2023-04-27T21:47:54.540548Z","iopub.status.idle":"2023-04-27T21:47:54.678626Z","shell.execute_reply.started":"2023-04-27T21:47:54.540502Z","shell.execute_reply":"2023-04-27T21:47:54.677539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualise Sign Count Histrogram","metadata":{}},{"cell_type":"code","source":"def display_sign_histogram(dataset):\n    fig, ax = plt.subplots(figsize=(5, 3))\n    value_counts = dict(train[\"sign\"].value_counts())\n    signs = [i for i in range(len(value_counts.keys()))]\n    values = list(value_counts.values())\n    ax.bar(signs, values, width=1.0)\n    \n    ax.set_ylabel('Count')\n    ax.set_xlabel('Label ID')\n    ax.set_title('ASL Label Histogram')\n    plt.show()\n    \ndisplay_sign_histogram(train)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:54.680663Z","iopub.execute_input":"2023-04-27T21:47:54.681255Z","iopub.status.idle":"2023-04-27T21:47:55.220859Z","shell.execute_reply.started":"2023-04-27T21:47:54.681213Z","shell.execute_reply":"2023-04-27T21:47:55.219596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"# drop participant ID column\n# add skeleton_columns\ndef take_first_n_coords_util(coord, row): \n    # We associate each landmark (eg \"left_hand\") with a total number of coordinate points\n    landmark = row['type']\n    max_coords = SKELETON_SIZE[landmark]\n        \n    # We only work with the first SKELETON_SIZE[type] coords for each type. \n    coords = row[coord][:max_coords]\n        \n    # If a row contains less corodinates than SKELETON_SIZE[type], we add a padding \n    padded_coords = np.pad(coords, (0, max_coords-len(coords)), 'constant', constant_values=(0,))\n\n    return padded_coords\n\ndef take_first_n_coords(row):\n    row['x'] = take_first_n_coords_util('x', row)\n    row['y'] = take_first_n_coords_util('y', row)\n    row['z'] = take_first_n_coords_util('z', row)\n    row['skeleton'] = np.concatenate([row['x'], row['y'], row['z']],axis=None)\n    return row","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:55.222317Z","iopub.execute_input":"2023-04-27T21:47:55.222664Z","iopub.status.idle":"2023-04-27T21:47:55.230658Z","shell.execute_reply.started":"2023-04-27T21:47:55.222630Z","shell.execute_reply":"2023-04-27T21:47:55.229408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"face_map = {\n    \"silhouette\": [10, 21, 54, 58, 67, 93, 103, 109, 127, 132, 136, 148, 149, 150, 152, 162, 168, 172, 176, 234, 251, 284, 288, 297, 323, 332, 338, 356, 361, 365, 377, 378, 379, 389, 397, 400, 454],\n    \"lips\": [0, 13, 14, 17, 37, 39, 40, 61, 78, 80, 81, 82, 84, 87, 88, 91, 95, 146, 178, 181, 185, 191, 267, 269, 270, 291, 308, 310, 311, 312, 314, 317, 318, 321, 324, 375, 402, 405, 409, 415],\n    \"right_eye\": [7, 22, 23, 24, 25, 26, 31, 33, 110, 112, 130, 133, 144, 145, 153, 154, 155, 157, 158, 159, 160, 161, 163, 173, 226, 228, 229, 230, 231, 232, 233, 243, 244, 246],\n    \"left_eye\": [249, 252, 253, 254, 255, 256, 261, 263, 339, 341, 359, 362, 373, 374, 380, 381, 382, 384, 385, 386, 387, 388, 390, 398, 446, 448, 449, 450, 451, 452, 453, 463, 464, 466],\n    \"right_eyebrow\": [35, 46, 52, 53, 55, 63, 65, 66, 70, 105, 107, 124, 156, 193],\n    \"left_eyebrow\": [265, 276, 282, 283, 285, 293, 295, 296, 300, 334, 336, 353, 383, 417],\n    \"nose\": [1,2,98,327],\n    \"cheeks\": [205,425]\n}\n\ndef get_skeleton(path, facial_features = None):\n    parquet = pd.read_parquet(\"/kaggle/input/asl-signs/\" + file_path)\n \n    if facial_features != None:\n        take_face_indices = list(set(np.concatenate(list(map(lambda x: face_map[x], facial_features)))))\n        SKELETON_SIZE[\"face\"]  = len(take_face_indices)\n        print(\"Face indices:\", SKELETON_SIZE[\"face\"])\n        parquet = parquet[~((parquet[\"type\"] == \"face\") & (~parquet[\"landmark_index\"].isin(take_face_indices))) ]\n    else: \n        SKELETON_SIZE[\"face\"] = 468\n        \n    # Produces dataframe with columns: \n    # frame: int, type: str, x:float[], y:float[], z:float[] \n    concatenated_vectors = parquet.groupby(['frame', 'type']).agg({'x': list, 'y': list, 'z': list}).reset_index()\n    \n    # Take the first N coords (N is landmark specific)\n    # Then concatenate all x,y,z and store as (local landmark) \"skeleton\" (basically a skeleton of a body part)\n    concatenated_vectors = concatenated_vectors.apply(take_first_n_coords, axis=1)\n\n    # Group all concatenated landmark skeletons based on their frame (basically we group all body parts together based on frame)\n    concatenated_vectors = concatenated_vectors.groupby('frame').agg({'skeleton': list}).reset_index()\n        \n    # Each row represents a (frame -> skeleton)\n    skeleton = concatenated_vectors['skeleton'].apply(lambda x: pd.Series(np.concatenate(x)))\n    \n    # Take first \"TOTAL_FRAMES\" number of frames, and set all NaN to 0\n    skeleton = skeleton[:TOTAL_FRAMES].fillna(0)\n    return skeleton","metadata":{"execution":{"iopub.status.busy":"2023-04-27T22:03:30.449331Z","iopub.execute_input":"2023-04-27T22:03:30.449793Z","iopub.status.idle":"2023-04-27T22:03:30.469383Z","shell.execute_reply.started":"2023-04-27T22:03:30.449752Z","shell.execute_reply":"2023-04-27T22:03:30.468500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, file_path in enumerate(tqdm(train['path'].sample(1, random_state=0))):\n    skeleton = get_skeleton(file_path, [\"silhouette\", \"lips\", \"right_eye\", \"left_eye\", \"right_eyebrow\", \"left_eyebrow\", \"nose\", \"cheeks\"])\n    \nskeleton","metadata":{"execution":{"iopub.status.busy":"2023-04-27T22:06:32.173621Z","iopub.execute_input":"2023-04-27T22:06:32.174090Z","iopub.status.idle":"2023-04-27T22:06:32.321098Z","shell.execute_reply.started":"2023-04-27T22:06:32.174048Z","shell.execute_reply":"2023-04-27T22:06:32.320237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"face_map = {\n    \"silhouette\": [10, 21, 54, 58, 67, 93, 103, 109, 127, 132, 136, 148, 149, 150, 152, 162, 168, 172, 176, 234, 251, 284, 288, 297, 323, 332, 338, 356, 361, 365, 377, 378, 379, 389, 397, 400, 454],\n    \"lips\": [0, 13, 14, 17, 37, 39, 40, 61, 78, 80, 81, 82, 84, 87, 88, 91, 95, 146, 178, 181, 185, 191, 267, 269, 270, 291, 308, 310, 311, 312, 314, 317, 318, 321, 324, 375, 402, 405, 409, 415],\n    \"right_eye\": [7, 22, 23, 24, 25, 26, 31, 33, 110, 112, 130, 133, 144, 145, 153, 154, 155, 157, 158, 159, 160, 161, 163, 173, 226, 228, 229, 230, 231, 232, 233, 243, 244, 246],\n    \"left_eye\": [249, 252, 253, 254, 255, 256, 261, 263, 339, 341, 359, 362, 373, 374, 380, 381, 382, 384, 385, 386, 387, 388, 390, 398, 446, 448, 449, 450, 451, 452, 453, 463, 464, 466],\n    \"right_eyebrow\": [35, 46, 52, 53, 55, 63, 65, 66, 70, 105, 107, 124, 156, 193],\n    \"left_eyebrow\": [265, 276, 282, 283, 285, 293, 295, 296, 300, 334, 336, 353, 383, 417],\n    \"nose\": [1,2,98,327],\n    \"cheeks\": [205,425]\n}\n\ncolors = {\n    \"silhouette\": \"black\",\n    \"lips\": \"maroon\",\n    \"right_eye\": \"blue\",\n    \"right_eyebrow\": \"purple\",\n    \"left_eye\": \"green\",\n    \"left_eyebrow\": \"orange\",\n    \"nose\": \"brown\",\n    \"cheeks\": \"gold\"\n}","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:55.422724Z","iopub.execute_input":"2023-04-27T21:47:55.423186Z","iopub.status.idle":"2023-04-27T21:47:55.438042Z","shell.execute_reply.started":"2023-04-27T21:47:55.423137Z","shell.execute_reply":"2023-04-27T21:47:55.436959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, file_path in enumerate(tqdm(train['path'].sample(1, random_state=0))):\n    skeleton = pd.read_parquet(\"/kaggle/input/asl-signs/\" + file_path)\n\nx = list(skeleton['x'])[0:468] \ny = list(skeleton['y'])[0:468]\nz = list(skeleton['z'])[0:468]\n\nface_map_inv = {v: k for k, values in face_map.items() for v in values}\nface_colors = [(colors[face_map_inv[i]] if i in face_map_inv else \"grey\") for i in range(468)]\n\n# plot the points with colors\n\nfig = plt.figure(figsize=(6, 6))\nax = fig.add_subplot(111, projection='3d')\nax.scatter(x, y, z, c=face_colors, alpha=.6)\n# ax.scatter(x, y, z, alpha=.6)\nax.set_xlabel('X')\nax.set_ylabel('Y')\nax.set_zlabel('Z')\nax.view_init(elev=70, azim=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:55.439148Z","iopub.execute_input":"2023-04-27T21:47:55.439583Z","iopub.status.idle":"2023-04-27T21:47:55.690209Z","shell.execute_reply.started":"2023-04-27T21:47:55.439548Z","shell.execute_reply":"2023-04-27T21:47:55.688864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = list()\n\nfor idx, file_path in enumerate(tqdm(train['path'])):\n    parquet = pd.read_parquet(\"/kaggle/input/asl-signs/\" + file_path)\n    frames_col = list(parquet[\"frame\"])\n    max_frame = frames_col[-1]\n    min_frame = frames_col[0]\n    frames.append(max_frame - min_frame + 1)\n    \nfig,ax = plt.subplots(figsize=(5, 3))\nax.hist(frames, bins=100)\n\nax.set_ylabel('Count')\nax.set_xlabel('Number of Frames')\nax.set_title('Frame Histogram')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:55.692019Z","iopub.execute_input":"2023-04-27T21:47:55.692509Z","iopub.status.idle":"2023-04-27T21:47:58.592829Z","shell.execute_reply.started":"2023-04-27T21:47:55.692463Z","shell.execute_reply":"2023-04-27T21:47:58.590728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_null_distribution(landmark, plot_title, bins=100):\n    landmark_missing_percentage = list()\n    fully_empty = list()\n    partially_empty = 0\n\n    for idx, file_path in enumerate(tqdm(train['path'])):\n        parquet = pd.read_parquet(\"/kaggle/input/asl-signs/\" + file_path)\n        hand = parquet[parquet[\"type\"] == landmark]\n        hand_i0 = list(hand[hand[\"landmark_index\"] == 0]['x'])\n        isEm = False\n        if np.isnan(hand_i0).all():\n            fully_empty.append(len(hand_i0))\n        else: \n            maxframes = len(hand_i0)\n            for index, entry in enumerate(hand_i0):\n                if math.isnan(entry):\n                    isEm = True\n                    landmark_missing_percentage.append((1 + index) / maxframes)\n            \n            if isEm:\n                partially_empty += 1\n        \n    fig,ax = plt.subplots(figsize=(10, 3))\n    ax.set_title(plot_title)\n    ax.hist(landmark_missing_percentage, bins=bins)\n    plt.show()\n    print(\"Fully Empty\", pd.DataFrame(fully_empty).describe())\n    print(\"Partially Empty:\", partially_empty)\n    return landmark_missing_percentage\n    \nbins_size = 200\nright_hand_pp = show_null_distribution(\"right_hand\", \"Occurance of null values during video for the right hand\", bins_size)\nleft_hand_pp =show_null_distribution(\"left_hand\", \"Occurance of null values during video for the left hand\", bins_size)","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:58.594285Z","iopub.status.idle":"2023-04-27T21:47:58.594998Z","shell.execute_reply.started":"2023-04-27T21:47:58.594780Z","shell.execute_reply":"2023-04-27T21:47:58.594805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\ndef conv1D(x,w, p=2 , s=1): \n  '''\n  x : input vector\n  w : filter\n  p : padding size\n  s : stride\n  '''\n  assert len(w) <= len(x), \"x should be bigger than w\"\n  assert p >= 0, \"padding cannot be negative\"\n\n  w_r = np.array(w[::-1]) #rotation of w \n  x_padded = np.array(x)\n\n  if p > 0 :\n    zeros = np.zeros(shape = p)\n    x_padded = np.concatenate([zeros, x_padded, zeros]) #add zeros around original vector\n\n  out = []\n  #iterate through the original array s cells per step\n  for i in range(0, int((len(x_padded) - len(w_r))) + 1 , s):\n    out.append(np.sum(x_padded[i:i + w_r.shape[0]] * w_r)) #formula we have seen before\n  return np.array(out)\n\n# right_hand_pp1 = list(filter(lambda x: x != 0 and x!= 1, map(fn, right_hand_pp)))\n# left_hand_pp1 = list(filter(lambda x: x != 0 and x!= 1,map(fn, left_hand_pp)))\n\ndef plot_hist(data, name, bins_size):\n    title = \"Number of missing '{}' values during skeleton sequences\".format(name)\n    fig,ax = plt.subplots(figsize=(10, 3))\n    counts, bins, bars = ax.hist(data, bins=bins_size)\n    original = counts\n    ax.set_title(title)\n    plt.show()\n\n    fig,ax = plt.subplots(figsize=(10, 3))\n    \n    kernel = [1/16, 4/16, 6/16, 4/16, 1/16]\n    # kernel = [1/4, 2/4, 1/4]\n    for _ in range(20):\n        counts = conv1D(counts, kernel)\n    \n#     og = ax.bar(np.arange(len(original))/len(original) + 0.01, original, width=0.02)\n#     est = ax.bar(np.arange(len(counts))/len(counts) + 0.01, counts, width=0.014, alpha=0.5)\n    og = ax.bar(np.arange(len(original))/len(original), original, width=0.005)\n    est = ax.bar(np.arange(len(counts))/len(counts), counts, width=0.005, alpha=0.6)\n    \n    \n    ax.text(0.03, -0.16, 'Start', transform=ax.transAxes)\n    ax.text(0.93, -0.16, 'End', transform=ax.transAxes)\n    \n    og.set_label(\"Histogram of NaNs\")\n    est.set_label(\"Estimated distribution\")\n    \n    \n    ax.set_ylabel('NaN count')\n    ax.set_xlabel('Time (normalised duration of sequence)')\n    \n    ax.legend()\n    ax.set_title(title)\n    plt.show()\n\n\nbins_size = 200\nplot_hist(right_hand_pp, \"right hand\", bins_size)\nplot_hist(left_hand_pp, \"left hand\", bins_size)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:58.596555Z","iopub.status.idle":"2023-04-27T21:47:58.596945Z","shell.execute_reply.started":"2023-04-27T21:47:58.596754Z","shell.execute_reply":"2023-04-27T21:47:58.596774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fully_empty = 0\npartially_empty = 0\n\nfor idx, file_path in enumerate(tqdm(train['path'])):\n    parquet = pd.read_parquet(\"/kaggle/input/asl-signs/\" + file_path)\n    lhand = parquet[parquet[\"type\"] == \"left_hand\"]\n    rhand = parquet[parquet[\"type\"] == \"right_hand\"]\n    \n    lhand_i0 = list(lhand[lhand[\"landmark_index\"] == 0]['x'])\n    rhand_i0 = list(rhand[rhand[\"landmark_index\"] == 0]['x'])\n    \n    isEm = False\n    if np.isnan(lhand_i0).all() and np.isnan(rhand_i0).all():\n        fully_empty += 1\n    else: \n        for index in range(len(lhand_i0)):\n            lentry = lhand_i0[index]\n            rentry = rhand_i0[index]\n            if math.isnan(lentry) and math.isnan(rentry):\n                isEm = True \n            \n        if isEm:\n            partially_empty += 1\n        \nprint(\"Fully Empty:\", fully_empty)\nprint(\"Partially Empty:\", partially_empty)\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-27T21:47:58.597842Z","iopub.status.idle":"2023-04-27T21:47:58.598204Z","shell.execute_reply.started":"2023-04-27T21:47:58.598023Z","shell.execute_reply":"2023-04-27T21:47:58.598043Z"},"trusted":true},"execution_count":null,"outputs":[]}]}