{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport os\nimport glob\ndir = '/kaggle/input/asl-signs'\ntrain = pd.read_csv(f'{dir}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:28.240146Z","iopub.execute_input":"2023-04-24T20:12:28.240616Z","iopub.status.idle":"2023-04-24T20:12:29.256874Z","shell.execute_reply.started":"2023-04-24T20:12:28.240577Z","shell.execute_reply":"2023-04-24T20:12:29.255741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\n\nparticipants = os.listdir(LANDMARK_FILES_DIR)\n\nprint(f\"Total number of participants = {len(participants)}\")\nprint(f\"Average number of sequences per participant = {len(glob.glob(LANDMARK_FILES_DIR + '/*/*.parquet'))/len(participants)}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:29.259234Z","iopub.execute_input":"2023-04-24T20:12:29.259734Z","iopub.status.idle":"2023-04-24T20:12:33.250518Z","shell.execute_reply.started":"2023-04-24T20:12:29.259684Z","shell.execute_reply":"2023-04-24T20:12:33.249036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:33.252364Z","iopub.execute_input":"2023-04-24T20:12:33.252958Z","iopub.status.idle":"2023-04-24T20:12:33.293265Z","shell.execute_reply.started":"2023-04-24T20:12:33.252899Z","shell.execute_reply":"2023-04-24T20:12:33.291908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:33.295760Z","iopub.execute_input":"2023-04-24T20:12:33.296136Z","iopub.status.idle":"2023-04-24T20:12:33.334454Z","shell.execute_reply.started":"2023-04-24T20:12:33.296102Z","shell.execute_reply":"2023-04-24T20:12:33.332795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:33.336133Z","iopub.execute_input":"2023-04-24T20:12:33.336551Z","iopub.status.idle":"2023-04-24T20:12:33.344533Z","shell.execute_reply.started":"2023-04-24T20:12:33.336512Z","shell.execute_reply":"2023-04-24T20:12:33.342908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().head(40).sort_values(ascending=True).plot( kind=\"barh\", ax=ax, title=\"Top 40 Signs in Training Dataset\")\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:33.346497Z","iopub.execute_input":"2023-04-24T20:12:33.347129Z","iopub.status.idle":"2023-04-24T20:12:33.939896Z","shell.execute_reply.started":"2023-04-24T20:12:33.347090Z","shell.execute_reply":"2023-04-24T20:12:33.938887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\ntrain[\"sign\"].value_counts().tail(40).sort_values(ascending=True).plot( kind=\"barh\", ax=ax, title=\"Bottom 40 Signs in Training Dataset\",color='orange')\nax.set_xlabel(\"Number of Training Examples\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:33.941054Z","iopub.execute_input":"2023-04-24T20:12:33.942129Z","iopub.status.idle":"2023-04-24T20:12:34.424979Z","shell.execute_reply.started":"2023-04-24T20:12:33.942087Z","shell.execute_reply":"2023-04-24T20:12:34.423646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\ndef read_json(path):\n    with open(path, \"r\") as file:\n        json_data = json.load(file)\n    return json_data","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:34.426468Z","iopub.execute_input":"2023-04-24T20:12:34.427508Z","iopub.status.idle":"2023-04-24T20:12:34.433549Z","shell.execute_reply.started":"2023-04-24T20:12:34.427467Z","shell.execute_reply":"2023-04-24T20:12:34.432039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s2p_map = read_json(os.path.join(dir, \"sign_to_prediction_index_map.json\"))\np2s_map = {v: k for k, v in s2p_map.items()}\n\nencoder = lambda x: s2p_map.get(x)\ndecoder = lambda x: p2s_map.get(x)\n\ntrain[\"label\"] = train[\"sign\"].map(encoder)\nprint(f\"shape = {train.shape}\")\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:34.434765Z","iopub.execute_input":"2023-04-24T20:12:34.435103Z","iopub.status.idle":"2023-04-24T20:12:34.501398Z","shell.execute_reply.started":"2023-04-24T20:12:34.435070Z","shell.execute_reply":"2023-04-24T20:12:34.499910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sequence Landmarks Data¶\nLets have a look at the dataframe of one sample sequence!","metadata":{}},{"cell_type":"code","source":"example_fn = train.query('sign == \"listen\"')[\"path\"].values[0]\n\nexample_fn","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:34.507061Z","iopub.execute_input":"2023-04-24T20:12:34.507895Z","iopub.status.idle":"2023-04-24T20:12:34.526372Z","shell.execute_reply.started":"2023-04-24T20:12:34.507823Z","shell.execute_reply":"2023-04-24T20:12:34.524968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark = pd.read_parquet(f\"{dir}/{example_fn}\")\nexample_landmark.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:34.527760Z","iopub.execute_input":"2023-04-24T20:12:34.528245Z","iopub.status.idle":"2023-04-24T20:12:34.661015Z","shell.execute_reply.started":"2023-04-24T20:12:34.528176Z","shell.execute_reply":"2023-04-24T20:12:34.660115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_frames = example_landmark[\"frame\"].nunique()\nunique_types = example_landmark[\"type\"].nunique()\ntypes_in_video = example_landmark[\"type\"].unique()\nprint(f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:34.662573Z","iopub.execute_input":"2023-04-24T20:12:34.663238Z","iopub.status.idle":"2023-04-24T20:12:34.673447Z","shell.execute_reply.started":"2023-04-24T20:12:34.663177Z","shell.execute_reply":"2023-04-24T20:12:34.671744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets Compare for a bunch of parquet files what type of data we have.¶\nWe notice the number of frames is not consistent\nAlmost every file has 4 types of landmarks: face, left_hand, pose and right_hand.","metadata":{}},{"cell_type":"code","source":"listen_files = train.query('sign == \"listen\"')[\"path\"].values\nfor i, f in enumerate(listen_files):\n    example_landmark = pd.read_parquet(f\"{dir}/{f}\")\n    unique_frames = example_landmark[\"frame\"].nunique()\n    unique_types = example_landmark[\"type\"].nunique()\n    types_in_video = example_landmark[\"type\"].unique()\n    print(\n        f\"The file has {unique_frames} unique frames and {unique_types} unique types: {types_in_video}\"\n    )\n    if i == 20:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:34.675581Z","iopub.execute_input":"2023-04-24T20:12:34.675991Z","iopub.status.idle":"2023-04-24T20:12:35.307986Z","shell.execute_reply.started":"2023-04-24T20:12:34.675955Z","shell.execute_reply":"2023-04-24T20:12:35.306981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Metadata for Training Dataset","metadata":{}},{"cell_type":"code","source":"N_PARQUETS_TO_READ = 40000 # So we don't have to load all 95k\n\ncombined_meta = {}\nfor i, d in tqdm(train.iterrows(), total=len(train)):\n    file_path = d[\"path\"]\n    example_landmark = pd.read_parquet(f\"{dir}/{file_path}\")\n    # Get the number of landmarks with x,y,z data per type\n    meta = (\n        example_landmark.dropna(subset=[\"x\", \"y\", \"z\"])[\"type\"].value_counts().to_dict()\n    )\n    meta[\"frames\"] = example_landmark[\"frame\"].nunique()\n    xyz_meta = (\n        example_landmark.agg(\n            {\n                \"x\": [\"min\", \"max\", \"mean\"],\n                \"y\": [\"min\", \"max\", \"mean\"],\n                \"z\": [\"min\", \"max\", \"mean\"],\n            }\n        )\n        .unstack()\n        .to_dict()\n    )\n\n    for key in xyz_meta.keys():\n        new_key = key[0] + \"_\" + key[1]\n        meta[new_key] = xyz_meta[key]\n    combined_meta[file_path] = meta\n    if i >= N_PARQUETS_TO_READ:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:12:35.309348Z","iopub.execute_input":"2023-04-24T20:12:35.309895Z","iopub.status.idle":"2023-04-24T20:38:34.450759Z","shell.execute_reply.started":"2023-04-24T20:12:35.309860Z","shell.execute_reply":"2023-04-24T20:38:34.449293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_with_meta = train.merge(pd.DataFrame(combined_meta).T.reset_index().rename(columns={\"index\": \"path\"}),how=\"left\",)\ntrain_with_meta.to_parquet(\"train_with_meta.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:34.453569Z","iopub.execute_input":"2023-04-24T20:38:34.453951Z","iopub.status.idle":"2023-04-24T20:38:36.588247Z","shell.execute_reply.started":"2023-04-24T20:38:34.453916Z","shell.execute_reply":"2023-04-24T20:38:36.587072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What are the most frequent types of landmarks provided?<br>\nFace has a lot more datapoints because mediapipe provides 468 3D datapoints per frame.","metadata":{}},{"cell_type":"code","source":"train_with_meta[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]].sum().sort_values().plot(kind=\"barh\", title=\"Sum of Rows by Landmark Type\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:36.590039Z","iopub.execute_input":"2023-04-24T20:38:36.590425Z","iopub.status.idle":"2023-04-24T20:38:36.791756Z","shell.execute_reply.started":"2023-04-24T20:38:36.590388Z","shell.execute_reply":"2023-04-24T20:38:36.790297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Every parquet file has at least some datapoints for all four types of landmarks:\n\n- Face, pose, left hand and right hand.","metadata":{}},{"cell_type":"code","source":"# checking to see if the number of landmarks for this type is zero\n(train_with_meta.query(\"index < 1000\").fillna(0)[[\"face\", \"pose\", \"left_hand\", \"right_hand\"]]> 0).mean().plot(kind=\"barh\", title=\"Rate of Frame/Keypoints with Data\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:36.793290Z","iopub.execute_input":"2023-04-24T20:38:36.793650Z","iopub.status.idle":"2023-04-24T20:38:37.012549Z","shell.execute_reply.started":"2023-04-24T20:38:36.793615Z","shell.execute_reply":"2023-04-24T20:38:37.009984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_fn = train_with_meta.dropna().query('sign == \"shhh\"')[\"path\"].values[0]\nexample_landmark = pd.read_parquet(f\"{dir}/{example_fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:37.013853Z","iopub.execute_input":"2023-04-24T20:38:37.014167Z","iopub.status.idle":"2023-04-24T20:38:37.061103Z","shell.execute_reply.started":"2023-04-24T20:38:37.014136Z","shell.execute_reply":"2023-04-24T20:38:37.060037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.query(\"frame == 25\")[\"type\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:37.063143Z","iopub.execute_input":"2023-04-24T20:38:37.064422Z","iopub.status.idle":"2023-04-24T20:38:37.078938Z","shell.execute_reply.started":"2023-04-24T20:38:37.064366Z","shell.execute_reply":"2023-04-24T20:38:37.077553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"no_xyz\"] = example_landmark[\"x\"].isna()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:37.080726Z","iopub.execute_input":"2023-04-24T20:38:37.081292Z","iopub.status.idle":"2023-04-24T20:38:37.088483Z","shell.execute_reply.started":"2023-04-24T20:38:37.081244Z","shell.execute_reply":"2023-04-24T20:38:37.087165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark.groupby(\"frame\")[\"no_xyz\"].sum().plot(title=\"missing xyz per frame\", kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:37.089965Z","iopub.execute_input":"2023-04-24T20:38:37.090332Z","iopub.status.idle":"2023-04-24T20:38:37.431934Z","shell.execute_reply.started":"2023-04-24T20:38:37.090300Z","shell.execute_reply":"2023-04-24T20:38:37.430942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3D plot of Landmarks from \"shhh\" example\nPick frame 17 because we have no missing xyz data","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\n\nexample_frame = example_landmark.query(\"frame == 17\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:37.433512Z","iopub.execute_input":"2023-04-24T20:38:37.434634Z","iopub.status.idle":"2023-04-24T20:38:41.822068Z","shell.execute_reply.started":"2023-04-24T20:38:37.434587Z","shell.execute_reply":"2023-04-24T20:38:41.820677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark[\"y_\"] = example_landmark[\"y\"] * -1\nexample_frame = example_landmark.query(\"frame == 17 and type== 'face'\")\npx.scatter(example_frame, x=\"x\", y=\"y_\", color=\"type\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:41.825450Z","iopub.execute_input":"2023-04-24T20:38:41.826251Z","iopub.status.idle":"2023-04-24T20:38:41.930941Z","shell.execute_reply.started":"2023-04-24T20:38:41.826192Z","shell.execute_reply":"2023-04-24T20:38:41.929798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Try to draw the example with mediapipe's hand connections?","metadata":{}},{"cell_type":"code","source":"!pip install mediapipe --quiet","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:41.934311Z","iopub.execute_input":"2023-04-24T20:38:41.935454Z","iopub.status.idle":"2023-04-24T20:38:57.300750Z","shell.execute_reply.started":"2023-04-24T20:38:41.935396Z","shell.execute_reply":"2023-04-24T20:38:57.299095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:57.303330Z","iopub.execute_input":"2023-04-24T20:38:57.303878Z","iopub.status.idle":"2023-04-24T20:38:58.144128Z","shell.execute_reply.started":"2023-04-24T20:38:57.303817Z","shell.execute_reply":"2023-04-24T20:38:58.143037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\", \"right_hand\",\"face\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:58.145811Z","iopub.execute_input":"2023-04-24T20:38:58.146463Z","iopub.status.idle":"2023-04-24T20:38:58.831496Z","shell.execute_reply.started":"2023-04-24T20:38:58.146423Z","shell.execute_reply":"2023-04-24T20:38:58.830529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"left_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:58.833095Z","iopub.execute_input":"2023-04-24T20:38:58.833749Z","iopub.status.idle":"2023-04-24T20:38:59.182966Z","shell.execute_reply.started":"2023-04-24T20:38:58.833712Z","shell.execute_reply":"2023-04-24T20:38:59.181620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nexample_landmark[\"y_\"] = example_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [ \"right_hand\"]:\n    example_hand = example_landmark.query(\"frame == 17 and type == @hand\")\n\n    ax.scatter(example_hand[\"x\"], example_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = example_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = example_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"purple\")\nax.set_title(\"Shhh - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.193544Z","iopub.execute_input":"2023-04-24T20:38:59.194225Z","iopub.status.idle":"2023-04-24T20:38:59.558543Z","shell.execute_reply.started":"2023-04-24T20:38:59.194171Z","shell.execute_reply":"2023-04-24T20:38:59.557155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listen_fn = train_with_meta.query('sign == \"listen\"')[\"path\"].values[0]\nlisten_landmark = pd.read_parquet(f\"{dir}/{listen_fn}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.560116Z","iopub.execute_input":"2023-04-24T20:38:59.560501Z","iopub.status.idle":"2023-04-24T20:38:59.583966Z","shell.execute_reply.started":"2023-04-24T20:38:59.560467Z","shell.execute_reply":"2023-04-24T20:38:59.582763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listen_landmark.head(100)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.585407Z","iopub.execute_input":"2023-04-24T20:38:59.586276Z","iopub.status.idle":"2023-04-24T20:38:59.611253Z","shell.execute_reply.started":"2023-04-24T20:38:59.586238Z","shell.execute_reply":"2023-04-24T20:38:59.609898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listen_landmark.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.612690Z","iopub.execute_input":"2023-04-24T20:38:59.613125Z","iopub.status.idle":"2023-04-24T20:38:59.619935Z","shell.execute_reply.started":"2023-04-24T20:38:59.613089Z","shell.execute_reply":"2023-04-24T20:38:59.618617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listen_landmark.tail(700)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.621710Z","iopub.execute_input":"2023-04-24T20:38:59.622149Z","iopub.status.idle":"2023-04-24T20:38:59.643614Z","shell.execute_reply.started":"2023-04-24T20:38:59.622105Z","shell.execute_reply":"2023-04-24T20:38:59.642170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\nlisten_frame = example_landmark.query(\"frame == 40\")\npx.scatter_3d(example_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.645295Z","iopub.execute_input":"2023-04-24T20:38:59.646051Z","iopub.status.idle":"2023-04-24T20:38:59.930662Z","shell.execute_reply.started":"2023-04-24T20:38:59.646010Z","shell.execute_reply":"2023-04-24T20:38:59.929390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listen_landmark[\"y_\"] = listen_landmark[\"y\"] * -1\nlisten_frame = listen_landmark.query(\"frame == 40 and type== 'face'\")\npx.scatter(listen_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:38:59.932315Z","iopub.execute_input":"2023-04-24T20:38:59.932780Z","iopub.status.idle":"2023-04-24T20:39:00.000832Z","shell.execute_reply.started":"2023-04-24T20:38:59.932721Z","shell.execute_reply":"2023-04-24T20:38:59.999617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nlisten_landmark[\"y_\"] = listen_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in ['face','left_hand',\"right_hand\"]:\n    listen_hand = listen_landmark.query(\"frame == 40 and type == @hand\")\n\n    ax.scatter(listen_hand[\"x\"],listen_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = listen_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = listen_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"green\")\nax.set_title(\"Listen - Hands Data\")\nplt.show()\n\n###data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxITEBUTEhIVEBUVFRIVGBUWEBUVDxAVGBUWGBUVFRUYHSggGB0lGxYVITEhJSkrLi4uFx8zODMtNygtLisBCgoKDg0OFRAQFS0ZFR0tLSstKystKystLS0rKy0tLSsrLS03Nys3LTctLS0rLS0tNystLSs3LS03LS0tLS0rLf/AABEIAOEA4QMBIgACEQEDEQH/xAAcAAEAAQUBAQAAAAAAAAAAAAAABgECBAUHAwj/xABCEAACAQIDBQQHBQYFBAMAAAAAAQIDEQQFEiExUWFxBkGBkQcTIjKhscEjQlJyghRikqLR4TRDs/DxdIOywhUkU//EABgBAQADAQAAAAAAAAAAAAAAAAABAgME/8QAHhEBAQEBAQEBAQADAAAAAAAAAAECEQMxIRIyQVH/2gAMAwEAAhEDEQA/AO4gAAAAAAAAAAAAAB4YzF06UHOrONOC3yk0orxYHuCEY/0l4aLtSp1K/O3q4eGrb8DXQ9K0LrVhmlfa41lJpck4q/miP6g6QDVZD2hw+Li5UJ6nG2qDVqkL8Y/VbDakgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFlWooxcpNRjFNtvckldtgaftV2kpYKlqn7c5XUKaftTfe+UVsu/qcgzXOK+Mqa60tVvdgtlKmuEY/V3fM1/afOZYnE1K0m2pSagn9ymm9EUu7Z8WzBwleV0o3bfDa2Y60tIzq2EbMeWCaW0luU9n5ySlWlpv92PveL7jeU8lpJe5fqZ3TaefXOcvxFWjNVKUpU5R3Sjs8+K5PYdq7GdqY4uGmdoVoq8o901u1x+q7rkengadraI+RoswwMqDVajJwcXsadnG+zy5Fs+iuvLjsYNV2WzKWIwlOrO2qSalbdqjJxbS7r2vbmbU6GIAAAAAAAAAAAAAAAAAAAAAAAAAABDvSlmbpYHRF2lXkqfPRZyn8El+omJyf0s4hyxdOnsahS1c05yd7+EIldX8HNMTJ6rImvYzLIwWuSvJ9/wCFciL4XDasTG+52+bX0J/h6bStF6F3vvS5czn0385/tuIzV968zMpUbq+wi9PF4VT0JuUut3fmb/BYi1tO4rxt1fXhY1GOxEJJ02mtSau1sv3bTbZhUtz6Gj/+dT3QlKF1GT0+zG97X7+7gJEare+ifFSlh6tOX+XV2ctS2rzTfiTkgfo7j6vFYyn3S9VViuT1KX0J4dWfjkv0ABZAAAAAAAAAAAAAAAAAAAAAAGNjsbClG8vBL3pPgi/GVXGnKUVdxi2lxsQLMcfV1anH1mp2e3dyXApvf8r4x/SSYDtHrqqEoadTsmp3afNW+JybtHjJVcZWqT3+snG17qKg9CXlH5k8yrHUaFWdWrphJUZOCb9qpb2pRhfZq2bt7ucyxFdznKct85Sk+sm2/iyn9didSTXFmVO9WPGMn5P+9zoFGnGULPvRGK+TxpUFVeybcem3uNpk+LvsZlW+M8+thgsmjGpGUdK0e69G1dbOzfNo2VSNj0wtRHnmyajG33nb6kL84OV7HhHK6aepXtfVpv7OrjbiW0cRFw3Sv+Vr4WM+W5dEwWStdk+K0ZvTX/6UnB/zNfGK8zpBxTFY7TmcZr/L9U+XsyUnfzO1nR5/HJ6fQAGigAAAAAAAAAAAAAAAAAAAAAMhGc4dU6zp8bSS5P8AumvAmtSajFye5Jt9FvIDicXrnKrO95bkvux7o/777mXrzjXx71j5vlqrUnB7Hvi/wvuIbgsoquvGE4Sik05O3s2XPvJ1hsypy+8lxUtjXmemx7VZrjcwl46LiaqO9scQoQpxtqu3Jq9rpLdfxNb2fxMakVJbOK70U7ZVr14rujFJ9W/+COZDmDo4iUX7rk/DmWk/FNa5p1DCMza1VafaasuO412CrKSTXeeuIoKdr93l5FWveqvMaVkrrZ37LGRKWy9zDWE47uSMbPsaqVCTWx20xXN7F5b/AACNXiDKvqxU5filO3S+z4Hc+zGN9dhKU27vTpl+aPsv5X8TgdKOmUXzR2b0cVL4SSvuqysuCcYv5tm3nf1ya/6lYANlAAAAAAAAAAAAAAAAAAAAABbVgpRcXuaafRkIq4bROVOW+Lt1W9PxROSL9pZU3Xgozj6zQ9UVJa1FNOLa3r3n1M/WdjTy1zTUzw0Xvin4FtKhGmnp2LhfYui7jJs+BqM+quNGW3a7RXj/AGOZ2W/iIZz9pOpNbU5bOi2I0lXCfaKXFbfl/Qk1DDXpy6MwcMk5JPoy2a59TrJyvGVKNl70PiuhJ8Jm0JLa7fMwcvwq91q5uKGXwX3RV89i5YvV7quQztBWlOvJN3UUkl3J7Lsn9PDpLYrECzWNq8+v1ENfGBiKVl8Tono3zOnFShKSi56XG7SUmk7rruIJWXu9GUwtB1Kcqate+wtm8rLU/He0yp87YDtTj8HJwpV5RUXZ05KNSns3pKV9K/K0TbIvS2m1HF0dO69Wk211dN7bdG+h0dYddTBi5dmFKvTVSjUjVhLdKLuuafB8ntMolIAAAAAAAAAAAAAAAAAANF21z39iwNWura0tNNPapVJbIJrvV9r5JnJvRrOdXE4jEVZOc56dUnvlJtt/Cyt3KxsvTtm96lDCL7ideXWWqEF4JT/iR7+jXL9GFjLvqNzfyXwM/S/i3nO6TGtuIf2qr7YQ5t/RfUl+JewgmeS14lLg4x+r+Zz11deuGp/ZyfJmhqq0rriSxU7U5r9xkWrL25Lp8imarUoyXEKVr70SelTViBZBilCpZ/e2eJMaNZroXaSdjPm1bYc2ziadedu6X1Jtj8Vpptrezm9OvqqSu73f1JU0ypvbHoemUStOT4HjOXtdEMqn9pKPFEs6j+Pbc5N97b83cxkjMxsbTkubMS505+OXX1t+zfaGvgqvrKMtj9+m7ulVXCS48Gtq+B9A5FmkMVh6den7s1ez3xadpRfRprwPmdM6V6IO0bhUeDqNaKmqdO+9VLe1FPg0m+qfEklddAAXAAAAAAAAAAAAAAA1fajNP2bB16/fTpycec3sgv4mgODdopSx2c1t7Uq0oK692nS9jwvov1kdWyjDqEVFbEkkvAgHo7ytvViJ3cpNpN96vtl4s6VhY2Rz7va28p+dWY2WwhHq719b75Sf9CaY5+y+hFMNSuk+ZjutWbB7+cWiL4qNqz8fgyS32J9CNZw7T5tzfxK4NNbWxOn2l92afhtJTgO1uGkkpTdN9+qLt5ohGMeyXU1kzeZ6zu7n46NnfaTDum1Tqa5We5O3myHZdP7S/U1lNmbgHZsXPEf3a2Sq3lLoW4WVqsHxX1PDCv3/ABMrB072ffGS8n/dEJYecxtWl5/M1bJD2qoWnF8Y28iPM6MXsc25+lzJy/GSo1YVYe9TnGa4Xi00ny2GKXIuo+m8lzKOJw9OvDZGpFSs98X3xfR3XgZpzL0L5xenVwsntg/W003917JpdJWf62dNIaS9gAAkAAAAAAAAAAA5x6ZMa5U6GChfVXqa5bNihTtv/VKL/SdHPnrtB2ndXN5YiUtVKnUdOFvdVGLcbx/NtnfvuiL8HQsnwap04wSskkjd01sMDBTUoqUWnGSTTW5o2C3HNXVPjDx/uvoyP5ctnkSDGPYyN4b3brqZei0UnK0ZX/E/mRjOKntp8/mSTGPY+cyLZvH2nzk/JJIYRpgZnTtt4mslG5usXTc6KffHf04msVPd1N4x0xY7jLwT2mM0e+DlaSJqsbPCU9s1yMzI17Vn3r5f8FtCH2k+cb+aPTKItTi+dilaRkdsaVqdN8/miGzJ92yp/wD1ovhJfU59Nmvj/iw9f8iLL0WRL0bMm77JZs8LjKNa9oxklPnTlsn8HfwR9HI+WYH0P2EzF18voTfvKGiTe9yptwb8dN/EVbNb8AELgAAAAAAAAAAj3pAzJ4fLcRUT0ycNEX3qVRqCa6ar+B82o7P6da8lhKEE7RlWblz005WT87+Bxa5FVqWdj+10sK1TqJzot/qp3748VyOpYXOcPUpqpCrBxffqStxunuOAgpcyr59LHUO1Hbakoyp4d+sm04617kOa/E/gZuQS10It/hXw2fQ5PS3o6r2XdqCT7thh7Z5G3nq6r1qUfb27opyfUi+aQvPp9dv1JNiKr0yf43b9KNLWpb31M8r1h5bG8p03tunbnsNXiqOm3U2M56K0ZLu0v+p4dp2o13FbLXfntNsstNLUV79X8ytCO0xYVN/PabLCw2ovqcUzet/gKWpxlxi4+Kf9DJy3C/yyLslpbNLXCS+KNtgaVtX5l9TDVbxjdrad8FJ8HF/FHNKh1vtFQ1YSqv3b+W36HJaqN/C/jn9vpEuR5xZejoYvSLO0ehnE6sHVh+Cs7cbShF/NM4smdR9CWK9vE0+MaU1x9lyi/wDyiRU5+urgAhoAAAAAAAAAADnXpywkpZfTmrWpYiEpcVGUJw2fqlA4cfVuZYCnXozo1YqcKkXGSa7n9VvT5Hy/neV1MLiKmHqe9Tk43tZTW+M1ycWn4kVXTCFwUIVZmW0tVRLmdPy6Oiil37fN7F8/gc67N7cRHxfkdIpztpT438rv+hz+zp8VmPqpPT3Qjd9XuMGu/sm+nxZi5titsn+KcV4Qh/Wx74qVqdKP4nF+BTOV7prM1smm391W63a+hqO09fViJ/pXlFHpm+K1OEfwqSfX1k38mjW5pO9ao/35fM6MZYb0xESDJoqS/Lf+xHzc9man26i90k112bC+5+K5v6l2XStKPda6/n/ub6pStdrvV/8AfmRGrVtOK4wqPyeomdOopU1Lua8rnJucdOapWkpUeqs/qcer09MpR/C2vJ2Oo4TE+3VpcGpLmnsZzfPI2xFX8835u5t4/nYx9mtgz1R4U2esWdLB6onXofxKjmOl/wCZRqwXVOE/lBkERIuwOLVLMcNJ7vWaOmtOC+MkQT6+iAAQ1AAAAAAAAAAAI52v7GYbHxXrE4VIq0asPfiuDW6UeT8LEjAHFn6GcTrt+1UdH4/Vz1r/ALd7P+I2+D9DFFL7XF1Zv9ynCnHylqfxOpHnXqqEXKTsopyb7kkrthHI4RjeziwGZTpKp6yPqtcG17embcUpc1Z9dj2bllY3FtNbdzf0NZnfbNYvGSruLpw0RhThKzlGKera13uTb5XsabF5xqd1cx1ntWzrkZuYYi7X5mvkZmaY77aCT92EV5pkceNu7vdsfijwqYqTk5Mt/CP6ZFet7ae+0m/5nsMapO7b4tvzLKlS5aaSKdeiZkYHE6KkZfhlF+TMQDh1J8Tilrvfc526SRLclzOPqFeS2R2+D/ucv9a+Jf6+Vraml1KXzlXm+JdPOVDGTad47bcO5kezmup1ZyW1NswbhlpmRTWrXgt56HjOXtHpEsq9Ys2vZr/GYb/qKH+pE1MTb9mIt43DW2v9oof6kSUPpcAFWwAAAAAAAAAAAAAEG9MObepy2UE7SryVJcdPvVP5Vb9ROTjHp1zLVXoYdf5cJVZcLzemPkoS/iQRr45cVuLFAzXAoVCVSpbcqmSLrlUW3CYF6KosLkSLrlJMWD2ID0yzLalepppq75uy8zc4nsjjKcdTouceMGpW8E7/AAJD6O8utT1v7zudHoRM7rjXPn2OAabOzVvmSv0a5ZKtmNFx3Un66T4KO7xcnFf8El9ImQKdL9opxSnD37KznDi+LX9TcehzJnTw08RJWddpR2bdELq/jJvyRbvYpccvHQgAFgAAAAAAAAAAAAAPnH0m4/12a4h3uoSjSXJU4pSX8evzZ37P8zjhsNVryaSpwlLa7KUrezHq5WS6ny1WquUpSk7yk3KT4ybu35sK6WlSguFFbAXKpkpLCxW6F1xAWK2KalxLrriARcWpriV1LiSLkLXaXFpFNS4mfkGG9biYRW1J3f0IvwjqfZbB6KMVyRJabMHL6OmKMtsxrrzORWrh/W/Zr76ceVmtrfgSbDUI04RhBaYwioxS3JJWS8jSdnq0JVakb+3BQ2cFK+1fw/7uSAvmM93tAAWUAAAAAAAAAAAAAEe7ef4Gp+al/qROVMqArVC4AIAASgAAAAAVKAAVL8s/xUPyv5oqCKnP10CluLpAGLsWdjv8TiPD/wBSYAGsc1+gAJAAAAAB/9k=","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:00.002515Z","iopub.execute_input":"2023-04-24T20:39:00.002874Z","iopub.status.idle":"2023-04-24T20:39:00.680961Z","shell.execute_reply.started":"2023-04-24T20:39:00.002839Z","shell.execute_reply":"2023-04-24T20:39:00.679727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nlook_fn = train.query('sign == \"look\"')[\"path\"].values[0]\n\nlook_landmark = pd.read_parquet(f\"{dir}/{look_fn}\")\nprint(f\"Sample shape = {look_landmark.shape}\")\nlook_landmark.sample(300)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:00.682699Z","iopub.execute_input":"2023-04-24T20:39:00.683151Z","iopub.status.idle":"2023-04-24T20:39:00.737079Z","shell.execute_reply.started":"2023-04-24T20:39:00.683104Z","shell.execute_reply":"2023-04-24T20:39:00.735677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport plotly.express as px\nlook_frame = look_landmark.query(\"frame == 40\")\npx.scatter_3d(look_frame, x=\"x\", y=\"y\", z=\"z\", color=\"type\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:00.738889Z","iopub.execute_input":"2023-04-24T20:39:00.740000Z","iopub.status.idle":"2023-04-24T20:39:00.820718Z","shell.execute_reply.started":"2023-04-24T20:39:00.739947Z","shell.execute_reply":"2023-04-24T20:39:00.819474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listen_landmark[\"y_\"] = listen_landmark[\"y\"] * -1\nlisten_frame = listen_landmark.query(\"frame == 40 and type== 'face'\")\npx.scatter(listen_frame, x=\"x\", y=\"y_\", color=\"type\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:00.822312Z","iopub.execute_input":"2023-04-24T20:39:00.822931Z","iopub.status.idle":"2023-04-24T20:39:00.890988Z","shell.execute_reply.started":"2023-04-24T20:39:00.822893Z","shell.execute_reply":"2023-04-24T20:39:00.889711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nlook_landmark[\"y_\"] = look_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in [\"right_hand\"]:\n    look_hand = look_landmark.query(\"frame == 30 and type == @hand\")\n\n    ax.scatter(look_hand[\"x\"],look_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = look_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = look_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"green\")\nax.set_title(\"Listen - Hands Data\")\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:00.892567Z","iopub.execute_input":"2023-04-24T20:39:00.892911Z","iopub.status.idle":"2023-04-24T20:39:01.244404Z","shell.execute_reply.started":"2023-04-24T20:39:00.892880Z","shell.execute_reply":"2023-04-24T20:39:01.243146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nlook_landmark[\"y_\"] = look_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in ['face','left_hand',\"right_hand\"]:\n    look_hand = look_landmark.query(\"frame == 31 and type == @hand\")\n\n    ax.scatter(look_hand[\"x\"],look_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = look_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = look_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"green\")\nax.set_title(\"Look - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:01.246241Z","iopub.execute_input":"2023-04-24T20:39:01.246612Z","iopub.status.idle":"2023-04-24T20:39:01.888149Z","shell.execute_reply.started":"2023-04-24T20:39:01.246576Z","shell.execute_reply":"2023-04-24T20:39:01.886923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mediapipe as mp\n\nmp_hands = mp.solutions.hands\n\n\nlook_landmark[\"y_\"] = look_landmark[\"y\"] * -1\n\nfig, ax = plt.subplots(figsize=(5, 5))\n\nfor hand in ['face','left_hand',\"right_hand\"]:\n    look_hand = look_landmark.query(\"frame == 32 and type == @hand\")\n\n    ax.scatter(look_hand[\"x\"],look_hand[\"y_\"])\n\n    for connection in mp_hands.HAND_CONNECTIONS:\n        point_a = connection[0]\n        point_b = connection[1]\n        x1, y1 = look_hand.query(\"landmark_index == @point_a\")[[\"x\", \"y_\"]].values[0]\n        x2, y2 = look_hand.query(\"landmark_index == @point_b\")[[\"x\", \"y_\"]].values[0]\n        plt.plot([x1, x2], [y1, y2], color=\"green\")\nax.set_title(\"Look - Hands Data\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:01.889520Z","iopub.execute_input":"2023-04-24T20:39:01.890651Z","iopub.status.idle":"2023-04-24T20:39:02.544883Z","shell.execute_reply.started":"2023-04-24T20:39:01.890613Z","shell.execute_reply":"2023-04-24T20:39:02.543511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#path_to_sign = 'train_landmark_files/16069/1011655866.parquet'\npath_to_sign=train.query('sign == \"look\"')[\"path\"].values[0]\nsign = pd.read_parquet(f'{dir}/{path_to_sign}')\nsign.y = sign.y * -1","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:02.546403Z","iopub.execute_input":"2023-04-24T20:39:02.546782Z","iopub.status.idle":"2023-04-24T20:39:02.569813Z","shell.execute_reply.started":"2023-04-24T20:39:02.546745Z","shell.execute_reply":"2023-04-24T20:39:02.568693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hand_points(hand):\n    x = [[hand.iloc[0].x, hand.iloc[1].x, hand.iloc[2].x, hand.iloc[3].x, hand.iloc[4].x], # Thumb\n         [hand.iloc[5].x, hand.iloc[6].x, hand.iloc[7].x, hand.iloc[8].x], # Index\n         [hand.iloc[9].x, hand.iloc[10].x, hand.iloc[11].x, hand.iloc[12].x], \n         [hand.iloc[13].x, hand.iloc[14].x, hand.iloc[15].x, hand.iloc[16].x], \n         [hand.iloc[17].x, hand.iloc[18].x, hand.iloc[19].x, hand.iloc[20].x], \n         [hand.iloc[0].x, hand.iloc[5].x, hand.iloc[9].x, hand.iloc[13].x, hand.iloc[17].x, hand.iloc[0].x]]\n\n    y = [[hand.iloc[0].y, hand.iloc[1].y, hand.iloc[2].y, hand.iloc[3].y, hand.iloc[4].y],  #Thumb\n         [hand.iloc[5].y, hand.iloc[6].y, hand.iloc[7].y, hand.iloc[8].y], # Index\n         [hand.iloc[9].y, hand.iloc[10].y, hand.iloc[11].y, hand.iloc[12].y], \n         [hand.iloc[13].y, hand.iloc[14].y, hand.iloc[15].y, hand.iloc[16].y], \n         [hand.iloc[17].y, hand.iloc[18].y, hand.iloc[19].y, hand.iloc[20].y], \n         [hand.iloc[0].y, hand.iloc[5].y, hand.iloc[9].y, hand.iloc[13].y, hand.iloc[17].y, hand.iloc[0].y]] \n    return x, y\n\ndef get_pose_points(pose):\n    x = [[pose.iloc[8].x, pose.iloc[6].x, pose.iloc[5].x, pose.iloc[4].x, pose.iloc[0].x, pose.iloc[1].x, pose.iloc[2].x, pose.iloc[3].x, pose.iloc[7].x], \n         [pose.iloc[10].x, pose.iloc[9].x], \n         [pose.iloc[22].x, pose.iloc[16].x, pose.iloc[20].x, pose.iloc[18].x, pose.iloc[16].x, pose.iloc[14].x, pose.iloc[12].x, \n          pose.iloc[11].x, pose.iloc[13].x, pose.iloc[15].x, pose.iloc[17].x, pose.iloc[19].x, pose.iloc[15].x, pose.iloc[21].x], \n         [pose.iloc[12].x, pose.iloc[24].x, pose.iloc[26].x, pose.iloc[28].x, pose.iloc[30].x, pose.iloc[32].x, pose.iloc[28].x], \n         [pose.iloc[11].x, pose.iloc[23].x, pose.iloc[25].x, pose.iloc[27].x, pose.iloc[29].x, pose.iloc[31].x, pose.iloc[27].x], \n         [pose.iloc[24].x, pose.iloc[23].x]\n        ]\n\n    y = [[pose.iloc[8].y, pose.iloc[6].y, pose.iloc[5].y, pose.iloc[4].y, pose.iloc[0].y, pose.iloc[1].y, pose.iloc[2].y, pose.iloc[3].y, pose.iloc[7].y], \n         [pose.iloc[10].y, pose.iloc[9].y], \n         [pose.iloc[22].y, pose.iloc[16].y, pose.iloc[20].y, pose.iloc[18].y, pose.iloc[16].y, pose.iloc[14].y, pose.iloc[12].y, \n          pose.iloc[11].y, pose.iloc[13].y, pose.iloc[15].y, pose.iloc[17].y, pose.iloc[19].y, pose.iloc[15].y, pose.iloc[21].y], \n         [pose.iloc[12].y, pose.iloc[24].y, pose.iloc[26].y, pose.iloc[28].y, pose.iloc[30].y, pose.iloc[32].y, pose.iloc[28].y], \n         [pose.iloc[11].y, pose.iloc[23].y, pose.iloc[25].y, pose.iloc[27].y, pose.iloc[29].y, pose.iloc[31].y, pose.iloc[27].y], \n         [pose.iloc[24].y, pose.iloc[23].y]\n        ]\n    return x, y","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:02.571394Z","iopub.execute_input":"2023-04-24T20:39:02.571776Z","iopub.status.idle":"2023-04-24T20:39:02.601106Z","shell.execute_reply.started":"2023-04-24T20:39:02.571739Z","shell.execute_reply":"2023-04-24T20:39:02.599415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def animation_frame(f):\n    frame = sign[sign.frame==f]\n    left = frame[frame.type=='left_hand']\n    right = frame[frame.type=='right_hand']\n    pose = frame[frame.type=='pose']\n    face = frame[frame.type=='face'][['x', 'y']].values\n    lx, ly = get_hand_points(left)\n    rx, ry = get_hand_points(right)\n    px, py = get_pose_points(pose)\n    ax.clear()\n    ax.plot(face[:,0], face[:,1], '.')\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    for i in range(len(rx)):\n        ax.plot(rx[i], ry[i])\n    for i in range(len(px)):\n        ax.plot(px[i], py[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n        \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:02.603384Z","iopub.execute_input":"2023-04-24T20:39:02.606558Z","iopub.status.idle":"2023-04-24T20:39:05.624422Z","shell.execute_reply.started":"2023-04-24T20:39:02.606506Z","shell.execute_reply":"2023-04-24T20:39:05.622871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sign = sign[sign.type=='right_hand'].dropna()\ndef animation_frame(f):\n    frame = sign[sign.frame==f]\n    left = frame[frame.type=='right_hand']\n    lx, ly = get_hand_points(left)\n    ax.clear()\n    for i in range(len(lx)):\n        ax.plot(lx[i], ly[i])\n    plt.xlim(xmin, xmax)\n    plt.ylim(ymin, ymax)\n\n        \nprint(f\"The sign being shown here is: {train[train.path==f'{path_to_sign}'].sign.values[0]}\")\n\n## These values set the limits on the graph to stabilize the video\nxmin = sign.x.min() - 0.2\nxmax = sign.x.max() + 0.2\nymin = sign.y.min() - 0.2\nymax = sign.y.max() + 0.2\n\nfig, ax = plt.subplots()\nl, = ax.plot([], [])\nanimation = FuncAnimation(fig, func=animation_frame, frames=sign.frame.unique())\n\nHTML(animation.to_html5_video())","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:05.626783Z","iopub.execute_input":"2023-04-24T20:39:05.628017Z","iopub.status.idle":"2023-04-24T20:39:07.298917Z","shell.execute_reply.started":"2023-04-24T20:39:05.627954Z","shell.execute_reply":"2023-04-24T20:39:07.297336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport numpy as np\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:07.301318Z","iopub.execute_input":"2023-04-24T20:39:07.301726Z","iopub.status.idle":"2023-04-24T20:39:09.736312Z","shell.execute_reply.started":"2023-04-24T20:39:07.301680Z","shell.execute_reply":"2023-04-24T20:39:09.734971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport numpy as np\n\nclass FeatureGen(nn.Module):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n\n    def forward(self, x):\n        x = torch.from_numpy(x)\n        x = torch.where(torch.isnan(x), torch.zeros_like(x), x)\n        x = torch.mean(x, axis=0)\n        return x\n    \nfeature_converter = FeatureGen()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:39:09.738011Z","iopub.execute_input":"2023-04-24T20:39:09.739645Z","iopub.status.idle":"2023-04-24T20:39:09.747728Z","shell.execute_reply.started":"2023-04-24T20:39:09.739601Z","shell.execute_reply":"2023-04-24T20:39:09.746284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_length = (len(train))\ndata_lenght_experiment = int(len(train)/10)\n\nprint(\"Lenght of data for modeling :\", data_length)\nprint(f\"Percentage of total data {data_lenght_experiment/data_length*100:.1f}%\")","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:29.461739Z","iopub.execute_input":"2023-04-24T22:11:29.462375Z","iopub.status.idle":"2023-04-24T22:11:29.474162Z","shell.execute_reply.started":"2023-04-24T22:11:29.462306Z","shell.execute_reply":"2023-04-24T22:11:29.472526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    data_columns = [\"x\", \"y\", \"z\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:31.162961Z","iopub.execute_input":"2023-04-24T22:11:31.163473Z","iopub.status.idle":"2023-04-24T22:11:31.171520Z","shell.execute_reply.started":"2023-04-24T22:11:31.163428Z","shell.execute_reply":"2023-04-24T22:11:31.170313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_row(row):\n    x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row.path))\n    x = feature_converter(x)\n    return x, row.label","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:32.630033Z","iopub.execute_input":"2023-04-24T22:11:32.631158Z","iopub.status.idle":"2023-04-24T22:11:32.638423Z","shell.execute_reply.started":"2023-04-24T22:11:32.631100Z","shell.execute_reply":"2023-04-24T22:11:32.636552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\nROWS_PER_FRAME = 543\n\ndef convert_and_save_data():\n    np_features = np.zeros((data_lenght_experiment, ROWS_PER_FRAME, 3))\n    np_labels = np.zeros(data_lenght_experiment)\n\n    print(f\"Total data to processe : {data_lenght_experiment}\")\n    for index, row in tqdm(train.iterrows()):\n        if index > data_lenght_experiment - 1:\n            break\n        data = load_relevant_data_subset(f\"{dir}/{row.path}\")\n        feature, label = convert_row(row)\n        np_features[index, :, :] = feature.to(torch.double)\n        np_labels[index] = label\n\n    np.save(\"features.npy\", np_features)\n    np.save(\"labels.npy\", np_labels)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:34.058948Z","iopub.execute_input":"2023-04-24T22:11:34.059449Z","iopub.status.idle":"2023-04-24T22:11:34.070024Z","shell.execute_reply.started":"2023-04-24T22:11:34.059403Z","shell.execute_reply":"2023-04-24T22:11:34.068114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/*.npy","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:37.465312Z","iopub.execute_input":"2023-04-24T22:11:37.465817Z","iopub.status.idle":"2023-04-24T22:11:38.645819Z","shell.execute_reply.started":"2023-04-24T22:11:37.465767Z","shell.execute_reply":"2023-04-24T22:11:38.644285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:38.674876Z","iopub.execute_input":"2023-04-24T22:11:38.675456Z","iopub.status.idle":"2023-04-24T22:11:38.682172Z","shell.execute_reply.started":"2023-04-24T22:11:38.675406Z","shell.execute_reply":"2023-04-24T22:11:38.680673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    features = torch.from_numpy(np.load(\"/kaggle/working/features.npy\")).float()\n    labels = torch.from_numpy(np.load(\"/kaggle/working/labels.npy\")).long()\nexcept:\n    convert_and_save_data()\nfinally:\n    features = torch.from_numpy(np.load(\"/kaggle/working/features.npy\")).float()\n    labels = torch.from_numpy(np.load(\"/kaggle/working/labels.npy\")).long()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:11:39.357595Z","iopub.execute_input":"2023-04-24T22:11:39.358318Z","iopub.status.idle":"2023-04-24T22:15:30.781442Z","shell.execute_reply.started":"2023-04-24T22:11:39.358245Z","shell.execute_reply":"2023-04-24T22:15:30.780013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(\n    features, labels, test_size=0.2, stratify=labels, random_state=42\n)\n\nprint(\"Training Shape\", X_train.shape, y_train.shape)\nprint(\"Testing Shape\", X_val.shape, y_val.shape) ","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:18:52.889200Z","iopub.execute_input":"2023-04-24T22:18:52.890200Z","iopub.status.idle":"2023-04-24T22:18:52.952915Z","shell.execute_reply.started":"2023-04-24T22:18:52.890141Z","shell.execute_reply":"2023-04-24T22:18:52.951718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n\n    def __len__(self):\n        return len(self.y)\n\n    def __getitem__(self, i):\n        return self.X[i], self.y[i]","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:18:56.560600Z","iopub.execute_input":"2023-04-24T22:18:56.561530Z","iopub.status.idle":"2023-04-24T22:18:56.570943Z","shell.execute_reply.started":"2023-04-24T22:18:56.561481Z","shell.execute_reply":"2023-04-24T22:18:56.568813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = Dataset(X_train, y_train)\nval_dataset = Dataset(X_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:18:58.945772Z","iopub.execute_input":"2023-04-24T22:18:58.947075Z","iopub.status.idle":"2023-04-24T22:18:58.965034Z","shell.execute_reply.started":"2023-04-24T22:18:58.947023Z","shell.execute_reply":"2023-04-24T22:18:58.963502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_workers = 1\nbatch_size = 128\n\ntrain_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True,num_workers=num_workers, pin_memory=True, drop_last=True)\nval_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False,num_workers=num_workers, pin_memory=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:19:00.771853Z","iopub.execute_input":"2023-04-24T22:19:00.774576Z","iopub.status.idle":"2023-04-24T22:19:00.823367Z","shell.execute_reply.started":"2023-04-24T22:19:00.774516Z","shell.execute_reply":"2023-04-24T22:19:00.821999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.autograd import Variable ","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:19:04.391796Z","iopub.execute_input":"2023-04-24T22:19:04.392766Z","iopub.status.idle":"2023-04-24T22:19:04.399603Z","shell.execute_reply.started":"2023-04-24T22:19:04.392692Z","shell.execute_reply":"2023-04-24T22:19:04.397546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n        self.layer1 = nn.Linear(3, 128)\n        self.layer2 = nn.Linear(128, 64)\n        self.layer3 = nn.Linear(64, 32)\n        self.layer4 = nn.Linear(32, 16)\n        self.layer5 = nn.Linear(16 * 543, 250)\n        self.relu = nn.ReLU()\n        self.flatten = nn.Flatten()\n\n    def forward(self, x):\n        x = self.relu(self.layer1(x))\n        x = self.relu(self.layer2(x))\n        x = self.relu(self.layer3(x))\n        x = self.relu(self.layer4(x))\n        x = self.flatten(x)\n        x = self.layer5(x)\n        return x\n\nmodel = Model()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:19:08.149484Z","iopub.execute_input":"2023-04-24T22:19:08.149965Z","iopub.status.idle":"2023-04-24T22:19:08.192512Z","shell.execute_reply.started":"2023-04-24T22:19:08.149919Z","shell.execute_reply":"2023-04-24T22:19:08.190949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:19:09.249908Z","iopub.execute_input":"2023-04-24T22:19:09.250499Z","iopub.status.idle":"2023-04-24T22:19:09.262705Z","shell.execute_reply.started":"2023-04-24T22:19:09.250452Z","shell.execute_reply":"2023-04-24T22:19:09.261269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate = 0.0001 #0.001 lr\n\ncriterion = torch.nn.CrossEntropyLoss()    # mean-squared error for regression\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) ","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:19:30.564568Z","iopub.execute_input":"2023-04-24T22:19:30.565124Z","iopub.status.idle":"2023-04-24T22:19:30.572222Z","shell.execute_reply.started":"2023-04-24T22:19:30.565081Z","shell.execute_reply":"2023-04-24T22:19:30.571102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter\n\n\nPATH_LOG = '/kaggle/working/asl_sign'\nwriter = SummaryWriter(PATH_LOG)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:19:31.471411Z","iopub.execute_input":"2023-04-24T22:19:31.472629Z","iopub.status.idle":"2023-04-24T22:19:31.503105Z","shell.execute_reply.started":"2023-04-24T22:19:31.472578Z","shell.execute_reply":"2023-04-24T22:19:31.501348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"num_epochs = 500\nfor epoch in range(num_epochs):\n    train_loss, train_correct, train_n, val_loss, val_correct, val_n = 0,0,0,0,0,0\n    model.train()\n    \n    for ibatch, (X, y) in enumerate(train_dataloader):\n        X, y = X.to(device), y.to(device)\n        optimizer.zero_grad()\n        y_pred = model(X)\n        loss = criterion(y_pred, y)\n        \n        train_n += y.size(0)\n        train_loss += loss.item()\n        train_correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()\n\n        loss.backward()\n        optimizer.step()\n        \n    train_loss /= ibatch   \n    train_correct /= train_n\n    \n    model.eval()\n    \n    for ibatch, (X, y) in enumerate(val_dataloader):\n        X, y = X.to(device), y.to(device)\n        with torch.no_grad():\n            y_pred = model(X)\n            loss = criterion(y_pred, y)\n            val_n += y.size(0)\n            val_loss += loss.item()\n            val_correct += (y_pred.argmax(1) == y).type(torch.float).sum().item()\n\n    val_loss /= ibatch   \n    val_correct /= val_n\n    \n    # write log\n    writer.add_scalar(\n        'loss',\n        train_loss,\n        epoch + 1\n    )\n    \n    writer.add_scalar(\n        'accuracy',\n        train_correct,\n        epoch + 1\n    )\n    \n    writer.add_scalar(\n        'val_loss',\n        val_loss,\n        epoch + 1\n    )\n    writer.add_scalar(\n        'val_accuracy',\n        val_correct,\n        epoch + 1\n    )\n        \n    print('Epoch %d/%d loss:%.4f accuracy:%.4f val_loss:%.4f val_accuracy:%.4f' %(epoch + 1, num_epochs, train_loss, train_correct, val_loss, val_correct))","metadata":{"execution":{"iopub.status.busy":"2023-04-24T22:20:01.072132Z","iopub.execute_input":"2023-04-24T22:20:01.072634Z","iopub.status.idle":"2023-04-25T00:46:03.393927Z","shell.execute_reply.started":"2023-04-24T22:20:01.072586Z","shell.execute_reply":"2023-04-25T00:46:03.392047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Save","metadata":{}},{"cell_type":"code","source":"torch.save(model.state_dict(), 'model.pth')","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:03.488306Z","iopub.execute_input":"2023-04-25T00:47:03.489020Z","iopub.status.idle":"2023-04-25T00:47:03.518868Z","shell.execute_reply.started":"2023-04-25T00:47:03.488954Z","shell.execute_reply":"2023-04-25T00:47:03.517431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchinfo import summary\n\nsummary(model=model, input_size=(128, 543, 3))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:05.384587Z","iopub.execute_input":"2023-04-25T00:47:05.385070Z","iopub.status.idle":"2023-04-25T00:47:05.505116Z","shell.execute_reply.started":"2023-04-25T00:47:05.385023Z","shell.execute_reply":"2023-04-25T00:47:05.504051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/asl_sign","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:10.958905Z","iopub.execute_input":"2023-04-25T00:47:10.959406Z","iopub.status.idle":"2023-04-25T00:47:12.092122Z","shell.execute_reply.started":"2023-04-25T00:47:10.959357Z","shell.execute_reply":"2023-04-25T00:47:12.090423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize result training","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport struct\nimport glob\nfrom tensorflow.python.summary.summary_iterator import summary_iterator","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:15.073466Z","iopub.execute_input":"2023-04-25T00:47:15.074540Z","iopub.status.idle":"2023-04-25T00:47:15.081989Z","shell.execute_reply.started":"2023-04-25T00:47:15.074469Z","shell.execute_reply":"2023-04-25T00:47:15.080833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_event = glob.glob(f'{PATH_LOG}/*')[0]\n\ntrain_loss, train_correct, train_n, val_loss, val_correct, val_n = [],[],[],[],[],[]\nsteps=[]\nfor e in summary_iterator(log_event):\n    for v in e.summary.value:\n        if v.tag == 'accuracy':       \n            train_correct.append(v.simple_value)\n        if v.tag == 'val_accuracy':       \n            val_correct.append(v.simple_value)\n        if v.tag == 'loss':\n            train_loss.append(v.simple_value)\n        if v.tag == 'val_loss':\n            val_loss.append(v.simple_value)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:16.777726Z","iopub.execute_input":"2023-04-25T00:47:16.778254Z","iopub.status.idle":"2023-04-25T00:47:17.182491Z","shell.execute_reply.started":"2023-04-25T00:47:16.778191Z","shell.execute_reply":"2023-04-25T00:47:17.181423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"] = [7.50, 3.50]\nplt.rcParams[\"figure.autolayout\"] = True\n\nax1 = plt.subplot()\nl1, = ax1.plot(val_correct, color='red')\nax2 = ax1.twinx()\nl2, = ax2.plot(train_correct, color='blue')\n\nplt.legend([l1, l2], [\"Accuracy\", \"Val Accuracy\"])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:22.606578Z","iopub.execute_input":"2023-04-25T00:47:22.607087Z","iopub.status.idle":"2023-04-25T00:47:23.010340Z","shell.execute_reply.started":"2023-04-25T00:47:22.607043Z","shell.execute_reply":"2023-04-25T00:47:23.008776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"] = [7.50, 3.50]\nplt.rcParams[\"figure.autolayout\"] = True\n\nax1 = plt.subplot()\nl1, = ax1.plot(val_loss, color='red')\nax2 = ax1.twinx()\nl2, = ax2.plot(train_loss, color='blue')\n\nplt.legend([l1, l2], [\"Val Loss\", \"Train Loss\"])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:40.310428Z","iopub.execute_input":"2023-04-25T00:47:40.310873Z","iopub.status.idle":"2023-04-25T00:47:40.640674Z","shell.execute_reply.started":"2023-04-25T00:47:40.310834Z","shell.execute_reply":"2023-04-25T00:47:40.639348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Export Model TfLite","metadata":{}},{"cell_type":"markdown","source":"## Inference Model","metadata":{"execution":{"iopub.status.busy":"2023-04-23T23:16:53.317214Z","iopub.execute_input":"2023-04-23T23:16:53.317719Z","iopub.status.idle":"2023-04-23T23:16:53.326277Z","shell.execute_reply.started":"2023-04-23T23:16:53.317679Z","shell.execute_reply":"2023-04-23T23:16:53.324353Z"}}},{"cell_type":"code","source":"class Model_infe(Model):\n    \n    def __init__(self):\n        super().__init__()\n        self.softmax = nn.Softmax()\n        \n    def forward(self, x):\n        x = torch.where(torch.isnan(x), torch.tensor(0.0, dtype=torch.float32).to(device), x)\n        x = torch.mean(x, dim=0, keepdim=False)\n        x = self.relu(self.layer1(x))\n\n        x = self.relu(self.layer2(x))\n        x = self.relu(self.layer3(x))\n        x = self.relu(self.layer4(x))\n        x = self.flatten(x)\n        x = self.layer5(x)\n        return self.softmax(x)\n    \nmodel_infe = Model_infe()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:46.757169Z","iopub.execute_input":"2023-04-25T00:47:46.757920Z","iopub.status.idle":"2023-04-25T00:47:46.794858Z","shell.execute_reply.started":"2023-04-25T00:47:46.757852Z","shell.execute_reply":"2023-04-25T00:47:46.793404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_infe.load_state_dict(torch.load('/kaggle/working/model.pth'), strict=False)\nmodel_infe = model_infe.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:49.052050Z","iopub.execute_input":"2023-04-25T00:47:49.052864Z","iopub.status.idle":"2023-04-25T00:47:49.076718Z","shell.execute_reply.started":"2023-04-25T00:47:49.052813Z","shell.execute_reply":"2023-04-25T00:47:49.073770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EXPORT ONNX","metadata":{}},{"cell_type":"code","source":"input_size = (1, 543, 3)\nbatch_size = 1\nsaved_onnx = 'model.onnx'\n\ndummy_input = torch.rand((batch_size, *input_size)).to(device)\n\nmodel_infe.eval()\npreds = model_infe(dummy_input)\npreds","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:51.918740Z","iopub.execute_input":"2023-04-25T00:47:51.919180Z","iopub.status.idle":"2023-04-25T00:47:51.973790Z","shell.execute_reply.started":"2023-04-25T00:47:51.919138Z","shell.execute_reply":"2023-04-25T00:47:51.972461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.onnx.export(\n    model_infe,\n    dummy_input, \n    saved_onnx,\n    verbose=False,\n    input_names=['inputs'],\n    output_names=['outputs'],\n#     export_params=True,\n    opset_version=11\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:55.610892Z","iopub.execute_input":"2023-04-25T00:47:55.611391Z","iopub.status.idle":"2023-04-25T00:47:55.800275Z","shell.execute_reply.started":"2023-04-25T00:47:55.611343Z","shell.execute_reply":"2023-04-25T00:47:55.798384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# verify onnx\nimport onnx\n\n# Load the ONNX model\nmodel_onnx = onnx.load(saved_onnx)\n\n# Check that the model is well formed\nonnx.checker.check_model(model_onnx)\n\n# Print a human readable representation of the graph\nprint(onnx.helper.printable_graph(model_onnx.graph))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:47:58.012042Z","iopub.execute_input":"2023-04-25T00:47:58.012559Z","iopub.status.idle":"2023-04-25T00:47:58.169640Z","shell.execute_reply.started":"2023-04-25T00:47:58.012515Z","shell.execute_reply":"2023-04-25T00:47:58.167653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Export TfLite","metadata":{}},{"cell_type":"code","source":"# Install library\n!pip install onnx-tf","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:02.416185Z","iopub.execute_input":"2023-04-25T00:48:02.416668Z","iopub.status.idle":"2023-04-25T00:48:16.892490Z","shell.execute_reply.started":"2023-04-25T00:48:02.416627Z","shell.execute_reply":"2023-04-25T00:48:16.891144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from onnx_tf.backend import prepare\n\ntf_rep = prepare(model_onnx)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:16.894524Z","iopub.execute_input":"2023-04-25T00:48:16.894913Z","iopub.status.idle":"2023-04-25T00:48:20.171525Z","shell.execute_reply.started":"2023-04-25T00:48:16.894872Z","shell.execute_reply":"2023-04-25T00:48:20.170292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### onnx to Tensorflow FrozenGraph(.pb)\nNow that a tf_rep variable has been created, the converted model can be exported to a .pb file and stored within this notebook.","metadata":{}},{"cell_type":"code","source":"pb_path = \"model.pb\"\ntf_rep.export_graph(pb_path)\n\nassert os.path.exists(pb_path)\nprint(\".pb model converted successfully.\")","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:20.175376Z","iopub.execute_input":"2023-04-25T00:48:20.175909Z","iopub.status.idle":"2023-04-25T00:48:24.290980Z","shell.execute_reply.started":"2023-04-25T00:48:20.175862Z","shell.execute_reply":"2023-04-25T00:48:24.289477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_nodes = tf_rep.inputs\noutput_nodes = tf_rep.outputs\nprint(\"The names of the input nodes are: {}\".format(input_nodes))\nprint(\"The names of the output nodes are: {}\".format(output_nodes))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:26.589007Z","iopub.execute_input":"2023-04-25T00:48:26.589818Z","iopub.status.idle":"2023-04-25T00:48:26.597904Z","shell.execute_reply.started":"2023-04-25T00:48:26.589770Z","shell.execute_reply":"2023-04-25T00:48:26.596345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"converter = tf.lite.TFLiteConverter.from_saved_model(pb_path)\ntflite_rep = converter.convert()\n\ntflite_model_path = 'model.tflite'\nwith open(tflite_model_path, 'wb') as f:\n    f.write(tflite_rep)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:29.373720Z","iopub.execute_input":"2023-04-25T00:48:29.374198Z","iopub.status.idle":"2023-04-25T00:48:30.186518Z","shell.execute_reply.started":"2023-04-25T00:48:29.374152Z","shell.execute_reply":"2023-04-25T00:48:30.185149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction","metadata":{}},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:33.725632Z","iopub.execute_input":"2023-04-25T00:48:33.726111Z","iopub.status.idle":"2023-04-25T00:48:35.342551Z","shell.execute_reply.started":"2023-04-25T00:48:33.726070Z","shell.execute_reply":"2023-04-25T00:48:35.340917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install library\n!pip3 install tflite_runtime","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:37.532068Z","iopub.execute_input":"2023-04-25T00:48:37.533289Z","iopub.status.idle":"2023-04-25T00:48:49.857574Z","shell.execute_reply.started":"2023-04-25T00:48:37.533225Z","shell.execute_reply":"2023-04-25T00:48:49.855930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tflite_runtime.interpreter as tflite","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:49.860777Z","iopub.execute_input":"2023-04-25T00:48:49.861408Z","iopub.status.idle":"2023-04-25T00:48:49.884012Z","shell.execute_reply.started":"2023-04-25T00:48:49.861339Z","shell.execute_reply":"2023-04-25T00:48:49.882732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"interpreter = tflite.Interpreter(tflite_model_path)\nfound_signatures = list(interpreter.get_signature_list().keys())\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\nlist_label = train['sign'].unique()\n\nfor i in range(100):\n    frames = load_relevant_data_subset(f'{dir}/{train.iloc[i].path}')\n    output = prediction_fn(inputs=frames)\n    sign = np.argmax(output[\"outputs\"])\n\n    print(f\"Predicted label: {p2s_map[sign]}, Actual Label: {train.iloc[i].sign}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-25T00:48:52.566231Z","iopub.execute_input":"2023-04-25T00:48:52.567282Z","iopub.status.idle":"2023-04-25T00:48:54.274908Z","shell.execute_reply.started":"2023-04-25T00:48:52.567205Z","shell.execute_reply":"2023-04-25T00:48:54.273871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}