{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Isolated Sign Language Recognition\nThis notebook was from the start of our project, just exploring the data and getting a feel for what's going on, no actual models here","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-03-04T15:43:30.341797Z","iopub.execute_input":"2023-03-04T15:43:30.342391Z","iopub.status.idle":"2023-03-04T15:43:30.349186Z","shell.execute_reply.started":"2023-03-04T15:43:30.342336Z","shell.execute_reply":"2023-03-04T15:43:30.347675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = \"../input/asl-signs\"\ntrain = pd.read_csv(f'{BASE_DIR}/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-04T15:43:32.892288Z","iopub.execute_input":"2023-03-04T15:43:32.893548Z","iopub.status.idle":"2023-03-04T15:43:33.037011Z","shell.execute_reply.started":"2023-03-04T15:43:32.893464Z","shell.execute_reply":"2023-03-04T15:43:33.035911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis\ntrain.csv:\n- 94,477 observations (or videos where each represents one sign)\n- variables: \n    - path: path to the landmark file\n    - participant_id: unique identifier to data contributor\n    - sequence_id: unique identifier to the landmark sequence\n    - sign: the label for the landmark sequence, the word being signed in the video","metadata":{}},{"cell_type":"code","source":"train.info()\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T16:01:58.656341Z","iopub.execute_input":"2023-03-04T16:01:58.656723Z","iopub.status.idle":"2023-03-04T16:01:58.691808Z","shell.execute_reply.started":"2023-03-04T16:01:58.656689Z","shell.execute_reply":"2023-03-04T16:01:58.690573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What signs are we looking at? \n\n- 250 total signs where each has 299-415 observations\n- focuses on the first 250 words that an infant would learn in any language, so most of the words are simple and relevant to children (like \"listen\", \"mouse\", or \"bird\")","metadata":{}},{"cell_type":"code","source":"train.sign.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T17:27:08.336876Z","iopub.execute_input":"2023-03-04T17:27:08.337280Z","iopub.status.idle":"2023-03-04T17:27:08.353355Z","shell.execute_reply.started":"2023-03-04T17:27:08.337245Z","shell.execute_reply":"2023-03-04T17:27:08.352154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sign.value_counts().head(25).sort_values(ascending = True).plot(\n    kind = \"barh\", title = \"Top 25 Signs in train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-03-04T16:20:52.688767Z","iopub.execute_input":"2023-03-04T16:20:52.689213Z","iopub.status.idle":"2023-03-04T16:20:53.049751Z","shell.execute_reply.started":"2023-03-04T16:20:52.689177Z","shell.execute_reply":"2023-03-04T16:20:53.048541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Landmark files\nEach observation in train.csv has its own landmark file\nlandmark file: \n- variables: \n    - frame: the frame number in the raw video\n    - row_id: unique identifier for the row (frame-type-id)\n    - type: type of landmark, ['face', 'pose', 'left_hand', 'right_hand']\n    - landmark_index: identifies the spot we are looking at on the type (ie for hand, 0 = wrist, 20 = pinky tip) \n    - x, y, z: normalized spatial coordinates of the landmark, \"The MediaPipe model is not fully trained to predict depth so you may wish to ignore the z values.\" ","metadata":{}},{"cell_type":"markdown","source":"## Example landmark file\nThis landmark file has 6 frames. This person was only signing with their right hand because all values for left_hand are NaN, but their hand must not appear until the last frame (40) because all other values are NaN","metadata":{}},{"cell_type":"code","source":"filename = train.query('sign == \"listen\"')[\"path\"].values[0]\nlandmark = pd.read_parquet(f'{BASE_DIR}/{filename}')\nlandmark[(landmark.frame == 40) & (landmark.type == \"right_hand\")].head()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T17:29:55.851712Z","iopub.execute_input":"2023-03-04T17:29:55.852105Z","iopub.status.idle":"2023-03-04T17:29:55.893760Z","shell.execute_reply.started":"2023-03-04T17:29:55.852070Z","shell.execute_reply":"2023-03-04T17:29:55.892305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# left hand has no observations, right hand has 21\nlandmark.dropna(subset = ['x','y','z']).type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-03-04T17:52:00.815743Z","iopub.execute_input":"2023-03-04T17:52:00.816155Z","iopub.status.idle":"2023-03-04T17:52:00.828898Z","shell.execute_reply.started":"2023-03-04T17:52:00.816119Z","shell.execute_reply":"2023-03-04T17:52:00.827721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To Do: 3D plot ","metadata":{}},{"cell_type":"markdown","source":"# Evaluation\nThe evaluation metric for this contest is simple classification accuracy.\n\nEach video is loaded with the following function: ","metadata":{}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tflite_runtime.interpreter as tflite\n# interpreter = tflite.Interpreter(model_path)\n\n# found_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\n# prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n# output = prediction_fn(inputs=frames)\n# sign = np.argmax(output[\"outputs\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}