{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install mediapipe","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:58:50.923230Z","iopub.execute_input":"2023-04-16T14:58:50.923654Z","iopub.status.idle":"2023-04-16T14:59:06.252632Z","shell.execute_reply.started":"2023-04-16T14:58:50.923614Z","shell.execute_reply":"2023-04-16T14:59:06.251106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as anim\nfrom matplotlib import rc\nrc('animation', html='jshtml')\nimport mediapipe as mp\nmp_holistic = mp.solutions.holistic","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-16T14:59:06.254934Z","iopub.execute_input":"2023-04-16T14:59:06.255456Z","iopub.status.idle":"2023-04-16T14:59:06.557336Z","shell.execute_reply.started":"2023-04-16T14:59:06.255414Z","shell.execute_reply":"2023-04-16T14:59:06.556006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = Path('/kaggle/input/')\nASL_DIR = INPUT_DIR / 'asl-signs'","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.559096Z","iopub.execute_input":"2023-04-16T14:59:06.559595Z","iopub.status.idle":"2023-04-16T14:59:06.567069Z","shell.execute_reply.started":"2023-04-16T14:59:06.559546Z","shell.execute_reply":"2023-04-16T14:59:06.565856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(ASL_DIR / 'train.csv')\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.570064Z","iopub.execute_input":"2023-04-16T14:59:06.570437Z","iopub.status.idle":"2023-04-16T14:59:06.804567Z","shell.execute_reply.started":"2023-04-16T14:59:06.570399Z","shell.execute_reply":"2023-04-16T14:59:06.803323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sign.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.805938Z","iopub.execute_input":"2023-04-16T14:59:06.806287Z","iopub.status.idle":"2023-04-16T14:59:06.827865Z","shell.execute_reply.started":"2023-04-16T14:59:06.806247Z","shell.execute_reply":"2023-04-16T14:59:06.826241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.participant_id.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.829491Z","iopub.execute_input":"2023-04-16T14:59:06.830344Z","iopub.status.idle":"2023-04-16T14:59:06.841747Z","shell.execute_reply.started":"2023-04-16T14:59:06.830305Z","shell.execute_reply":"2023-04-16T14:59:06.840722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.participant_id.value_counts().size","metadata":{"execution":{"iopub.status.busy":"2023-04-16T15:00:32.570475Z","iopub.execute_input":"2023-04-16T15:00:32.570871Z","iopub.status.idle":"2023-04-16T15:00:32.581339Z","shell.execute_reply.started":"2023-04-16T15:00:32.570835Z","shell.execute_reply":"2023-04-16T15:00:32.579818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.query('sign==\"look\"')","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.843026Z","iopub.execute_input":"2023-04-16T14:59:06.843362Z","iopub.status.idle":"2023-04-16T14:59:06.870444Z","shell.execute_reply.started":"2023-04-16T14:59:06.843331Z","shell.execute_reply":"2023-04-16T14:59:06.869137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.872192Z","iopub.execute_input":"2023-04-16T14:59:06.872539Z","iopub.status.idle":"2023-04-16T14:59:06.879185Z","shell.execute_reply.started":"2023-04-16T14:59:06.872506Z","shell.execute_reply":"2023-04-16T14:59:06.877922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"entry = train_df.iloc[567]","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.880758Z","iopub.execute_input":"2023-04-16T14:59:06.881380Z","iopub.status.idle":"2023-04-16T14:59:06.890732Z","shell.execute_reply.started":"2023-04-16T14:59:06.881342Z","shell.execute_reply":"2023-04-16T14:59:06.889486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = load_relevant_data_subset(ASL_DIR / entry.path)\nframes.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:06.894201Z","iopub.execute_input":"2023-04-16T14:59:06.894619Z","iopub.status.idle":"2023-04-16T14:59:07.028344Z","shell.execute_reply.started":"2023-04-16T14:59:06.894572Z","shell.execute_reply":"2023-04-16T14:59:07.026584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sequence = pd.read_parquet(ASL_DIR / entry.path)\nsequence","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:07.030068Z","iopub.execute_input":"2023-04-16T14:59:07.030874Z","iopub.status.idle":"2023-04-16T14:59:07.072126Z","shell.execute_reply.started":"2023-04-16T14:59:07.030823Z","shell.execute_reply":"2023-04-16T14:59:07.071178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def landmark_order(sequence):\n    n_frames = int(len(sequence) / ROWS_PER_FRAME)\n    data = sequence.values.reshape(n_frames, ROWS_PER_FRAME, len(sequence.columns))\n    return np.unique(data[:,:,2:4].astype(str), axis=0)\n\nreference_order = landmark_order(sequence)\n\ndef aggregate_info(path):\n    sequence = pd.read_parquet(ASL_DIR / path)\n    start_frame = sequence.frame.min()\n    end_frame = sequence.frame.max()\n    n_frames = end_frame - start_frame + 1\n    order_consistent = (landmark_order(sequence) == reference_order).all()\n    agg = pd.concat({\n        'any': sequence.groupby(['frame','type'])[['x','y','z']].agg(lambda c: c.isnull().any()).any(axis=1),\n        'all': sequence.groupby(['frame','type'])[['x','y','z']].agg(lambda c: c.isnull().all()).all(axis=1),\n    }, axis=1).reset_index().groupby('type').sum()\n    del sequence\n    agg = pd.melt(agg.reset_index(), id_vars=['type'], value_vars=['any','all'])\n    agg.index = agg.type + '_' + agg.variable\n    info = agg['value'].copy().sort_index()\n    del agg\n    info.name = 'info'\n    info['start_frame'] = start_frame\n    info['end_frame'] = end_frame\n    info['n_frames'] = n_frames\n    info['order_consistent'] = order_consistent\n    return info","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:43.239100Z","iopub.execute_input":"2023-04-16T14:59:43.239574Z","iopub.status.idle":"2023-04-16T14:59:43.263542Z","shell.execute_reply.started":"2023-04-16T14:59:43.239533Z","shell.execute_reply":"2023-04-16T14:59:43.262538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg = train_df.path.apply(aggregate_info)\nagg.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-04-16T14:59:49.770687Z","iopub.execute_input":"2023-04-16T14:59:49.771871Z","iopub.status.idle":"2023-04-16T15:00:28.300610Z","shell.execute_reply.started":"2023-04-16T14:59:49.771818Z","shell.execute_reply":"2023-04-16T15:00:28.298715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg.to_parquet('agg.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-03-11T17:59:08.613166Z","iopub.execute_input":"2023-03-11T17:59:08.614089Z","iopub.status.idle":"2023-03-11T17:59:08.653348Z","shell.execute_reply.started":"2023-03-11T17:59:08.614035Z","shell.execute_reply":"2023-03-11T17:59:08.651838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df = pd.read_parquet('agg.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-03-11T17:59:44.307095Z","iopub.execute_input":"2023-03-11T17:59:44.308392Z","iopub.status.idle":"2023-03-11T17:59:44.349877Z","shell.execute_reply.started":"2023-03-11T17:59:44.308326Z","shell.execute_reply":"2023-03-11T17:59:44.348657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T17:59:57.328870Z","iopub.execute_input":"2023-03-11T17:59:57.330276Z","iopub.status.idle":"2023-03-11T17:59:57.463563Z","shell.execute_reply.started":"2023-03-11T17:59:57.330222Z","shell.execute_reply":"2023-03-11T17:59:57.462173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_null = agg[['face_all','left_hand_all','right_hand_all','pose_all']]\nall_null.columns = [['face','left_hand','right_hand','pose']]\nany_null = agg[['face_any','left_hand_any','right_hand_any','pose_any']]\nany_null.columns = [['face','left_hand','right_hand','pose']]\nlen(agg) - (all_null == any_null).count()","metadata":{"execution":{"iopub.status.busy":"2023-03-11T12:18:42.961553Z","iopub.execute_input":"2023-03-11T12:18:42.962082Z","iopub.status.idle":"2023-03-11T12:18:42.980479Z","shell.execute_reply.started":"2023-03-11T12:18:42.962039Z","shell.execute_reply":"2023-03-11T12:18:42.978848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot one frame of landmarks\ndef plot_frame(axis, sequence, i, face=False):\n    axis.clear()\n    axis.set_xlim([sequence.x.min(), sequence.x.max()])\n    axis.set_ylim([sequence.y.min(), sequence.y.max()])\n    axis.set_aspect('equal', adjustable='box')\n    axis.invert_yaxis()\n    frame = sequence.query('frame == @i')\n    frame = frame.set_index('landmark_index')\n    plot_landmarks(axis, frame, \"pose\", mp_holistic.POSE_CONNECTIONS)\n    plot_landmarks(axis, frame, \"left_hand\", mp_holistic.HAND_CONNECTIONS)\n    plot_landmarks(axis, frame, \"right_hand\", mp_holistic.HAND_CONNECTIONS)\n    if face:\n        plot_landmarks(axis, frame, \"face\", mp_holistic.FACEMESH_TESSELATION)\n\n# Plot landmarks of one type (face, pose, left_hand, right_hand) in a frame\ndef plot_landmarks(axis, frame, kind, connections):\n    landmarks = frame.query('type==@kind')\n    indices = np.array(list(connections))\n    left = landmarks.loc[indices[:,0],['x','y']].reset_index(drop=True).rename(columns={'x':'x0','y':'y0'})\n    right = landmarks.loc[indices[:,1],['x','y']].reset_index(drop=True).rename(columns={'x':'x1','y':'y1'})\n    pairs = pd.concat([left, right], axis=1)\n    axis.plot(pairs[['x0','x1']].T, pairs[['y0','y1']].T, color='gray')","metadata":{"execution":{"iopub.status.busy":"2023-03-11T12:14:41.211306Z","iopub.execute_input":"2023-03-11T12:14:41.212336Z","iopub.status.idle":"2023-03-11T12:14:41.227648Z","shell.execute_reply.started":"2023-03-11T12:14:41.212255Z","shell.execute_reply":"2023-03-11T12:14:41.226194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"frames = sequence.frame.unique()\nfig = plt.figure()\naxis = plt.axes()\naxis.set_aspect('equal', adjustable='box')\naxis.invert_yaxis()\nplt.close()\n\nprint(entry.sign)\n\nanimation = anim.FuncAnimation(fig, lambda i: plot_frame(axis,sequence,i,False), frames=frames, interval=100)\nanimation","metadata":{"execution":{"iopub.status.busy":"2023-03-11T10:57:39.487300Z","iopub.execute_input":"2023-03-11T10:57:39.487877Z","iopub.status.idle":"2023-03-11T10:57:42.407754Z","shell.execute_reply.started":"2023-03-11T10:57:39.487819Z","shell.execute_reply":"2023-03-11T10:57:42.406639Z"},"trusted":true},"execution_count":null,"outputs":[]}]}