{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Competition Link: https://www.kaggle.com/competitions/asl-signs\n### CNN-LSTM EDA Link:https://www.kaggle.com/code/geyiming/cnn-lstm-eda","metadata":{}},{"cell_type":"markdown","source":"## Install","metadata":{}},{"cell_type":"code","source":"# Install\n!pip install -q itables 2> /dev/null\n!pip install -q flatbuffers 2> /dev/null\n!pip install -q mediapipe 2> /dev/null","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:06.018605Z","iopub.execute_input":"2023-04-12T11:57:06.019219Z","iopub.status.idle":"2023-04-12T11:57:45.781507Z","shell.execute_reply.started":"2023-04-12T11:57:06.019129Z","shell.execute_reply":"2023-04-12T11:57:45.779487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"# Imports\n\n# activate interactive mode of pd.dataframe\nimport pandas as pd\nfrom itables import init_notebook_mode\ninit_notebook_mode(all_interactive=True, connected=True)\n\nimport os\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport itertools\n\nimport tensorflow as tf\n\n#pytorch model\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\n\nimport seaborn as sns\nimport mediapipe as mp\nimport matplotlib.pyplot as plt\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n\nfrom matplotlib import animation\nfrom pathlib import Path\nimport IPython\nfrom IPython import display\nfrom IPython.core.display import display, HTML, Javascript\nfrom IPython.display import Markdown as md\n\nimport mediapipe as mp\nfrom mediapipe.framework.formats import landmark_pb2\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:45.784476Z","iopub.execute_input":"2023-04-12T11:57:45.784955Z","iopub.status.idle":"2023-04-12T11:57:45.798417Z","shell.execute_reply.started":"2023-04-12T11:57:45.784902Z","shell.execute_reply":"2023-04-12T11:57:45.796792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"# Config\nclass Config:\n    INPUT_ROOT = Path('/kaggle/input/asl-signs/')\n    OUTPUT_ROOT = Path('kaggle/working')\n    INDEX_MAP_FILE = INPUT_ROOT / 'sign_to_prediction_index_map.json'\n    TRAN_FILE = INPUT_ROOT / 'train.csv'\n    INDEX = 'sequence_id'\n    ROW_ID = 'row_id'\n\ndef read_index_map(file_path=Config.INDEX_MAP_FILE):\n    \"\"\"Reads the sign to predict as json file.\"\"\"\n    with open(file_path, \"r\") as f:\n        result = json.load(f)\n    return result \n\nLANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nlabel_map = read_index_map()\n\ntrain_df = pd.read_csv(\"/kaggle/input/gislr-extended-train-dataframe/extended_train.csv\")\ntrain_df['label'] = train_df['sign'].map(label_map)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:45.800220Z","iopub.execute_input":"2023-04-12T11:57:45.800613Z","iopub.status.idle":"2023-04-12T11:57:46.721216Z","shell.execute_reply.started":"2023-04-12T11:57:45.800574Z","shell.execute_reply":"2023-04-12T11:57:46.719737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helpers","metadata":{}},{"cell_type":"code","source":"# Helpers\n\nROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\n# https://www.kaggle.com/code/ted0071/gislr-visualization\ndef read_index_map(file_path=Config.INDEX_MAP_FILE):\n    \"\"\"Reads the sign to predict as json file.\"\"\"\n    with open(file_path, \"r\") as f:\n        result = json.load(f)\n    return result    \n\ndef read_train(file_path=Config.TRAN_FILE):\n    \"\"\"Reads the train csv as pandas data frame.\"\"\"\n    train_df = pd.read_csv(file_path).set_index(Config.INDEX)\n    train_df['label'] = train_df['sign'].map(read_index_map())\n    return train_df\n\ndef read_landmark_data_by_path(file_path, input_root=Config.INPUT_ROOT):\n    \"\"\"Reads landmak data by the given file path.\"\"\"\n    data = pd.read_parquet(input_root / file_path)\n    return data.set_index(Config.ROW_ID)\n\ndef read_landmark_data_by_id(sequence_id, train_data):\n    \"\"\"Reads the landmark data by the given sequence id.\"\"\"\n    file_path = train_data.loc[sequence_id]['path']\n    return read_landmark_data_by_path(file_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:46.724735Z","iopub.execute_input":"2023-04-12T11:57:46.725183Z","iopub.status.idle":"2023-04-12T11:57:46.739280Z","shell.execute_reply.started":"2023-04-12T11:57:46.725137Z","shell.execute_reply":"2023-04-12T11:57:46.737119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper Functions\n# [C1] adjusted from Roland Abel: https://www.kaggle.com/code/ted0071/gislr-visualization\ntrain_data = read_train()\n\nmp_drawing = mp.solutions.drawing_utils\nmp_hands = mp.solutions.hands\nmp_face_mesh = mp.solutions.face_mesh\nmp_pose = mp.solutions.pose\n\n# contour connections\nCONTOURS = list(itertools.chain(*mp_face_mesh.FACEMESH_CONTOURS))\n\ndef create_blank_image(height, width):\n    return np.zeros((height, width, 3), np.uint8)\n\ndef get_landmarks_dataframe(data, frame_id, landmark_type):\n    \"\"\"Get a dataframe with the landmarks for the specified frame and landmark type.\"\"\"\n    df = data.groupby(['frame', 'type']).get_group((frame_id, landmark_type)).copy()\n    if landmark_type == 'face':\n        df.loc[~df['landmark_index'].isin(CONTOURS),'x'] = float('NaN') #-1*df[~df['landmark_index'].isin(CONTOURS)]['x'].values\n    return df\n\ndef get_landmark_list(df):\n    \"\"\"Get a list of normalized landmarks from the specified dataframe.\"\"\"\n    landmarks = [landmark_pb2.NormalizedLandmark(x=lm.x, y=lm.y, z=lm.z) for idx, lm in df.iterrows()]\n    landmark_list = landmark_pb2.NormalizedLandmarkList(landmark = landmarks)\n    return landmark_list\n\ndef draw_landmarks(image, landmark_list, connection_type, landmark_color, connection_color, thickness, circle_radius):\n    \"\"\"Draw landmarks and connections on the specified image.\"\"\"\n    mp_drawing.draw_landmarks(\n        image=image,\n        landmark_list=landmark_list, \n        connections=connection_type,\n        landmark_drawing_spec=mp_drawing.DrawingSpec(\n            color=landmark_color, \n            thickness=thickness, \n            circle_radius=circle_radius),\n        connection_drawing_spec=mp_drawing.DrawingSpec(\n            color=connection_color, \n            thickness=thickness, \n            circle_radius=circle_radius))\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:46.741759Z","iopub.execute_input":"2023-04-12T11:57:46.742490Z","iopub.status.idle":"2023-04-12T11:57:47.007056Z","shell.execute_reply.started":"2023-04-12T11:57:46.742441Z","shell.execute_reply":"2023-04-12T11:57:47.005473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean representation for frame idx map inspired by Darien Schettler [C3]\nIDX_MAP = {\"contours\"       : list(set(CONTOURS)),\n           \"left_hand\"      : np.arange(468, 489).tolist(),\n           \"upper_body\"     : np.arange(489, 511).tolist(),\n           \"right_hand\"     : np.arange(522, 543).tolist()}\n\nFIXED_FRAMES = 37 # based on the above observations","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:47.008930Z","iopub.execute_input":"2023-04-12T11:57:47.009376Z","iopub.status.idle":"2023-04-12T11:57:47.017694Z","shell.execute_reply.started":"2023-04-12T11:57:47.009334Z","shell.execute_reply":"2023-04-12T11:57:47.015912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Processing","metadata":{}},{"cell_type":"code","source":"class FeaturePreprocess(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n    def forward(self, x_in):\n        n_frames = x_in.shape[0]\n\n        # Normalization to a common mean by Heng CK [C4]\n        x_in = x_in - x_in[~torch.isnan(x_in)].mean(0,keepdim=True) \n        x_in = x_in / x_in[~torch.isnan(x_in)].std(0, keepdim=True)\n\n        # Landmarks reduction\n        contours = x_in[:, IDX_MAP['contours']]\n        lhand    = x_in[:, IDX_MAP['left_hand']]\n        pose     = x_in[:, IDX_MAP['upper_body']]\n        rhand    = x_in[:, IDX_MAP['right_hand']]\n       \n        x_in = torch.cat([contours,\n                          lhand,\n                          pose,\n                          rhand], 1) # (n_frames, 192, 3)\n        \n         # Replace nan with 0 before Interpolation\n        x_in[torch.isnan(x_in)] = 0\n        \n        # Frames interpolation inspired by Robert Hatch [C2]\n        # If n_frames < k, use linear interpolation,\n        # else, use nearest neighbor interpolation\n        x_in = x_in.permute(2,1,0) #(3, 192, n_frames)\n        if n_frames < FIXED_FRAMES:\n            x_in = F.interpolate(x_in, size=(FIXED_FRAMES), mode= 'linear')\n        else:\n            x_in = F.interpolate(x_in, size=(FIXED_FRAMES), mode= 'nearest-exact')\n        \n        return x_in.permute(2,1,0)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:47.019861Z","iopub.execute_input":"2023-04-12T11:57:47.020468Z","iopub.status.idle":"2023-04-12T11:57:47.039657Z","shell.execute_reply.started":"2023-04-12T11:57:47.020403Z","shell.execute_reply":"2023-04-12T11:57:47.037870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_in = torch.tensor(load_relevant_data_subset(train_df.path[0]))\nfeature_preprocess = FeaturePreprocess()\nfeature_preprocess(x_in).shape, x_in[0]\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:47.041487Z","iopub.execute_input":"2023-04-12T11:57:47.041895Z","iopub.status.idle":"2023-04-12T11:57:47.084891Z","shell.execute_reply.started":"2023-04-12T11:57:47.041860Z","shell.execute_reply":"2023-04-12T11:57:47.083318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save processing features","metadata":{}},{"cell_type":"code","source":"# adapted and adjusted from Robert Hatch [C2]\nright_handed_signer = [26734, 28656, 25571, 62590, 29302, \n                       49445, 53618, 18796,  4718,  2044, \n                       37779, 30680]\nleft_handed_signer  = [16069, 32319, 36257, 22343, 27610, \n                       61333, 34503, 55372, ]\nboth_hands_signer   = [37055 ]\nmessy = [29302 ]\n\ndef convert_row(row, right_handed=True):\n    x = torch.tensor(load_relevant_data_subset(row[1].path))\n    x = feature_preprocess(x).cpu().numpy()\n    return x, row[1].label\n\ndef convert_and_save_data(df):\n    total = df.shape[0]\n    npdata = np.zeros((total, 37, 192 ,3))\n    nplabels = np.zeros(total)\n    for i, row in tqdm(enumerate(df.iterrows()), total=total):\n        (x,y) = convert_row(row)\n        npdata[i,:,:,:] = x\n        nplabels[i] = y\n    \n    np.save(\"feature_data.npy\", npdata)\n    np.save(\"feature_labels.npy\", nplabels)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:47.086820Z","iopub.execute_input":"2023-04-12T11:57:47.087263Z","iopub.status.idle":"2023-04-12T11:57:47.098728Z","shell.execute_reply.started":"2023-04-12T11:57:47.087221Z","shell.execute_reply":"2023-04-12T11:57:47.097548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_and_save_data(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T11:57:47.102513Z","iopub.execute_input":"2023-04-12T11:57:47.102937Z","iopub.status.idle":"2023-04-12T12:36:47.973275Z","shell.execute_reply.started":"2023-04-12T11:57:47.102899Z","shell.execute_reply":"2023-04-12T12:36:47.970889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Because of not enough storage on Kaggle Output, we cannot load feature_data.npy and feature_labels.npy.Please have a look at the CNN-LSTM MODEL.ipynb.","metadata":{}}]}