{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-06T20:01:48.161440Z","iopub.execute_input":"2023-03-06T20:01:48.162608Z","iopub.status.idle":"2023-03-06T20:02:02.188842Z","shell.execute_reply.started":"2023-03-06T20:01:48.162562Z","shell.execute_reply":"2023-03-06T20:02:02.187369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"QUICK_TEST = False\nQUICK_LIMIT = 200","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:02.191457Z","iopub.execute_input":"2023-03-06T20:02:02.192456Z","iopub.status.idle":"2023-03-06T20:02:02.199208Z","shell.execute_reply.started":"2023-03-06T20:02:02.192401Z","shell.execute_reply":"2023-03-06T20:02:02.197845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\nlabel_map = json.load(open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\", \"r\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:02.200640Z","iopub.execute_input":"2023-03-06T20:02:02.201465Z","iopub.status.idle":"2023-03-06T20:02:02.254866Z","shell.execute_reply.started":"2023-03-06T20:02:02.201414Z","shell.execute_reply":"2023-03-06T20:02:02.253660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:02.258528Z","iopub.execute_input":"2023-03-06T20:02:02.259564Z","iopub.status.idle":"2023-03-06T20:02:02.268424Z","shell.execute_reply.started":"2023-03-06T20:02:02.259506Z","shell.execute_reply":"2023-03-06T20:02:02.267281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def right_hand_percentage(x):\n    right = tf.gather(x, right_hand_landmarks, axis=1)\n    left = tf.gather(x, left_hand_landmarks, axis=1)\n    right_count = tf.reduce_sum(tf.where(tf.math.is_nan(right), tf.zeros_like(right), tf.ones_like(right)))\n    left_count = tf.reduce_sum(tf.where(tf.math.is_nan(left), tf.zeros_like(left), tf.ones_like(left)))\n    return right_count / (left_count+right_count)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:02.270465Z","iopub.execute_input":"2023-03-06T20:02:02.271482Z","iopub.status.idle":"2023-03-06T20:02:02.281325Z","shell.execute_reply.started":"2023-03-06T20:02:02.271427Z","shell.execute_reply":"2023-03-06T20:02:02.280335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREPROCESSING","metadata":{}},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"NUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET+21\nRIGHT_HAND_OFFSET = POSE_OFFSET+33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [61, 185, 40, 39, 37,  0, 267, 269, 270, 409,\n                 291,146, 91,181, 84, 17, 314, 405, 321, 375, \n                 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, \n                 95, 88, 178, 87, 14,317, 402, 318, 324, 308]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET+21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET+21))\n\npoint_landmarks = [item for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks] for item in sublist]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nINPUT_SHAPE = (NUM_FRAMES,LANDMARKS*3)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:02.283398Z","iopub.execute_input":"2023-03-06T20:02:02.284328Z","iopub.status.idle":"2023-03-06T20:02:02.298336Z","shell.execute_reply.started":"2023-03-06T20:02:02.284276Z","shell.execute_reply":"2023-03-06T20:02:02.296946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def torch_nan_mean(x, axis=0):\n    return torch.nansum(torch.where(torch.isnan(x), torch.zeros_like(x), x), dim=axis) / torch.nansum(torch.where(torch.isnan(x), torch.zeros_like(x), torch.ones_like(x)), dim=axis)\n\ndef torch_nan_std(x, axis=0):\n    d = x - torch_nan_mean(x, axis=axis)\n    return torch.sqrt(torch_nan_mean(d * d, axis=axis))\n\ndef torch_flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = torch_nan_mean(x, axis=axis)\n    x_std  = torch_nan_std(x,  axis=axis)\n\n    x_out = torch.cat([x_mean, x_std], 0)\n    if x_out.shape[0] > 168:\n        x_out = x_out[:168,:]\n    x_out = x_out.reshape((1, INPUT_SHAPE[1]*2))\n    x_out = torch.where(torch.isfinite(x_out), x_out, torch.zeros_like(x_out))\n    return x_out","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:02.314223Z","iopub.execute_input":"2023-03-06T20:02:02.315245Z","iopub.status.idle":"2023-03-06T20:02:02.333051Z","shell.execute_reply.started":"2023-03-06T20:02:02.315191Z","shell.execute_reply":"2023-03-06T20:02:02.331089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch\n\nclass FeatureGen(nn.Module):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n    \n    def forward(self, x_in):\n#         print(right_hand_percentage(x))\n        \n        x_list = [torch.mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], dim=1, keepdim=True) for av_set in averaging_sets]\n        x_list.append(torch.index_select(x_in, dim=1, index=torch.tensor(point_landmarks)))\n        x = torch.cat(x_list, dim=1)\n        x_padded = x.numpy()\n        for i in range(SEGMENTS):\n            p0 = 1 if (x_padded.shape[0] % SEGMENTS) > 0 and i % 2 != 0 else 0\n            p1 = 1 if (x_padded.shape[0] % SEGMENTS) > 0 and i % 2 == 0 else 0\n            paddings = [(0, 0), (p0, p1), (0, 0)]\n            x_padded = np.pad(x_padded, paddings, mode=\"symmetric\")\n        x_list = torch.tensor_split(torch.tensor(x_padded), SEGMENTS)\n        x_list = [torch_flatten_means_and_stds(_x, axis=0) for _x in x_list]\n        x_list.append(torch_flatten_means_and_stds(x, axis=0))\n        torch_x = torch.where(torch.isfinite(x), x, torch.mean(x[torch.isfinite(x)], dim=0, keepdim=True))\n        x = x.permute(2, 0, 1)\n        x = F.interpolate(x.unsqueeze(0), size=[NUM_FRAMES, LANDMARKS], mode='bilinear')\n        x = x.squeeze(0).permute(1, 2, 0)\n        x = torch.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n        x = torch.where(torch.isnan(x), torch.zeros_like(x), x)\n        x_list.append(x)\n        x = torch.cat(x_list, dim=1)\n        return x\n\nfeature_converter = FeatureGen()\nfeature_converter(torch.tensor(load_relevant_data_subset(f'/kaggle/input/asl-signs/{pd.read_csv(TRAIN_FILE).path[1]}')))","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:49.306735Z","iopub.execute_input":"2023-03-06T20:02:49.307234Z","iopub.status.idle":"2023-03-06T20:02:49.607204Z","shell.execute_reply.started":"2023-03-06T20:02:49.307193Z","shell.execute_reply":"2023-03-06T20:02:49.605696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pytorch Feature Preprocessing Layer","metadata":{}},{"cell_type":"markdown","source":"# Convert all train data up front","metadata":{}},{"cell_type":"code","source":"import multiprocessing as mp\n\ndef convert_row(row, right_handed=True):\n    x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n    x = feature_converter(torch.tensor(x)).cpu().numpy()\n    return x, row[1].label\n\nright_handed_signer = [26734, 28656, 25571, 62590, 29302, \n                       49445, 53618, 18796,  4718,  2044, \n                       37779, 30680]\nleft_handed_signer  = [16069, 32319, 36257, 22343, 27610, \n                       61333, 34503, 55372, ]\nboth_hands_signer   = [37055, ]\n\nmessy = [29302, ]\n\ndef convert_and_save_data():\n    df = pd.read_csv(TRAIN_FILE)\n    df['label'] = df['sign'].map(label_map)\n    total = df.shape[0]\n    if QUICK_TEST:\n        total = QUICK_LIMIT\n    npdata = np.zeros((total, INPUT_SHAPE[0]*INPUT_SHAPE[1] + (SEGMENTS+1)*INPUT_SHAPE[1]*2))\n    nplabels = np.zeros(total)\n    with mp.Pool() as pool:\n        results = pool.imap(convert_row, df.iterrows(), chunksize=250)\n        for i, (x,y) in tqdm(enumerate(results), total=df.shape[0]):\n            npdata[i,:] = x\n            nplabels[i] = y\n            if QUICK_TEST and i == QUICK_LIMIT - 1:\n                break\n    \n    np.save(\"feature_data.npy\", npdata)\n    np.save(\"feature_labels.npy\", nplabels)\n        \nconvert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T20:02:58.511527Z","iopub.execute_input":"2023-03-06T20:02:58.512027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.load(\"feature_data.npy\")\ny = np.load(\"feature_labels.npy\")\nprint(X.shape, y.shape)\n\nprint(X[0, :].shape, X[0, :])","metadata":{"execution":{"iopub.status.busy":"2023-03-04T19:53:05.689622Z","iopub.execute_input":"2023-03-04T19:53:05.689957Z","iopub.status.idle":"2023-03-04T19:53:05.712162Z","shell.execute_reply.started":"2023-03-04T19:53:05.689925Z","shell.execute_reply":"2023-03-04T19:53:05.710726Z"},"trusted":true},"execution_count":null,"outputs":[]}]}