{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-08T18:29:56.937365Z","iopub.execute_input":"2023-03-08T18:29:56.938067Z","iopub.status.idle":"2023-03-08T18:30:07.413287Z","shell.execute_reply.started":"2023-03-08T18:29:56.938014Z","shell.execute_reply":"2023-03-08T18:30:07.411919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"QUICK_TEST = False\nQUICK_LIMIT = 200","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:07.415646Z","iopub.execute_input":"2023-03-08T18:30:07.416645Z","iopub.status.idle":"2023-03-08T18:30:07.422506Z","shell.execute_reply.started":"2023-03-08T18:30:07.416600Z","shell.execute_reply":"2023-03-08T18:30:07.421290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LANDMARK_FILES_DIR = \"/kaggle/input/asl-signs/train_landmark_files\"\nTRAIN_FILE = \"/kaggle/input/asl-signs/train.csv\"\nlabel_map = json.load(open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\", \"r\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:07.424003Z","iopub.execute_input":"2023-03-08T18:30:07.424822Z","iopub.status.idle":"2023-03-08T18:30:07.449374Z","shell.execute_reply.started":"2023-03-08T18:30:07.424785Z","shell.execute_reply":"2023-03-08T18:30:07.448048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:07.452339Z","iopub.execute_input":"2023-03-08T18:30:07.452726Z","iopub.status.idle":"2023-03-08T18:30:07.459714Z","shell.execute_reply.started":"2023-03-08T18:30:07.452686Z","shell.execute_reply":"2023-03-08T18:30:07.458370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def right_hand_percentage(x):\n    right = tf.gather(x, right_hand_landmarks, axis=1)\n    left = tf.gather(x, left_hand_landmarks, axis=1)\n    right_count = tf.reduce_sum(tf.where(tf.math.is_nan(right), tf.zeros_like(right), tf.ones_like(right)))\n    left_count = tf.reduce_sum(tf.where(tf.math.is_nan(left), tf.zeros_like(left), tf.ones_like(left)))\n    return right_count / (left_count+right_count)","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:07.461469Z","iopub.execute_input":"2023-03-08T18:30:07.461805Z","iopub.status.idle":"2023-03-08T18:30:07.474349Z","shell.execute_reply.started":"2023-03-08T18:30:07.461773Z","shell.execute_reply":"2023-03-08T18:30:07.473299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PREPROCESSING","metadata":{}},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"NUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET+21\nRIGHT_HAND_OFFSET = POSE_OFFSET+33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [61, 185, 40, 39, 37,  0, 267, 269, 270, 409,\n                 291,146, 91,181, 84, 17, 314, 405, 321, 375, \n                 78, 191, 80, 81, 82, 13, 312, 311, 310, 415, \n                 95, 88, 178, 87, 14,317, 402, 318, 324, 308]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET+21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET+21))\n\npoint_landmarks = [item for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks] for item in sublist]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nINPUT_SHAPE = (NUM_FRAMES,LANDMARKS*3)","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:07.476177Z","iopub.execute_input":"2023-03-08T18:30:07.476535Z","iopub.status.idle":"2023-03-08T18:30:07.489113Z","shell.execute_reply.started":"2023-03-08T18:30:07.476501Z","shell.execute_reply":"2023-03-08T18:30:07.488034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions","metadata":{}},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis)\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std  = tf_nan_std(x,  axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1]*2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:07.490801Z","iopub.execute_input":"2023-03-08T18:30:07.491218Z","iopub.status.idle":"2023-03-08T18:30:07.501489Z","shell.execute_reply.started":"2023-03-08T18:30:07.491181Z","shell.execute_reply":"2023-03-08T18:30:07.500180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### TensorFlow Feature Preprocessing Layer","metadata":{}},{"cell_type":"code","source":"class FeatureGen(tf.keras.layers.Layer):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n    \n    def call(self, x_in):\n#         print(right_hand_percentage(x))\n        x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in averaging_sets]\n        x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n        x = tf.concat(x_list, 1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n            p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n        \n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n        x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x\n\nfeature_converter = FeatureGen()\n\n## One tests symbolic tensor, the other tests real data.\nprint(feature_converter(tf.keras.Input((543, 3), dtype=tf.float32, name=\"inputs\")))\nfeature_converter(load_relevant_data_subset(f'/kaggle/input/asl-signs/{pd.read_csv(TRAIN_FILE).path[1]}'))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-03-08T18:30:07.503037Z","iopub.execute_input":"2023-03-08T18:30:07.503467Z","iopub.status.idle":"2023-03-08T18:30:08.867726Z","shell.execute_reply.started":"2023-03-08T18:30:07.503430Z","shell.execute_reply":"2023-03-08T18:30:08.866544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Convert all train data up front","metadata":{}},{"cell_type":"code","source":"def convert_row(row, right_handed=True):\n    x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n    x = feature_converter(tf.convert_to_tensor(x)).cpu().numpy()\n    return x, row[1].label\n\nright_handed_signer = [26734, 28656, 25571, 62590, 29302, \n                       49445, 53618, 18796,  4718,  2044, \n                       37779, 30680]\nleft_handed_signer  = [16069, 32319, 36257, 22343, 27610, \n                       61333, 34503, 55372, ]\nboth_hands_signer   = [37055, ]\n\nmessy = [29302, ]\n\ndef convert_and_save_data():\n    df = pd.read_csv(TRAIN_FILE)\n    df['label'] = df['sign'].map(label_map)\n    total = df.shape[0]\n    if QUICK_TEST:\n        total = QUICK_LIMIT\n    npdata = np.zeros((total, INPUT_SHAPE[0]*INPUT_SHAPE[1] + (SEGMENTS+1)*INPUT_SHAPE[1]*2))\n    nplabels = np.zeros(total)\n    for i, row in tqdm(enumerate(df.iterrows()), total=total):\n        (x,y) = convert_row(row)\n        npdata[i,:] = x\n        nplabels[i] = y\n        if QUICK_TEST and i == QUICK_LIMIT - 1:\n            break\n    \n    np.save(\"feature_data.npy\", npdata)\n    np.save(\"feature_labels.npy\", nplabels)\n        \nconvert_and_save_data()","metadata":{"execution":{"iopub.status.busy":"2023-03-08T18:30:08.869064Z","iopub.execute_input":"2023-03-08T18:30:08.869401Z","iopub.status.idle":"2023-03-08T19:42:17.633858Z","shell.execute_reply.started":"2023-03-08T18:30:08.869369Z","shell.execute_reply":"2023-03-08T19:42:17.631702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"X = np.load(\"feature_data.npy\")\ny = np.load(\"feature_labels.npy\")\nprint(X.shape, y.shape)\n\nprint(X[0, :].shape, X[0, :])","metadata":{"execution":{"iopub.status.busy":"2023-03-08T19:42:17.640239Z","iopub.execute_input":"2023-03-08T19:42:17.640747Z","iopub.status.idle":"2023-03-08T19:42:21.817410Z","shell.execute_reply.started":"2023-03-08T19:42:17.640690Z","shell.execute_reply":"2023-03-08T19:42:21.816023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}