{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Isolated Sign Language Recognition with STGCN\n\nIn this notebook, I will create Sign Language Recognition model using STGCN. To build an efficient training pipeline, I will use TFRecord Dataset from https://www.kaggle.com/datasets/lonnieqin/islr-12-time-steps-tfrecords created by notebook https://www.kaggle.com/code/lonnieqin/islr-create-tfrecord for training.\nThe ST-GCN model archetecture was adapated from https://github.com/kdkalvik/ST-GCN\nIt will take about 1 hour to finish runing this notebook using GPU.","metadata":{"papermill":{"duration":0.008117,"end_time":"2023-03-02T08:44:33.967124","exception":false,"start_time":"2023-03-02T08:44:33.959007","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Configuration","metadata":{"papermill":{"duration":0.006539,"end_time":"2023-03-02T08:44:33.994281","exception":false,"start_time":"2023-03-02T08:44:33.987742","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    data_path = \"../input/asl-signs/\"\n    tf_record_path = \"/kaggle/input/islr-12-time-steps-tfrecords/\"\n    sequence_length = 12\n    rows_per_frame = 543\n    is_training = True\n    \n\n","metadata":{"papermill":{"duration":0.022144,"end_time":"2023-03-02T08:44:34.023146","exception":false,"start_time":"2023-03-02T08:44:34.001002","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:56.614781Z","iopub.execute_input":"2023-04-11T12:58:56.615563Z","iopub.status.idle":"2023-04-11T12:58:56.621239Z","shell.execute_reply.started":"2023-04-11T12:58:56.615519Z","shell.execute_reply":"2023-04-11T12:58:56.62016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Libraries","metadata":{"papermill":{"duration":0.006464,"end_time":"2023-03-02T08:44:34.036262","exception":false,"start_time":"2023-03-02T08:44:34.029798","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tqdm import tqdm\nimport json\nimport time\nimport os\nfrom kaggle_datasets import KaggleDatasets\nnp.random.seed(16)\ntf.random.set_seed(16)","metadata":{"papermill":{"duration":7.496016,"end_time":"2023-03-02T08:44:41.539085","exception":false,"start_time":"2023-03-02T08:44:34.043069","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:56.626196Z","iopub.execute_input":"2023-04-11T12:58:56.626527Z","iopub.status.idle":"2023-04-11T12:58:56.668924Z","shell.execute_reply.started":"2023-04-11T12:58:56.626489Z","shell.execute_reply":"2023-04-11T12:58:56.667134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities","metadata":{"papermill":{"duration":0.006496,"end_time":"2023-03-02T08:44:41.597734","exception":false,"start_time":"2023-03-02T08:44:41.591238","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\n\ndef load_relevant_data_subset_with_imputation(pq_path):\n    data_columns = ['x', 'y']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    data.replace(np.nan, 0, inplace=True)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float16)\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\ndef read_dict(file_path):\n    path = os.path.expanduser(file_path)\n    with open(path, \"r\") as f:\n        dic = json.load(f)\n    return dic","metadata":{"papermill":{"duration":0.017828,"end_time":"2023-03-02T08:44:41.622261","exception":false,"start_time":"2023-03-02T08:44:41.604433","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:56.674507Z","iopub.execute_input":"2023-04-11T12:58:56.675078Z","iopub.status.idle":"2023-04-11T12:58:56.686446Z","shell.execute_reply.started":"2023-04-11T12:58:56.675049Z","shell.execute_reply":"2023-04-11T12:58:56.68512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{"papermill":{"duration":0.006542,"end_time":"2023-03-02T08:44:41.635478","exception":false,"start_time":"2023-03-02T08:44:41.628936","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv(f\"{CFG.data_path}train.csv\")\ntrain.head()","metadata":{"papermill":{"duration":0.206559,"end_time":"2023-03-02T08:44:41.848795","exception":false,"start_time":"2023-03-02T08:44:41.642236","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:56.689534Z","iopub.execute_input":"2023-04-11T12:58:56.689809Z","iopub.status.idle":"2023-04-11T12:58:56.798939Z","shell.execute_reply.started":"2023-04-11T12:58:56.689782Z","shell.execute_reply":"2023-04-11T12:58:56.797937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 21 participants. Each of them created about 3000 to 5000 training records.","metadata":{"papermill":{"duration":0.007434,"end_time":"2023-03-02T08:44:41.8635","exception":false,"start_time":"2023-03-02T08:44:41.856066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train.participant_id.nunique()","metadata":{"papermill":{"duration":0.024082,"end_time":"2023-03-02T08:44:41.894435","exception":false,"start_time":"2023-03-02T08:44:41.870353","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:56.802227Z","iopub.execute_input":"2023-04-11T12:58:56.802612Z","iopub.status.idle":"2023-04-11T12:58:56.810963Z","shell.execute_reply.started":"2023-04-11T12:58:56.802581Z","shell.execute_reply":"2023-04-11T12:58:56.809903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.participant_id.value_counts().plot(kind=\"bar\")","metadata":{"papermill":{"duration":0.339293,"end_time":"2023-03-02T08:44:42.240816","exception":false,"start_time":"2023-03-02T08:44:41.901523","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:56.812396Z","iopub.execute_input":"2023-04-11T12:58:56.813164Z","iopub.status.idle":"2023-04-11T12:58:57.114956Z","shell.execute_reply.started":"2023-04-11T12:58:56.813125Z","shell.execute_reply":"2023-04-11T12:58:57.113876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 94477 training samples in total.","metadata":{"papermill":{"duration":0.007382,"end_time":"2023-03-02T08:44:42.255978","exception":false,"start_time":"2023-03-02T08:44:42.248596","status":"completed"},"tags":[]}},{"cell_type":"code","source":"len(train)","metadata":{"papermill":{"duration":0.016757,"end_time":"2023-03-02T08:44:42.280157","exception":false,"start_time":"2023-03-02T08:44:42.2634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:57.116442Z","iopub.execute_input":"2023-04-11T12:58:57.11745Z","iopub.status.idle":"2023-04-11T12:58:57.125193Z","shell.execute_reply.started":"2023-04-11T12:58:57.117408Z","shell.execute_reply":"2023-04-11T12:58:57.123998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 250 kinds of sign languages that we need to make prediction on.","metadata":{"papermill":{"duration":0.007305,"end_time":"2023-03-02T08:44:42.294963","exception":false,"start_time":"2023-03-02T08:44:42.287658","status":"completed"},"tags":[]}},{"cell_type":"code","source":"label_index = read_dict(f\"{CFG.data_path}sign_to_prediction_index_map.json\")\nindex_label = dict([(label_index[key], key) for key in label_index])\nprint(label_index)\ntrain[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\ntrain.head()","metadata":{"papermill":{"duration":0.059155,"end_time":"2023-03-02T08:44:42.361615","exception":false,"start_time":"2023-03-02T08:44:42.30246","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:57.126797Z","iopub.execute_input":"2023-04-11T12:58:57.128139Z","iopub.status.idle":"2023-04-11T12:58:57.175756Z","shell.execute_reply.started":"2023-04-11T12:58:57.128045Z","shell.execute_reply":"2023-04-11T12:58:57.174644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Tensorflow Dataset","metadata":{"papermill":{"duration":0.007629,"end_time":"2023-03-02T08:44:42.377","exception":false,"start_time":"2023-03-02T08:44:42.369371","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def decode_function(record_bytes):\n    return tf.io.parse_single_example(\n          # Data\n          record_bytes,\n          # Schema\n          {\n              \"feature\": tf.io.FixedLenFeature([12 * 543 * 3], dtype=tf.float32),\n              \"label\": tf.io.FixedLenFeature([], dtype=tf.int64)\n          }\n      )\ndef preprocess(item):\n    features = item[\"feature\"]\n#     features = tf.reshape(features, (1,CFG.sequence_length, 543,3))\n    features=tf.reshape(features, (1,12, 543, 3))\n#         \"face\"       : np.arange(0, 468),\n#     \"left_hand\"  : np.arange(468, 489),\n#     \"pose\"       : np.arange(489, 522),\n#     \"right_hand\" : np.arange(522, 543),\n    features=tf.transpose(features, perm=[3, 1, 2, 0])\n    features1=features[:,-5:,468:489,:]\n    features2=features[:,-5:,522:543,:]\n    features=tf.concat([features1, features2],2)\n    print(features.shape)\n    return features, item[\"label\"]         \ndef make_dataset(file_paths, batch_size=128, mode=\"train\"):\n    ds = tf.data.TFRecordDataset(file_paths)\n    ds = ds.map(decode_function)\n    ds = ds.map(preprocess)\n    options = tf.data.Options()\n    if mode == \"train\":\n        ds = ds.shuffle(1024)\n        options.experimental_deterministic = False\n    ds = ds.batch(batch_size, drop_remainder=True)\n    ds = ds.with_options(options) \n    ds = ds.cache().prefetch(tf.data.AUTOTUNE)\n    return ds","metadata":{"papermill":{"duration":0.019748,"end_time":"2023-03-02T08:44:42.404541","exception":false,"start_time":"2023-03-02T08:44:42.384793","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:57.179554Z","iopub.execute_input":"2023-04-11T12:58:57.179919Z","iopub.status.idle":"2023-04-11T12:58:57.189544Z","shell.execute_reply.started":"2023-04-11T12:58:57.179883Z","shell.execute_reply":"2023-04-11T12:58:57.188445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_ids = np.array(sorted(train.participant_id.unique()))\ntrain_ds = make_dataset([f\"{CFG.tf_record_path}{identifier}.tfrecords\" for identifier in unique_ids[:-4]])\nvalid_ds = make_dataset([f\"{CFG.tf_record_path}{identifier}.tfrecords\" for identifier in unique_ids[-4:]], mode=\"valid\")","metadata":{"papermill":{"duration":0.017938,"end_time":"2023-03-02T08:44:42.43015","exception":false,"start_time":"2023-03-02T08:44:42.412212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:57.19104Z","iopub.execute_input":"2023-04-11T12:58:57.191684Z","iopub.status.idle":"2023-04-11T12:58:57.337322Z","shell.execute_reply.started":"2023-04-11T12:58:57.191648Z","shell.execute_reply":"2023-04-11T12:58:57.336122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n\ndef edge2mat(link, num_node):\n    A = np.zeros((num_node, num_node))\n    for i, j in link:\n        A[j, i] = 1\n    return A\n\n\ndef normalize_digraph(A):  # 除以每列的和\n    Dl = np.sum(A, 0)\n    h, w = A.shape\n    Dn = np.zeros((w, w))\n    for i in range(w):\n        if Dl[i] > 0:\n            Dn[i, i] = Dl[i] ** (-1)\n    AD = np.dot(A, Dn)\n    return AD\n\n\ndef get_spatial_graph(num_node, self_link, inward, outward):\n    I = edge2mat(self_link, num_node)\n    In = normalize_digraph(edge2mat(inward, num_node))\n    Out = normalize_digraph(edge2mat(outward, num_node))\n    A = np.stack((I, In, Out))\n    return A\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:58:57.33942Z","iopub.execute_input":"2023-04-11T12:58:57.34014Z","iopub.status.idle":"2023-04-11T12:58:57.349775Z","shell.execute_reply.started":"2023-04-11T12:58:57.340082Z","shell.execute_reply":"2023-04-11T12:58:57.348625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![handlandmark](https://developers.google.com/static/mediapipe/images/solutions/hand-landmarks.png) create the node graph for hand landmarks ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_node = 42\nself_link = [(i, i) for i in range(num_node)]\ninward_ori_index = [(1, 2), (2, 3), (3, 4), (4, 5), (1, 6), (6, 7), (7, 8),\n                    (8, 9), (6, 10), (10, 11), (11, 12), (12, 13), (10, 14),\n                    (14, 15), (15, 16), (16, 17), (14, 18), (18, 19), (19, 20),\n                    (20, 21), (18, 1)]\ninward_ori_index2=[(1+21, 2+21), (2+21, 3+21), (3+21, 4+21), (4+21, 5+21), (1+21, 6), \n                   (6+21, 7+21), (7+21, 8+21), (8+21, 9+21), (6+21, 10+21), \n                    (10+21, 11+21), (11+21, 12+21), (12+21, 13+21), (10+21, 14+21),\n                    (14+21, 15+21), (15+21, 16+21), (16+21, 17+21), (14+21, 18+21), (18+21, 19+21), (19+21, 20+21),\n                    (20+21, 21+21), (18+21, 1+21)]\ninward_ori_index.extend(inward_ori_index2)\ninward = [(i - 1, j - 1) for (i, j) in inward_ori_index]\noutward = [(j, i) for (i, j) in inward]\nneighbor = inward + outward","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:58:57.351521Z","iopub.execute_input":"2023-04-11T12:58:57.351992Z","iopub.status.idle":"2023-04-11T12:58:57.36521Z","shell.execute_reply.started":"2023-04-11T12:58:57.351954Z","shell.execute_reply":"2023-04-11T12:58:57.36397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n\nclass Graph:\n    def __init__(self, labeling_mode='spatial'):\n        self.A = self.get_adjacency_matrix(labeling_mode)\n        self.num_node = num_node\n        self.self_link = self_link\n        self.inward = inward\n        self.outward = outward\n        self.neighbor = neighbor\n\n    def get_adjacency_matrix(self, labeling_mode=None):\n        if labeling_mode is None:\n            return self.A\n        if labeling_mode == 'spatial':\n            A = get_spatial_graph(num_node, self_link, inward, outward)\n        else:\n            raise ValueError()\n        return A\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:58:57.366764Z","iopub.execute_input":"2023-04-11T12:58:57.367147Z","iopub.status.idle":"2023-04-11T12:58:57.375429Z","shell.execute_reply.started":"2023-04-11T12:58:57.367105Z","shell.execute_reply":"2023-04-11T12:58:57.374151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{"papermill":{"duration":0.007677,"end_time":"2023-03-02T08:44:42.445922","exception":false,"start_time":"2023-03-02T08:44:42.438245","status":"completed"},"tags":[]}},{"cell_type":"code","source":"REGULARIZER = tf.keras.regularizers.l2(l=0.001)\nINITIALIZER = tf.keras.initializers.VarianceScaling(scale=2.,\n                                                    mode=\"fan_out\",\n                                                    distribution=\"truncated_normal\")\nclass SGCN(tf.keras.Model):\n    def __init__(self, filters, kernel_size=3):\n        super().__init__()\n        self.kernel_size = kernel_size\n        self.conv = tf.keras.layers.Conv2D(filters*kernel_size,\n                                           kernel_size=1,\n                                           padding='same',\n                                           kernel_initializer=INITIALIZER,\n                                           data_format='channels_first',\n                                           kernel_regularizer=REGULARIZER)\n\n    # N, C, T, V\n    def call(self, x, A, training):\n        x = self.conv(x)\n\n        N = tf.shape(x)[0]\n        C = tf.shape(x)[1]\n        T = tf.shape(x)[2]\n        V = tf.shape(x)[3]\n\n        x = tf.reshape(x, [N, self.kernel_size, C//self.kernel_size, T, V])\n        x = tf.einsum('nkctv,kvw->nctw', x, A)\n        return x, A\n\n\n\"\"\"Applies a spatial temporal graph convolution over an input graph sequence.\n    Args:\n        filters (int): Number of channels produced by the convolution\n        kernel_size (tuple): Size of the temporal convolving kernel and graph convolving kernel\n        stride (int, optional): Stride of the temporal convolution. Default: 1\n        activation (activation function/name, optional): activation function to use\n        residual (bool, optional): If ``True``, applies a residual mechanism. Default: ``True``\n        downsample (bool, optional): If ``True``, applies a downsampling residual mechanism. Default: ``True``\n                                     the value is used only when residual is ``True``\n    Shape:\n        - Input[0]: Input graph sequence in :math:`(N, in_channels, T_{in}, V)` format\n        - Input[1]: Input graph adjacency matrix in :math:`(K, V, V)` format\n        - Output[0]: Outpu graph sequence in :math:`(N, out_channels, T_{out}, V)` format\n        - Output[1]: Graph adjacency matrix for output data in :math:`(K, V, V)` format\n        where\n            :math:`N` is a batch size,\n            :math:`K` is the spatial kernel size, as :math:`K == kernel_size[1]`,\n            :math:`T_{in}/T_{out}` is a length of input/output sequence,\n            :math:`V` is the number of graph nodes.\n\"\"\"\nclass STGCN(tf.keras.Model):\n    def __init__(self, filters, kernel_size=[9, 3], stride=1, activation='relu',\n                 residual=True, downsample=False):\n        super().__init__()\n        self.sgcn = SGCN(filters, kernel_size=kernel_size[1])\n\n        self.tgcn = tf.keras.Sequential()\n        self.tgcn.add(tf.keras.layers.BatchNormalization(axis=1))\n        self.tgcn.add(tf.keras.layers.Activation(activation))\n        self.tgcn.add(tf.keras.layers.Conv2D(filters,\n                                                kernel_size=[kernel_size[0], 1],\n                                                strides=[stride, 1],\n                                                padding='same',\n                                                kernel_initializer=INITIALIZER,\n                                                data_format='channels_first',\n                                                kernel_regularizer=REGULARIZER))\n        self.tgcn.add(tf.keras.layers.BatchNormalization(axis=1))\n\n        self.act = tf.keras.layers.Activation(activation)\n\n        if not residual:\n            self.residual = lambda x, training=False: 0\n        elif residual and stride == 1 and not downsample:\n            self.residual = lambda x, training=False: x\n        else:\n            self.residual = tf.keras.Sequential()\n            self.residual.add(tf.keras.layers.Conv2D(filters,\n                                                        kernel_size=[1, 1],\n                                                        strides=[stride, 1],\n                                                        padding='same',\n                                                        kernel_initializer=INITIALIZER,\n                                                        data_format='channels_first',\n                                                        kernel_regularizer=REGULARIZER))\n            self.residual.add(tf.keras.layers.BatchNormalization(axis=1))\n\n    def call(self, x, A, training=True):\n        res = self.residual(x, training=training)\n        x, A = self.sgcn(x, A, training=training)\n        x = self.tgcn(x, training=training)\n        x += res\n        x = self.act(x)\n        return x, A\n\n\n\"\"\"Spatial temporal graph convolutional networks.\n    Args:\n        num_class (int): Number of classes for the classification task\n    Shape:\n        - Input: :math:`(N, in_channels, T_{in}, V_{in}, M_{in})`\n        - Output: :math:`(N, num_class)` where\n            :math:`N` is a batch size,\n            :math:`T_{in}` is a length of input sequence,\n            :math:`V_{in}` is the number of graph nodes,\n            :math:`M_{in}` is the number of instance in a frame.\n\"\"\"\nclass Model(tf.keras.Model):\n    def __init__(self, num_classes=250):\n        super().__init__()\n\n        graph = Graph()\n        self.A = tf.Variable(graph.A,\n                             dtype=tf.float32,\n                             trainable=False,\n                             name='adjacency_matrix')\n\n        self.data_bn = tf.keras.layers.BatchNormalization(axis=1)\n\n        self.STGCN_layers = []\n        self.STGCN_layers.append(STGCN(64, residual=False))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(64))\n        self.STGCN_layers.append(STGCN(128, stride=2, downsample=True))\n        self.STGCN_layers.append(STGCN(128))\n        self.STGCN_layers.append(STGCN(128))\n        self.STGCN_layers.append(STGCN(256, stride=2, downsample=True))\n        self.STGCN_layers.append(STGCN(256))\n        self.STGCN_layers.append(STGCN(256))\n\n        self.pool = tf.keras.layers.GlobalAveragePooling2D(data_format='channels_first')\n\n        self.logits = tf.keras.layers.Conv2D(num_classes,\n                                             kernel_size=1,\n                                             padding='same',\n                                             kernel_initializer=INITIALIZER,\n                                             data_format='channels_first',\n                                             kernel_regularizer=REGULARIZER)\n\n    def call(self, x, training):\n        N = tf.shape(x)[0]\n        C = tf.shape(x)[1]\n        T = tf.shape(x)[2]\n        V = tf.shape(x)[3]\n        M = tf.shape(x)[4]\n\n        x = tf.transpose(x, perm=[0, 4, 3, 1, 2])\n        x = tf.reshape(x, [N * M, V * C, T])\n        x = self.data_bn(x, training=training)\n        x = tf.reshape(x, [N, M, V, C, T])\n        x = tf.transpose(x, perm=[0, 1, 3, 4, 2])\n        x = tf.reshape(x, [N * M, C, T, V])\n\n        A = self.A\n        for layer in self.STGCN_layers:\n            x, A = layer(x, A, training=training)\n\n        # N*M,C,T,V\n        x = self.pool(x)\n        x = tf.reshape(x, [N, M, -1, 1, 1])\n        x = tf.reduce_mean(x, axis=1)\n        x = self.logits(x)\n        x = tf.reshape(x, [N, -1])\n#         print(x.shape)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2023-04-11T13:11:12.896395Z","iopub.execute_input":"2023-04-11T13:11:12.896973Z","iopub.status.idle":"2023-04-11T13:11:12.926081Z","shell.execute_reply.started":"2023-04-11T13:11:12.896933Z","shell.execute_reply":"2023-04-11T13:11:12.924974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model =  Model(num_classes=250)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T13:11:22.985955Z","iopub.execute_input":"2023-04-11T13:11:22.98688Z","iopub.status.idle":"2023-04-11T13:11:23.123583Z","shell.execute_reply.started":"2023-04-11T13:11:22.986841Z","shell.execute_reply":"2023-04-11T13:11:23.122505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate an optimizer to train the model.\noptimizer = tf.keras.optimizers.Adam()\n\n# Instantiate a loss function\nloss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n\n# Prepare the metrics.\ntrain_acc_metric = tf.keras.metrics.CategoricalAccuracy()\nval_acc_metric   = tf.keras.metrics.CategoricalAccuracy()\n\n# tensorboard writer \ntrain_writer = tf.summary.create_file_writer('logs/train/')\ntest_writer  = tf.summary.create_file_writer('logs/test/')\n\n\n@tf.function\ndef train_step(step, x, y):\n   '''\n   input: x, y <- typically batches \n   input: step <- batch step\n   return: loss value\n   '''\n\n    # start the scope of gradient \n   with tf.GradientTape() as tape:\n        logits = model(x, training=True) # forward pass\n        train_loss_value = loss_fn(y, logits) # compute loss \n\n    # compute gradient \n   grads = tape.gradient(train_loss_value, model.trainable_weights)\n\n    # update weights\n   optimizer.apply_gradients(zip(grads, model.trainable_weights))\n\n    # update metrics\n   train_acc_metric.update_state(y, logits)\n    \n    # write training loss and accuracy to the tensorboard\n   with train_writer.as_default():\n        tf.summary.scalar('loss', train_loss_value, step=step)\n        tf.summary.scalar(\n            'accuracy', train_acc_metric.result(), step=step\n        ) \n   return train_loss_value\n\n\n@tf.function\ndef test_step(step, x, y):\n   '''\n   input: x, y <- typically batches \n   input: step <- batch step\n   return: loss value\n   '''\n    # forward pass, no backprop, inference mode \n    val_logits = model(x, training=False) \n\n    # Compute the loss value \n    val_loss_value = loss_fn(y, val_logits)\n\n    # Update val metrics\n    val_acc_metric.update_state(y, val_logits)\n    \n    # write test loss and accuracy to the tensorboard\n    with test_writer.as_default():\n        tf.summary.scalar('val loss', val_loss_value, step=step)\n        tf.summary.scalar(\n            'val accuracy', val_acc_metric.result(), step=step\n        ) \n    return val_loss_value\n\n\n# custom training loop \nfor epoch in range(100):\n    t = time.time()\n    # batch training \n\n    # Iterate over the batches of the train dataset.\n    for train_batch_step, (x_batch_train, \\\n                           y_batch_train) in enumerate(train_dataset):\n        train_batch_step = tf.convert_to_tensor(\n                                train_batch_step, dtype=tf.int64\n                           )\n        train_loss_value = train_step(\n                                train_batch_step, \n                                x_batch_train, y_batch_train\n                           )\n\n    # evaluation on validation set \n    # Run a validation loop at the end of each epoch.\n    for test_batch_step, (x_batch_val, \\\n                          y_batch_val) in enumerate(val_dataset):\n        test_batch_step = tf.convert_to_tensor(\n                               test_batch_step, dtype=tf.int64\n                          )\n        val_loss_value = test_step(\n                                test_batch_step, x_batch_val, y_batch_val\n                          )\n\n\n    template = '\n        ETA: {} - epoch: {} loss: {}  acc: {} val loss: {} val acc: {}\\n\n    '\n    print(template.format(\n        round((time.time() - t)/60, 2), epoch + 1,\n        train_loss_value, float(train_acc_metric.result()),\n        val_loss_value, float(val_acc_metric.result())\n    ))\n        \n    # Reset metrics at the end of each epoch\n    train_acc_metric.reset_states()\n    val_acc_metric.reset_states()","metadata":{"execution":{"iopub.status.busy":"2023-04-11T13:14:02.971387Z","iopub.execute_input":"2023-04-11T13:14:02.971926Z","iopub.status.idle":"2023-04-11T13:14:02.986496Z","shell.execute_reply.started":"2023-04-11T13:14:02.971887Z","shell.execute_reply":"2023-04-11T13:14:02.985017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_model():\n          \n#     model =  Model(num_classes=250)\n#     model.build((128,3,5,42,1))\n#     model.compile(\n#         loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\n#             \"accuracy\",\n#         ]\n#     )\n#     return model","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:58:57.410823Z","iopub.execute_input":"2023-04-11T12:58:57.411215Z","iopub.status.idle":"2023-04-11T12:58:57.419881Z","shell.execute_reply.started":"2023-04-11T12:58:57.411175Z","shell.execute_reply":"2023-04-11T12:58:57.418879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def conv1d_lstm_block(inputs, filters):\n#     vector = tf.keras.layers.ConvLSTM1D(filters=32, kernel_size=8)(inputs)\n#     for f in filters:\n#         vector = tf.keras.layers.Conv1D(filters=f, kernel_size=8)(vector)\n#         vector = tf.keras.layers.MaxPooling1D()(vector)\n#     vector = tf.keras.layers.Dropout(0.3)(vector)\n#     return vector\n\n# def get_model():\n#     inputs = tf.keras.Input((CFG.sequence_length, 543, 3), dtype=tf.float32)\n#     face_inputs = inputs[:, :, 0:468, :]\n#     left_hand_inputs = inputs[:, :, 468:489, :]\n#     pose_inputs = inputs[:, :, 489:522, :]\n#     right_hand_inputs = inputs[:, :,522:,:]\n#     face_vector = conv1d_lstm_block(face_inputs, [32, 64])\n#     left_hand_vector = conv1d_lstm_block(left_hand_inputs, [64])\n#     right_hand_vector = conv1d_lstm_block(right_hand_inputs, [64])\n#     pose_vector = conv1d_lstm_block(pose_inputs, [64])\n#     vector = tf.keras.layers.Concatenate(axis=1)([face_vector, left_hand_vector, right_hand_vector, pose_vector])\n#     vector = tf.keras.layers.Flatten()(vector)\n#     output = tf.keras.layers.Dense(250, activation=\"softmax\")(vector)\n#     model = tf.keras.Model(inputs=inputs, outputs=output)\n#     model.compile(\n#         loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\n#             \"accuracy\",\n#         ]\n#     )\n#     return model","metadata":{"papermill":{"duration":0.252992,"end_time":"2023-03-02T08:44:42.706685","exception":false,"start_time":"2023-03-02T08:44:42.453693","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:57.421548Z","iopub.execute_input":"2023-04-11T12:58:57.421942Z","iopub.status.idle":"2023-04-11T12:58:57.429854Z","shell.execute_reply.started":"2023-04-11T12:58:57.421901Z","shell.execute_reply":"2023-04-11T12:58:57.428813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model()\nmodel.summary()\ntf.keras.utils.plot_model(model, show_shapes=True)","metadata":{"papermill":{"duration":3.974692,"end_time":"2023-03-02T08:44:46.689625","exception":false,"start_time":"2023-03-02T08:44:42.714933","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T12:58:57.431467Z","iopub.execute_input":"2023-04-11T12:58:57.432016Z","iopub.status.idle":"2023-04-11T12:58:58.610898Z","shell.execute_reply.started":"2023-04-11T12:58:57.431981Z","shell.execute_reply":"2023-04-11T12:58:58.609576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://towardsdatascience.com/model-sub-classing-and-custom-training-loop-from-scratch-in-tensorflow-2-cc1d4f10fb4e\noptimizer = tf.keras.optimizers.Adam()\ntrain_acc_metric = tf.keras.metrics.CategoricalAccuracy()\nval_acc_metric   = tf.keras.metrics.CategoricalAccuracy()\n# Instantiate a loss function\n# loss_fn = tf.keras.losses.CategoricalCrossentropy()\nloss_fn=tf.keras.losses.SparseCategoricalCrossentropy()\nfor epoch in range(30): # <----- start for loop, step 1\n\n  # <-------- start for loop, step 2\n  # Iterate over the batches of the dataset.\n  for step, (x_batch_train, y_batch_train) in enumerate(train_ds):\n\n    # <-------- start gradient tape scope, step 3\n    # Open a GradientTape to record the operations run\n    # during the forward pass, which enables auto-differentiation.\n    with tf.GradientTape() as tape:\n\n       # Run the forward pass of the layer.\n       # The operations that the layer applies\n       # to its inputs are going to be recorded\n       # on the GradientTape.\n       logits = model(x_batch_train, training=True) \n\n       # Compute the loss value for this minibatch.\n       loss_value = loss_fn(y_batch_train, logits)  \n       print(loss_value )\n\n    # compute the gradient of weights w.r.t. loss  <-------- step 5\n    # Use the gradient tape to automatically retrieve\n    # the gradients of the trainable variables with respect to the loss.\n    grads = tape.gradient(loss_value, model.trainable_weights)\n\n    # update the weight based on gradient  <---------- step 6\n    # Run one step of gradient descent by updating\n    # the value of the variables to minimize the loss.\n    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n    train_acc_metric.update_state(y_batch_train, logits)\n    print(train_acc_metric)","metadata":{"execution":{"iopub.status.busy":"2023-04-11T13:19:17.067991Z","iopub.execute_input":"2023-04-11T13:19:17.068946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.is_training:\n    file_name = \"model.h5\"\n#     callbacks = [\n#         tf.keras.callbacks.ModelCheckpoint(\n#             file_name, \n#             save_best_only=True, \n#             restore_best_weights=True, \n#             monitor=\"val_accuracy\",\n#             mode=\"max\"\n#         ),\n#         tf.keras.callbacks.EarlyStopping(\n#             patience=5, \n#             monitor=\"val_accuracy\",\n#             mode=\"max\"\n#         )\n#     ]\n    model.fit(train_ds, epochs=1, validation_data=valid_ds)\n    model.save('/kaggle/input/islr-convlstm1d/model.h5',save_format='tf')\n    model = tf.keras.models.load_model(file_name)\n# else:\n#     model = tf.keras.models.load_model(\"/kaggle/input/islr-convlstm1d/model.h5\")\nmodel.evaluate(valid_ds)   ","metadata":{"execution":{"iopub.status.busy":"2023-04-11T12:58:58.614684Z","iopub.execute_input":"2023-04-11T12:58:58.615002Z","iopub.status.idle":"2023-04-11T13:00:08.746327Z","shell.execute_reply.started":"2023-04-11T12:58:58.614967Z","shell.execute_reply":"2023-04-11T13:00:08.744036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Model for inference","metadata":{"papermill":{"duration":0.069654,"end_time":"2023-03-02T08:47:03.916954","exception":false,"start_time":"2023-03-02T08:47:03.8473","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# def get_inference_model(model):\n#     inputs = tf.keras.Input((543, 3), dtype=tf.float32, name=\"inputs\")\n#     vector = tf.image.resize(inputs, (CFG.sequence_length, 543))\n#     vector = tf.where(tf.math.is_nan(vector), tf.zeros_like(vector), vector)\n#     vector = tf.expand_dims(vector, axis=0)\n#     vector = model(vector)\n#     output = tf.keras.layers.Activation(activation=\"linear\", name=\"outputs\")(vector)\n#     inference_model = tf.keras.Model(inputs=inputs, outputs=output) \n#     inference_model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=[\"accuracy\"])\n#     return inference_model","metadata":{"papermill":{"duration":0.086334,"end_time":"2023-03-02T08:47:04.072776","exception":false,"start_time":"2023-03-02T08:47:03.986442","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T13:00:08.747883Z","iopub.status.idle":"2023-04-11T13:00:08.748454Z","shell.execute_reply.started":"2023-04-11T13:00:08.748177Z","shell.execute_reply":"2023-04-11T13:00:08.748205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inference_model = get_inference_model(model)\n# inference_model.summary()\n# tf.keras.utils.plot_model(inference_model, show_shapes=True)","metadata":{"papermill":{"duration":5.194577,"end_time":"2023-03-02T08:47:09.336507","exception":false,"start_time":"2023-03-02T08:47:04.14193","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T13:00:08.750352Z","iopub.status.idle":"2023-04-11T13:00:08.750858Z","shell.execute_reply.started":"2023-04-11T13:00:08.750603Z","shell.execute_reply":"2023-04-11T13:00:08.750633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create submission file","metadata":{"papermill":{"duration":0.071953,"end_time":"2023-03-02T08:47:09.784691","exception":false,"start_time":"2023-03-02T08:47:09.712738","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# converter = tf.lite.TFLiteConverter.from_keras_model(inference_model)\n# tflite_model = converter.convert()\n# model_path = \"model.tflite\"\n# # Save the model.\n# with open(model_path, 'wb') as f:\n#     f.write(tflite_model)\n# !zip submission.zip $model_path","metadata":{"papermill":{"duration":153.429402,"end_time":"2023-03-02T08:49:43.286145","exception":false,"start_time":"2023-03-02T08:47:09.856743","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T13:00:08.752458Z","iopub.status.idle":"2023-04-11T13:00:08.758015Z","shell.execute_reply.started":"2023-04-11T13:00:08.757823Z","shell.execute_reply":"2023-04-11T13:00:08.757844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Predictions","metadata":{"papermill":{"duration":0.1124,"end_time":"2023-03-02T08:49:46.762554","exception":false,"start_time":"2023-03-02T08:49:46.650154","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# !pip install tflite-runtime","metadata":{"papermill":{"duration":12.663338,"end_time":"2023-03-02T08:49:59.538473","exception":false,"start_time":"2023-03-02T08:49:46.875135","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T13:00:08.759302Z","iopub.status.idle":"2023-04-11T13:00:08.759928Z","shell.execute_reply.started":"2023-04-11T13:00:08.759618Z","shell.execute_reply":"2023-04-11T13:00:08.759646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tflite_runtime.interpreter as tflite\n# interpreter = tflite.Interpreter(model_path)\n# found_signatures = list(interpreter.get_signature_list().keys())\n# prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n# for i in tqdm(range(10000)):\n#     frames = load_relevant_data_subset(f'/kaggle/input/asl-signs/{train.iloc[i].path}')\n#     output = prediction_fn(inputs=frames)\n#     if i < 100:\n#         sign = np.argmax(output[\"outputs\"])\n#         print(f\"Predicted label: {index_label[sign]}, Actual Label: {train.iloc[i].sign}\")","metadata":{"papermill":{"duration":29.201279,"end_time":"2023-03-02T08:50:28.816671","exception":false,"start_time":"2023-03-02T08:49:59.615392","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-11T13:00:08.761722Z","iopub.status.idle":"2023-04-11T13:00:08.762695Z","shell.execute_reply.started":"2023-04-11T13:00:08.762388Z","shell.execute_reply":"2023-04-11T13:00:08.762418Z"},"trusted":true},"execution_count":null,"outputs":[]}]}