{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport sys\nimport os \n\nmodel_path = \"/kaggle/input/model-files\"\n\nif model_path not in sys.path: sys.path.insert(0, model_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:07:22.033175Z","iopub.execute_input":"2023-04-09T18:07:22.034351Z","iopub.status.idle":"2023-04-09T18:07:22.040319Z","shell.execute_reply.started":"2023-04-09T18:07:22.034286Z","shell.execute_reply":"2023-04-09T18:07:22.038095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# is gpu available?\nimport tensorflow as tf\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:07:22.057106Z","iopub.execute_input":"2023-04-09T18:07:22.057443Z","iopub.status.idle":"2023-04-09T18:07:22.06449Z","shell.execute_reply.started":"2023-04-09T18:07:22.057412Z","shell.execute_reply":"2023-04-09T18:07:22.063237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport math\nimport os \nimport json\n\nROWS_PER_FRAME = 543\n\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\ndef reshape_inputs(x, y=None, block_size=50):\n    t_, lm, c = x.shape\n\n    if t_ < block_size:\n        # Filling coords with zeros and predictions with -1 label\n        difference_x = tf.zeros((block_size - t_, lm, c), dtype=tf.float32)\n        if y is not None:\n            difference_y = tf.zeros((block_size - t_,), dtype=tf.int16) - 1\n\n        x = tf.concat([difference_x, x], axis=0)\n        if y is not None:\n            y = tf.concat([difference_y, y], axis=0)\n\n    elif t_ > block_size:\n        idxs_keep = tf.random.shuffle(tf.range(t_))[:block_size]\n\n        x = tf.gather(x, idxs_keep)\n        if y is not None:\n            y = tf.gather(y, idxs_keep)\n\n    if y is None:\n        return x\n\n    return x, y\n\n\nclass tf_DataSequence(tf.keras.utils.Sequence):\n    def __init__(self, data, sign_to_idx, root_loc=\"/kaggle/input/asl-signs/train_landmark_files/\", batch_size=32, drop_last_batch = True):\n        self.data = data\n        self.root = root_loc\n        self.sign_to_idx = sign_to_idx\n        self.batch_size = batch_size\n        self.drop_last = drop_last_batch\n\n    def __len__(self):\n        num_batches = len(self.data)//self.batch_size\n        \n        if not self.drop_last: \n            if len(self.data) % self.batch_size > 0: num_batches += 1\n        \n        return num_batches\n    \n    def __getitem__(self, idx):\n        \n        batch_x = []\n        batch_y = []\n\n        low = idx * self.batch_size\n        # Cap upper bound at array length; the last batch may be smaller\n        # if the total number of items is not a multiple of batch size.\n        high = min(low + self.batch_size, len(self.data))\n        \n        for i in range(low,high):\n            \n            item = self.data.iloc[i]\n\n            p_idx = item.path \n            path = self.root + item.path.split(\"s/\")[1]\n\n            X_numpy = load_relevant_data_subset(path)\n            t, c, dim = X_numpy.shape\n                    \n            sign = self.data[self.data.path == p_idx].sign.to_numpy()[0]\n            \n            # Dropping z prediction\n            X = tf.convert_to_tensor(X_numpy[:, :, :-1])\n            \n            # Setting NaNs to 0\n            X_ = tf.where(tf.math.is_nan(X), tf.zeros_like(X), X)\n\n            X_ = X_ - tf.reduce_mean(X_, axis=(1, 2), keepdims=True)\n            X_ /= (tf.math.reduce_std(X_, axis=(1, 2), keepdims=True) + 1e-8)\n\n            y = tf.convert_to_tensor([self.sign_to_idx[sign]] * t, dtype=tf.int16)\n            \n            X, y = reshape_inputs(X_, y)\n            \n            batch_y.append(y)\n            batch_x.append(X)\n        \n\n        return tf.stack(batch_x),  tf.stack(batch_y)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:07:22.070839Z","iopub.execute_input":"2023-04-09T18:07:22.071139Z","iopub.status.idle":"2023-04-09T18:07:22.094455Z","shell.execute_reply.started":"2023-04-09T18:07:22.071109Z","shell.execute_reply":"2023-04-09T18:07:22.093273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/asl-signs/train.csv\")\nratio = 0.9\nidxs = np.arange(len(data))\ntrain_idxs = np.random.choice(idxs, size=int(ratio*len(data)), replace=False)\nval_idxs = list(set(idxs)-set(train_idxs))\n\ntrain_data = data.iloc[train_idxs]\nval_data = data.iloc[val_idxs]\n\nwith open(\"/kaggle/input/asl-signs/sign_to_prediction_index_map.json\") as f:\n    sign_to_idx = json.load(f)\n\nidx_to_sign = {v:k for k,v in sign_to_idx.items()}\n\ndloader_train = tf_DataSequence(train_data, sign_to_idx, batch_size=128, drop_last_batch = True)\ndloader_val = tf_DataSequence(val_data, sign_to_idx, batch_size=128, drop_last_batch = True)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:07:22.098616Z","iopub.execute_input":"2023-04-09T18:07:22.09963Z","iopub.status.idle":"2023-04-09T18:07:22.246464Z","shell.execute_reply.started":"2023-04-09T18:07:22.099584Z","shell.execute_reply":"2023-04-09T18:07:22.245288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tqdm","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:07:22.248106Z","iopub.execute_input":"2023-04-09T18:07:22.248843Z","iopub.status.idle":"2023-04-09T18:07:33.946065Z","shell.execute_reply.started":"2023-04-09T18:07:22.248802Z","shell.execute_reply":"2023-04-09T18:07:33.944685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gpt_tf\nimport importlib\n\nimportlib.reload(gpt_tf)\nfrom gpt_tf import clsr_tsfrm as clsr_tsfrm_tf, GPTConfig\n\ntoy_idxs = np.random.choice(train_idxs,size=100, replace=False)\ntoy_data = data.iloc[toy_idxs]\ndloader_toy = tf_DataSequence(toy_data, sign_to_idx, batch_size=100,drop_last_batch = True)\n\n#params\nparams = dict(dims=2, block_size=50, landm_size=543, vocab_size=250, n_layer=2, n_head=4, n_embd=512, dropout=0.2, bias=True)\n\n# Define the model\ncfg = GPTConfig(**params)\ntf_tfr = clsr_tsfrm_tf(cfg)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:09:30.326185Z","iopub.execute_input":"2023-04-09T18:09:30.327362Z","iopub.status.idle":"2023-04-09T18:09:30.382819Z","shell.execute_reply.started":"2023-04-09T18:09:30.32732Z","shell.execute_reply":"2023-04-09T18:09:30.381521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\n\ndevice = \"GPU:0\" if tf.config.list_physical_devices(\"GPU\") else \"CPU:0\"\n \n\n# Define the optimizer\noptimizer = tf.keras.optimizers.Adam(learning_rate=1e-3,\n                                    beta_1=0.9,\n                                    beta_2=0.999\n                                    )\n\nnum_epochs = 5\neval_every = 1\n\nprint(f\"Using device: {device}\")\n\ncheck_evalSet = True\n\nls_tn_epchs = []\nloss_evl_epchs = []\n\nloss_wei = tf.linspace(1.0, 50.0, 50)\nloss_wei = tf.reshape(loss_wei, (1, 50))\nloss_wei = loss_wei/tf.norm(loss_wei)\n\nfor epoch in range(num_epochs):\n    loss_list = []\n    progress_train = tqdm(\n        iterable=enumerate(dloader_train),\n        desc=f\"Epoch {epoch+1} progress\",\n        total=len(dloader_train),\n        colour=\"green\"\n    )\n    for i, (X, y) in progress_train:\n        with tf.GradientTape() as tape:\n            logits, loss = tf_tfr(X, y, training=True)\n            weighted_loss = loss * loss_wei\n            avg_wei_loss = tf.reduce_mean(weighted_loss, axis=-1) #(B,)\n\n        gradients = tape.gradient(avg_wei_loss, tf_tfr.trainable_variables)\n        optimizer.apply_gradients(zip(gradients, tf_tfr.trainable_variables))\n        loss_it = tf.reduce_mean(avg_wei_loss).numpy()\n        loss_list.append(loss_it) #(B,)->(1)\n        progress_train.set_postfix({\"loss\": f\"{loss_it:.4f}\"}, refresh=True)\n        \n    loss_train = np.mean(loss_list)\n    ls_tn_epchs.append(loss_train)\n\n    if not check_evalSet:\n        print(f\"Training loss: {loss_train: .4f}, Epoch: {epoch + 1}/{num_epochs}\")\n\n    if check_evalSet:\n        loss_eval = 0\n        num_eval_batches = 0\n        for X, y in dloader_val:\n            \n            logits, loss_val = tf_tfr(X, y, training=False)\n            weighted_loss = loss_val * loss_wei\n            avg_wei_loss = tf.reduce_mean(weighted_loss, axis=-1)\n\n            loss_eval += tf.reduce_mean(avg_wei_loss).numpy()\n            num_eval_batches += 1\n\n        loss_eval /= num_eval_batches\n        loss_evl_epchs.append(loss_eval)\n\n        print(f\"Training loss: {loss_train: .4f}, Eval loss: {loss_eval: .4f}, Epoch: {epoch + 1}/{num_epochs}\")\n        ","metadata":{"execution":{"iopub.status.busy":"2023-04-09T18:09:30.386206Z","iopub.execute_input":"2023-04-09T18:09:30.386607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# float32 bits * byte/bit * 1KB/byte * 1MB/1000KB * num_params = approx. Space occupied by the model in (MB)\nmemory_approx = 32 * 1/8 * 1/1024 * 1e-3 * tf_tfr.get_num_params()\nprint(f\"the model weights approximately {memory_approx: .4f} MB.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the complete model (data processing + model + probs output)\n\nclass reshapedLayer(tf.keras.layers.Layer):\n    def __init__(self, block_size=50, **kwargs):\n        super().__init__(**kwargs)\n        self.block_size = block_size\n\n    def call(self, x):\n        shape = tf.shape(x)\n        \n        # t_, lm, c = shape is unpacking the array which is not allowed for symbolic tensors\n        t_ = shape[0]\n        lm = shape[1]\n        c = shape[2]\n\n        pad_length = self.block_size - t_\n        pad_length = tf.maximum(pad_length, 0)\n\n        difference_x = tf.zeros((pad_length, lm, c), dtype=tf.float32)\n        x = tf.concat([difference_x, x], axis=0)[:self.block_size]\n\n        return x\n\nclass DataTransf(tf.keras.Model):\n        \n    def __init__(self):\n        super().__init__()\n        self.reshape_Layer = reshapedLayer()\n\n\n    def call(self, x):\n        x = tf.convert_to_tensor(x)\n\n        x = x[:, :, :-1]\n        # Setting NaNs to 0\n        nan_mask = tf.math.is_nan(x)\n        x = tf.where(nan_mask, tf.zeros_like(x), x)\n\n        x = x - tf.reduce_mean(x, axis=(1, 2), keepdims=True)\n        x /= (tf.math.reduce_std(x, axis=(1, 2), keepdims=True) + 1e-8)\n\n        # if reshape: x = self.reshape_inputs(x)\n        x = self.reshape_Layer(x)\n        \n        return tf.expand_dims(x, axis=0)\n    \n\n\ndef load_relevant_data_subset(pq_path):\n    ROWS_PER_FRAME = 543\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\npq_path = \"/kaggle/input/asl-signs/train_landmark_files/25571/544698.parquet\"\n\n\ndatatfr = DataTransf()\n\nmodel_input = load_relevant_data_subset(pq_path)\n\nout = datatfr(model_input)\nprint(f\"out shape is: {out.shape}\")\n\nlogits, _ = tf_tfr(out)\nprint(f\"out shape is: {logits.shape}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class inference_model(tf.keras.Model):\n    \"\"\"model adapted to the input and output comepetition format.\"\"\"\n    def __init__(self, data_tfr_model, trained_model):\n        super().__init__()\n        self.model = trained_model\n        self.data_tfr_model = data_tfr_model      \n        self.flatten = tf.keras.layers.Flatten()\n    \n    # Build the computational graph for the model call one we call it (model(X))\n    @tf.function(input_signature=[\n      tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name='inputs')\n    ])    \n    def call(self, x):\n        x = self.data_tfr_model(x, training=False)\n        x, _ = self.model(x, training=False)\n        x = tf.nn.softmax(logits=x, axis=-1)  \n        return {'outputs': tf.reshape(x, [-1])}\n\n\ninf_model = inference_model(datatfr, tf_tfr)\n\ninput_model = load_relevant_data_subset(pq_path)\nprint(f\"in shape is: {input_model.shape}\")\n\nout_dic = inf_model(input_model,training=False)\nprint(f\"out shape is: {out_dic['outputs'].shape}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf_infer_model_path =  '/kaggle/working/tf_infer_model'\n# tf.saved_model.save(inf_model,tf_infer_model_path, signatures={'serving_default': inf_model.call})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the TFLiteConverter\nconverter = tf.lite.TFLiteConverter.from_keras_model(inf_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tflite-runtime\n!pip install tensorflow-addons","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the model\ntflite_model = converter.convert()\n\ntflite_path = \"my_model.tflite\"\n# Save the TFLite model to a file\nwith open(tflite_path, 'wb') as f:\n    f.write(tflite_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"found_signatures","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\npq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n#pq_path = \"/kaggle/input/asl-signs/train_landmark_files/36257/1015291697.parquet\"\n\nimport tflite_runtime.interpreter as tflite\ninterpreter = tflite.Interpreter(\"/kaggle/working/my_model.tflite\")\ninterpreter.allocate_tensors()\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\n\nframes = load_relevant_data_subset(pq_path)\nprint(frames.shape)\noutput = prediction_fn(inputs=frames)\n\nsign = np.argmax(output[\"outputs\"])\n\n# output example\nprint(f\"predicted '{idx_to_sign[sign]}' from probs with shape {output['outputs'].shape}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking model Size is < 40mb \n\nsize = os.path.getsize(\"/kaggle/working/my_model.tflite\")\n\nassert size < 40e6, \"model size is bigger than expected (40MB)\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking inference time constrains by running the model on multiple input instances\n\nimport time\n\nexamples = 100 \n\nf1 = os.listdir(\"/kaggle/input/asl-signs/train_landmark_files\")\npart = np.random.choice(f1, size=4)\n\ntimes = []\n\nfor p_ in part:\n    tmp = os.path.join(\"/kaggle/input/asl-signs/train_landmark_files\",p_)\n    dirs =  os.listdir(tmp)\n    rnd_select = np.random.choice(dirs, size=10)\n    \n    for file in rnd_select:\n        pq_path = os.path.join(tmp, file)\n       \n    start_time = time.time()\n    prediction_fn(inputs=load_relevant_data_subset(pq_path))\n\n    elapsed_time = (time.time() - start_time) * 1000  # Convert to milliseconds\n        \n    times += [elapsed_time]\n    \navg_time = sum(times)/len(times)\n\nprint(f\"avg inference time is {avg_time} compared to the required inference time  smaller than < 100ms\")\n    \nassert  avg_time < 100, \"Inference time needs to be less than a 100 ms.\" \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_path","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}