{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Thanks\n\n@mayukh18 for initial data version {https://www.kaggle.com/code/mayukh18/gislr-feature-data}\n\n@dschettler8845 for extended_train.csv and multiple components of my pipeline {https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline}\n\n@roberthatch for wonderful notebook {https://www.kaggle.com/code/roberthatch/gislr-lb-0-63-on-the-shoulders}\n\n@lonnieqin for wonderful notebook https://www.kaggle.com/code/lonnieqin/isolated-sign-language-recognition-with-dnn\n\n\nKindly let me know if I forgot to include any other good work","metadata":{}},{"cell_type":"markdown","source":"# Description\n\n1. Major Modifications in the coding structure\n\n2. Introduced GRU, MSD\n\n3. Didnt use [Mean & Std Features]. You can use it if you want.\n\n\nLook at version 4        LB: 0.64","metadata":{}},{"cell_type":"markdown","source":"# Notes\n\nAfter long time, I am publishing a notebook.\n\nKindly let me know if I made some stupid mistakes\n\nHappy Learning ----------------->\n","metadata":{}},{"cell_type":"code","source":"try:\n    import ipdb\nexcept:\n    !pip install -q --upgrade tensorflow-io\n    !pip install tflite-runtime\n    !pip install ipdb\n    !pip install wandb\n    !pip install hydra-core\n    !mkdir models\n    !cp -r /kaggle/input/gislr-extended-train-dataframe/extended_train.csv ./\n    !cp -r /kaggle/input/asl-signs/train_landmark_files/16069/1004211348.parquet .","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile common_func.py\n\nimport json\nimport os\nimport warnings\n\nfrom sklearn import metrics\n\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\nos.environ[\"TF_CUDNN_DETERMINISTIC\"] = \"1\"\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\"\nimport random\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom tqdm.notebook import tqdm\nfrom wandb.keras import WandbCallback, WandbMetricsLogger\n\nSAVE_DIR = \"./models/\"\n# SAVE_DIR = \"/kaggle/working/models/\"\nif Path(\"/kaggle/input/asl-signs/\").exists():\n    DATA_DIR = \"/kaggle/input/asl-signs/\"\n    ROOT_PATH = \"/kaggle/input/islr-external-data/\"\n    CSV_PATH = \"./\"\nelse:\n    DATA_DIR = \"/scratch/smart_data/islr_data/\"\n    ROOT_PATH = \"/scratch/smart_data/\"\n    CSV_PATH = \"/scratch/smart_data/islr_data/\"\n\nPARQ_PATH = \"/kaggle/input/asl-signs/\"\nNUMPY_PATH = \"/scratch/smart_data/numpy_files/\"\nLANDMARK_FILES_DIR = f\"{ROOT_PATH}train_landmark_files\"\nTRAIN_FILE = f\"{CSV_PATH}extended_new.csv\"\n\nROWS_PER_FRAME =543\n\n# Data Generation ###################################################################################\n\ndef tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis) / tf.reduce_sum(tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis)\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std  = tf_nan_std(x,  axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1]*2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out\n\nclass FeatureGen(tf.keras.layers.Layer):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n    \n    def call(self, x_in):\n#         print(right_hand_percentage(x))\n#         x_list = [tf.expand_dims(tf_nan_mean(x_in[:, av_set[0]:av_set[0]+av_set[1], :], axis=1), axis=1) for av_set in averaging_sets]\n#         x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n#         x = tf.concat(x_list, 1)\n        x = tf.gather(x_in, point_landmarks, axis=1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0) , 1, 0)\n            p1 = tf.where( ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0) , 1, 0)\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n        \n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)), [NUM_FRAMES, LANDMARKS])\n        x = tf.reshape(x, (1, INPUT_SHAPE[0]*INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x\n\ndef convert_row(row, right_handed=True):\n    x = load_relevant_data_subset(os.path.join(\"/kaggle/input/asl-signs\", row[1].path))\n    x = feature_converter(tf.convert_to_tensor(x)).cpu().numpy()\n    return x, row[1].label\n\ndef convert_and_save_data():\n    df = pd.read_csv(TRAIN_FILE)\n    df['label'] = df['sign'].map(label_map)\n    total = df.shape[0]\n    if QUICK_TEST:\n        total = QUICK_LIMIT\n    npdata = np.zeros((total, INPUT_SHAPE[0]*INPUT_SHAPE[1] + (SEGMENTS+1)*INPUT_SHAPE[1]*2))\n    nplabels = np.zeros(total)\n    for i, row in tqdm(enumerate(df.iterrows()), total=total):\n        (x,y) = convert_row(row)\n        npdata[i,:] = x\n        nplabels[i] = y\n        if QUICK_TEST and i == QUICK_LIMIT - 1:\n            break\n    \n    np.save(\"feature_data.npy\", npdata)\n    np.save(\"feature_labels.npy\", nplabels)\n    \n\ndef right_hand_percentage(x):\n    right = tf.gather(x, right_hand_landmarks, axis=1)\n    left = tf.gather(x, left_hand_landmarks, axis=1)\n    right_count = tf.reduce_sum(tf.where(tf.math.is_nan(right), tf.zeros_like(right), tf.ones_like(right)))\n    left_count = tf.reduce_sum(tf.where(tf.math.is_nan(left), tf.zeros_like(left), tf.ones_like(left)))\n    return right_count / (left_count+right_count)\n\n\n# Data Functions ###################################################################################\n\n\ndef prepare_main_csv(seed, num_splits, csv_path=CSV_PATH):\n    if Path(f\"{csv_path}/extended_new.csv\").exists():\n        data_csv = pd.read_csv(f\"{csv_path}/extended_new.csv\").reset_index(drop=True)\n    else:\n        data_csv = pd.read_csv(f\"{csv_path}/extended_train.csv\").reset_index(drop=True)\n\n        json_data = read_json_file()\n        json_df = pd.DataFrame.from_dict(json_data, orient=\"index\")\n        json_df = json_df.reset_index()\n        json_df.rename(columns={\"index\": \"sign\", 0: \"sign_val\"}, inplace=True)\n\n        data_csv = pd.merge(data_csv, json_df, on=\"sign\")\n        data_csv = data_csv.sample(frac=1.0, random_state=seed).reset_index(drop=True)\n        data_csv[\"hand\"] = data_csv[\"participant_id\"]\n        data_csv = data_csv.replace({\"hand\": di})\n        data_csv[\"fold_split\"] = (\n            data_csv[\"hand\"].astype(\"str\") + \"_\" + data_csv[\"sign_val\"].astype(\"str\")\n        )\n\n        # Splitting the data based on (hand & sign), grouped by participant_id\n        skf = StratifiedGroupKFold(n_splits=num_splits)\n        data_csv[\"fold\"] = -1\n        print(\"Splitting Data ------------------------------------>\")\n        for i, (train_index, test_index) in enumerate(\n            skf.split(data_csv.index, data_csv.fold_split, data_csv.participant_id)\n        ):\n            data_csv.loc[test_index, \"fold\"] = i\n            print(f\"fold {i} --> {len(test_index)}\")\n            print(data_csv.loc[test_index].participant_id.value_counts())\n        \n        data_csv.to_csv(f\"{csv_path}/extended_new.csv\", index=False)\n\n    return data_csv\n\n\ndef get_data(fold_num, cfg):\n    # Data Loading\n    print(\"Data Loading ----->\")\n    # train_x_full = np.load(f\"{ROOT_PATH}23_nonorm_feature_data.npy\").astype(np.float32)\n    # train_y_full = np.load(f\"{ROOT_PATH}23_nonorm_feature_labels.npy\").astype(np.uint8)\n    train_x_full = np.load(f\"{ROOT_PATH}feature_data.npy\").astype(np.float32)\n    train_y_full = np.load(f\"{ROOT_PATH}feature_labels.npy\").astype(np.uint8)\n\n    print(train_x_full.shape, train_y_full.shape)\n\n    if cfg['FLAG_DROP_Z']:\n        train_x_full = np.reshape(train_x_full, [train_x_full.shape[0], -1, 3])\n        train_x_full = train_x_full[:, :, 0:2]\n        train_x_full = np.reshape(train_x_full, [train_x_full.shape[0], -1])\n        print(train_x_full.shape, train_y_full.shape)\n\n    # Remove it with stratifiedkfold\n    train_df = prepare_main_csv(cfg['SEED'], cfg['NUM_SPLITS'])\n    train_idxs = train_df.index[train_df.fold != fold_num].to_numpy()\n    val_idxs = train_df.index[train_df.fold == fold_num].to_numpy()\n\n    train_x, train_y = train_x_full[train_idxs], train_y_full[train_idxs]\n    val_x, val_y = train_x_full[val_idxs], train_y_full[val_idxs]\n\n    del train_x_full, train_y_full, train_idxs, val_idxs\n\n    print(train_df[train_df.sequence_id == 1004211348])\n    return train_df, train_x, train_y, val_x, val_y\n\n\n# Utils ###################################################################################\n\ndef get_input_shape(num_frames, landmarks, flag_drop_z):\n    input_shape = (num_frames, landmarks * 3)\n\n    if flag_drop_z:\n        num_coords = 2\n    else:\n        num_coords = 3\n\n    return (num_frames, landmarks * num_coords)\n\n\ndef seed_it_all(seed=42):\n    \"\"\"Attempt to be Reproducible\"\"\"\n    tf.keras.backend.clear_session()\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    tf.keras.utils.set_random_seed(seed)\n    tf.config.experimental.enable_op_determinism()\n\n\ndef read_json_file(file_path=f\"{DATA_DIR}/sign_to_prediction_index_map.json\"):\n    with open(file_path, \"r\") as file:\n        json_data = json.load(file)\n    return json_data\n\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = [\"x\", \"y\", \"z\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\n\ndef load_npz(f):\n    data = np.load(f)\n    return data[\"data\"]\n\n\ndef uniform_soup():\n    soups = []\n    ## Instantiating model\n\n    tf.keras.backend.clear_session()\n    model = get_model()\n    model_paths = Path(SAVE_DIR).glob(\"*.h5\")\n\n    ## Iterating Over all models\n    for path in tqdm(model_paths):\n        ## loading model wieghts\n        print(f\"### Loading {path}\")\n        model.load_weights(str(path))\n\n        ## Adding model weights in soup list\n        soup = [np.array(weights) for weights in model.weights]\n        soups.append(soup)\n\n    ## Averaing all weights\n    mean_soup = np.array(soups).mean(axis=0)\n\n    ## Replacing model's weight with Unifrom Soup Weights\n    for w1, w2 in zip(model.weights, mean_soup):\n        tf.keras.backend.set_value(w1, w2)\n\n    model.save_weights(f\"{SAVE_DIR}/uniform_soup.h5\")\n\n\nclass TFLiteModel(tf.Module):\n    \"\"\"\n    TensorFlow Lite model that takes input tensors and applies:\n        – a preprocessing model\n        – the ASL model\n    \"\"\"\n\n    def __init__(self, asl_model):\n        \"\"\"\n        Initializes the TFLiteModel with the specified feature generation model and main model.\n        \"\"\"\n        super(TFLiteModel, self).__init__()\n\n        # Load the feature generation and main models\n        self.prep_inputs = FeatureGen()\n        self.asl_model = asl_model\n\n    @tf.function(\n        input_signature=[\n            tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name=\"inputs\")\n        ]\n    )\n    def __call__(self, inputs):\n        \"\"\"\n        Applies the feature generation model and main model to the input tensors.\n\n        Args:\n            inputs: Input tensor with shape [batch_size, 543, 3].\n\n        Returns:\n            A dictionary with a single key 'outputs' and corresponding output tensor.\n        \"\"\"\n        x = self.prep_inputs(tf.cast(inputs, dtype=tf.float32))\n        outputs = self.asl_model(x)[0, :]\n\n        # Return a dictionary with the output tensor\n        return {\"outputs\": outputs}\n\n\n# Metrics ###################################################################################\n\nlabel_ls = list(range(0, 250))\nAVG_TYPE = \"weighted\"\n\n\ndef get_metrics(labels, preds_class):\n    metric_dict = {\n        \"accuracy\": metrics.accuracy_score(labels, preds_class),\n        \"f1_score\": metrics.f1_score(\n            labels, preds_class, labels=label_ls, zero_division=0, average=AVG_TYPE\n        ),\n        \"precision\": metrics.precision_score(\n            labels, preds_class, labels=label_ls, zero_division=0, average=AVG_TYPE\n        ),\n        \"recall\": metrics.recall_score(\n            labels, preds_class, labels=label_ls, zero_division=0, average=AVG_TYPE\n        ),\n        # 'roc': metrics.roc_auc_score(labels, preds_softmax, average='macro', multi_class='ovo', labels=label_ls),\n    }\n    for k, v in metric_dict.items():\n        print(k, v)\n\n\ndef compute_evaluation_metrics(model, data_x, data_y, decoder):\n    \"\"\"\n    Computes the evaluation metrics for the given model on the given data and prints classwise confusion matrix.\n\n    Args:\n    - model: The trained model to evaluate.\n    - data_x: The input data to evaluate the model on.\n    - data_y: The target data to evaluate the model on.\n    - decoder: A function to decode the model's output into readable text.\n    \"\"\"\n    # Compute the predicted classes and confusion matrix\n    batch_size = 1024\n    y_pred = model.predict(data_x, batch_size=1024)\n    print(y_pred.shape)\n    y_pred_classes = tf.cast(np.argmax(y_pred, axis=1), tf.uint8)\n    confusion_mtx = tf.math.confusion_matrix(data_y, y_pred_classes)\n\n    # Compute the evaluation metrics by class\n    num_classes = confusion_mtx.shape[0]\n    classwise_performance = {}\n    for i in range(num_classes):\n        tp = confusion_mtx[i, i]\n        fp = tf.reduce_sum(confusion_mtx[:, i]) - tp\n        fn = tf.reduce_sum(confusion_mtx[i, :]) - tp\n        tn = tf.reduce_sum(confusion_mtx[i]) - (tp - fp - fn)\n\n        classwise_performance[i] = dict(\n            accuracy=(tp + tn) / (tp + fp + tn + fn),\n            precision=tp / (tp + fp),\n            recall=tp / (tp + fn),\n        )\n        classwise_performance[i][\"f1_score\"] = (\n            2\n            * (\n                classwise_performance[i][\"precision\"]\n                * classwise_performance[i][\"recall\"]\n            )\n            / (\n                classwise_performance[i][\"precision\"]\n                + classwise_performance[i][\"recall\"]\n            )\n        )\n\n    # Sort the classwise performance by f1_score and print the results\n    classwise_performance = dict(\n        sorted(\n            classwise_performance.items(), key=lambda x: x[1][\"f1_score\"], reverse=True\n        )\n    )\n    print(\"\\n\\n... CLASSWISE CONFUSION MATRIX... \\n\")\n    for i, perf in classwise_performance.items():\n        print(\n            f\"Class {i:<3}  ({decoder[i]:^13})  -->  Accuracy: {perf['accuracy']:.2f}, Precision: {perf['precision']:.2f}, Recall: {perf['recall']:.2f}, F1 Score: {perf['f1_score']:.2f}\"\n        )\n\n\n# Model Utils ################################################################################\n\noutput_bias = tf.keras.initializers.Constant(1.0 / 250.0)\n\n\nclass MSD(tf.keras.layers.Layer):\n    def __init__(\n        self,\n        units,\n        fold_num,\n        cfg,\n        **kwargs,\n    ):\n        super().__init__(**kwargs)\n\n        self.lin = tf.keras.layers.Dense(\n            units,\n            activation=None,\n            use_bias=True,\n            bias_initializer=output_bias,\n            # kernel_regularizer=R.l2(WEIGHT_REGULARIZE)\n        )\n\n        rate_dropout = cfg[\"MSD_DROPOUT\"]\n        if cfg[\"MSD_DROP_TYPE\"] == \"normal\":\n            self.dropouts = [\n                tf.keras.layers.Dropout((rate_dropout - 0.2), seed=135 + fold_num),\n                tf.keras.layers.Dropout((rate_dropout - 0.1), seed=690 + fold_num),\n                tf.keras.layers.Dropout((rate_dropout), seed=275 + fold_num),\n                tf.keras.layers.Dropout((rate_dropout + 0.1), seed=348 + fold_num),\n                tf.keras.layers.Dropout((rate_dropout + 0.2), seed=861 + fold_num),\n            ]\n\n        elif cfg[\"MSD_DROP_TYPE\"] == \"gaussian\":\n            self.dropouts = [\n                tf.keras.layers.GaussianDropout((rate_dropout - 0.2)),\n                tf.keras.layers.GaussianDropout((rate_dropout - 0.1)),\n                tf.keras.layers.GaussianDropout(rate_dropout),\n                tf.keras.layers.GaussianDropout((rate_dropout + 0.1)),\n                tf.keras.layers.GaussianDropout((rate_dropout + 0.2)),\n            ]\n\n    def call(self, inputs):\n        for ii, drop in enumerate(self.dropouts):\n            if ii == 0:\n                out = self.lin(drop(inputs)) / 5.0\n            else:\n                out += self.lin(drop(inputs)) / 5.0\n        return out\n\n\nclass ResidualBlock(tf.keras.layers.Layer):\n    def __init__(self, units, dropout):\n        super().__init__()\n        self.linear = tf.keras.layers.Dense(units)\n        self.bn = tf.keras.layers.BatchNormalization()\n        self.act = tf.keras.layers.Activation(\"gelu\")\n        if dropout != 0:\n            self.drop = tf.keras.layers.Dropout(dropout)\n            self.flag_use_drop = True\n        else:\n            self.flag_use_drop = False\n\n    def call(self, x):\n        x = self.linear(x)\n        x = self.bn(x)\n        x = self.act(x)\n        if self.flag_use_drop:\n            x = self.drop(x)\n        return x\n\n\nclass GRUModel(tf.keras.layers.Layer):\n    def __init__(self, units, dropout, num_blocks):\n        super().__init__()\n        self.start_gru = tf.keras.layers.GRU(\n            units=units, dropout=0.0, return_sequences=True\n        )\n        self.end_gru = tf.keras.layers.GRU(\n            units=units, dropout=dropout, return_sequences=False\n        )\n\n        if (num_blocks - 2) > 0:\n            self.gru_blocks = [\n                tf.keras.layers.GRU(units=units, dropout=dropout, return_sequences=True)\n                * (num_blocks - 2)\n            ]\n            self.flag_use_gru_blocks = True\n        else:\n            self.flag_use_gru_blocks = False\n\n    def call(self, x):\n        x = self.start_gru(x)\n        if self.flag_use_gru_blocks:\n            for blk in self.gru_blocks:\n                x = blk(x)\n        x = self.end_gru(x)\n        return x\n\n\ndef model_utils(cfg, fold_num):\n    metric_ls = [\n        tf.keras.metrics.SparseCategoricalAccuracy(),\n        tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5),\n    ]\n\n    cb_list = [\n        tf.keras.callbacks.EarlyStopping(\n            patience=5,\n            restore_best_weights=True,\n            verbose=1,\n            monitor=cfg[\"TARGET_METRIC\"],\n        ),\n        tf.keras.callbacks.ReduceLROnPlateau(patience=2, factor=0.8, verbose=1),\n        tf.keras.callbacks.ModelCheckpoint(\n            f\"{SAVE_DIR}/best_acc_{fold_num}.h5\",\n            monitor=cfg[\"TARGET_METRIC\"],\n            verbose=0,\n            save_best_only=True,\n            save_weights_only=True,\n            mode=\"max\",\n            save_freq=\"epoch\",\n        ),\n    ]\n\n    if cfg[\"FLAG_WANDB\"]:\n        cb_list += [#WandbMetricsLogger()\n            WandbCallback(\n                monitor=cfg[\"TARGET_METRIC\"],\n                log_weights=False,\n                log_evaluation=False,\n                save_model=False,\n            )\n        ]\n\n    opt = tfa.optimizers.AdamW(weight_decay=0, learning_rate=cfg[\"LR\"])\n    # opt = tf.keras.optimizers.Adam(learning_rate=LR)\n    # opt = tfa.optimizers.RectifiedAdam(learning_rate=LR)\n    # opt = tfa.optimizers.Lookahead(opt, sync_period=5)\n\n    return metric_ls, cb_list, opt\n\n\n######################################################################################################################\n\nlip_landmarks = [\n    61,\n    185,\n    40,\n    39,\n    37,\n    0,\n    267,\n    269,\n    270,\n    409,\n    291,\n    146,\n    91,\n    181,\n    84,\n    17,\n    314,\n    405,\n    321,\n    375,\n    78,\n    191,\n    80,\n    81,\n    82,\n    13,\n    312,\n    311,\n    310,\n    415,\n    95,\n    88,\n    178,\n    87,\n    14,\n    317,\n    402,\n    318,\n    324,\n    308,\n]\n\n# Analyzing Handedness\nleft_handed_signer = [\n    16069,\n    32319,\n    36257,\n    22343,\n    27610,\n    61333,\n    34503,\n    55372,\n    37055,\n]  # both_hands_signer-> 37055\nright_handed_signer = [\n    26734,\n    28656,\n    25571,\n    62590,\n    29302,\n    49445,\n    53618,\n    18796,\n    4718,\n    2044,\n    37779,\n    30680,\n]\nlip_landmarks = [\n    61,\n    185,\n    40,\n    39,\n    37,\n    0,\n    267,\n    269,\n    270,\n    409,\n    291,\n    146,\n    91,\n    181,\n    84,\n    17,\n    314,\n    405,\n    321,\n    375,\n    78,\n    191,\n    80,\n    81,\n    82,\n    13,\n    312,\n    311,\n    310,\n    415,\n    95,\n    88,\n    178,\n    87,\n    14,\n    317,\n    402,\n    318,\n    324,\n    308,\n]\n\ndi = {}\nfor k in left_handed_signer:\n    di[k] = 0\nfor k in right_handed_signer:\n    di[k] = 1\n\nleft_hand_landmarks = list(range(468, 468 + 21))\nright_hand_landmarks = list(range(522, 522 + 21))\n\naveraging_sets = [\n    [0, 468],\n    [489, 33],\n]  ## average over the entire face, and the entire 'pose'\n\npoint_landmarks = [\n    item\n    for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks]\n    for item in sublist\n]\n\nLANDMARKS = len(point_landmarks) #+ len(averaging_sets)\n\n# Fixed  ##################################################################################\n\nFLAG_DROP_Z = False\nROWS_PER_FRAME = 543\nNUM_FRAMES = 15\nINPUT_SHAPE = get_input_shape(NUM_FRAMES, LANDMARKS, FLAG_DROP_Z)\nSEGMENTS = 3\nNUM_BASE_FEATS = (SEGMENTS + 1) * INPUT_SHAPE[1] * 2\nFLAT_FRAME_SHAPE = NUM_BASE_FEATS + (INPUT_SHAPE[0] * INPUT_SHAPE[1])\ndecoder = {v: k for k, v in read_json_file().items()}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile trainer.py\n\nimport gc\nimport os\nimport pprint\nimport warnings\nimport wandb\nimport hydra\nfrom omegaconf import DictConfig\nfrom zipfile import ZipFile\ntry:\n    import tflite_runtime.interpreter as tflite\n    FLAG_INTERPRET = True\nexcept:\n    FLAG_INTERPRET = False\n    print(\"TFlite Interpretation not possible\")\nwarnings.filterwarnings(\"ignore\")\nos.environ[\"TF_DETERMINISTIC_OPS\"] = \"1\"\nos.environ[\"TF_CUDNN_DETERMINISTIC\"] = \"1\"\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\"\nimport time\nimport numpy as np\nimport tensorflow as tf\nfrom common_func import *\n\nseed_it_all()\nstart_time = time.time()\n\n# Flags  ##################################################################################\nif False:\n    mixed_precision.set_global_policy(\"mixed_float16\")\n    tf.config.optimizer.set_jit(True)\n\n# Model  ###################################################################################\n\n\ndef get_model(\n    cfg,\n    fold_num=0,\n    n_labels=250,\n    flat_frame_len=FLAT_FRAME_SHAPE,\n    flag_model_summary=False,\n    flag_with_cb_list=False,\n):\n    print(\"Model Loading ----->\")\n    _inputs = tf.keras.layers.Input(shape=(flat_frame_len,))\n\n    # import ipdb\n    # ipdb.set_trace()\n    x = _inputs[:, :NUM_BASE_FEATS]\n    x_conv = tf.reshape(_inputs[:, NUM_BASE_FEATS:], (-1, NUM_FRAMES, INPUT_SHAPE[1]))\n\n    # Concat Dilated Convolutions with actual data\n    gru_out = GRUModel(\n        cfg[\"NUM_GRU_UNITS\"], cfg[\"RATE_GRU_DROPOUT\"], cfg[\"NUM_GRU_BLOCKS\"]\n    )(x_conv)\n    \n    if cfg['FLAG_CONCAT_FEATS']:\n        x = tf.keras.layers.concatenate([gru_out, x], axis=1)\n    else:\n        x = gru_out\n    print(\"Concatenate Shape\", x.shape)\n\n    # Residual Block\n    x = ResidualBlock(cfg[\"NUM_RESIDUAL_UNITS\"], 0.25)(x)\n    x += ResidualBlock(cfg[\"NUM_RESIDUAL_UNITS\"], 0.0)(x)\n\n    # Final output MSD Layer\n    x = MSD(units=n_labels, fold_num=fold_num, cfg=cfg)(x)\n    _outputs = tf.keras.layers.Softmax(dtype=\"float32\")(x)\n\n    # Build the model\n    model = tf.keras.models.Model(inputs=_inputs, outputs=_outputs)\n    metric_ls, cb_list, opt = model_utils(cfg, fold_num)\n    model.compile(opt, \"sparse_categorical_crossentropy\", metrics=metric_ls)\n\n    if flag_model_summary:\n        model.summary()\n\n    if flag_with_cb_list:\n        return model, cb_list\n    else:\n        return model\n\n\ndef tflite_conversion(model):    \n    # TFLite Conversion\n    tflite_keras_model = TFLiteModel(model)\n    demo_output = tflite_keras_model(load_relevant_data_subset('1004211348.parquet'))[\"outputs\"]\n    decoder[np.argmax(demo_output.numpy(), axis=-1)]\n\n    keras_model_converter = tf.lite.TFLiteConverter.from_keras_model(tflite_keras_model)\n    tflite_model = keras_model_converter.convert()\n    \n    tf_lite_model_path = f'{SAVE_DIR}/model.tflite'\n    with open(tf_lite_model_path, 'wb') as f:\n        f.write(tflite_model)\n        \n    ZipFile('submission.zip', mode='w').write(tf_lite_model_path)\n\n    if FLAG_INTERPRET:\n        interpreter = tflite.Interpreter(tf_lite_model_path)\n        found_signatures = list(interpreter.get_signature_list().keys())\n        prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n        output = prediction_fn(inputs=load_relevant_data_subset('1004211348.parquet'))\n        sign = np.argmax(output[\"outputs\"])\n\n        print(\"PRED : \", decoder[sign])\n        # print(\"GT   : \", train_df.sign[0])\n\n    \n@hydra.main(version_base=None, config_path=\"./\", config_name=\"config\")\ndef my_app(cfg: DictConfig):\n    print(\"*\" * 75)\n    config = dict(cfg[\"CFG\"])\n    seed_it_all(config[\"SEED\"])\n    config[\"FLAG_DROP_Z\"] = FLAG_DROP_Z\n    if config[\"FLAG_DEBUG\"]:\n        config[\"NUM_EPOCHS\"] = 3\n        config[\"FLAG_WANDB\"] = config[\"FLAG_WANDB\"] and False\n    else:\n        config[\"FLAG_WANDB\"] = config[\"FLAG_WANDB\"] and True\n\n    pprint.pprint(config)    \n    true = np.array([])\n    oof = np.array([])\n    for fold_cnt, fold_num in enumerate(\n        range(config[\"FOLD_START\"], config[\"FOLD_END\"]+1)\n    ):\n        print(\"#\" * 25)\n        print(f\"### Fold {fold_num}\")\n        if config[\"FLAG_WANDB\"]:\n            wandb.init(project=\"isle_analysis\", group=config[\"DESCRIPTION\"])\n\n        seed_it_all(config[\"SEED\"] + fold_num)\n        train_df, train_x, train_y, val_x, val_y = get_data(\n            fold_num, config\n        )\n\n        model, cb_list = get_model(\n            config, fold_num, flag_with_cb_list=True, flag_model_summary=(fold_cnt == 0)\n        )\n\n        history = model.fit(\n            train_x,\n            train_y,\n            validation_data=(val_x, val_y),\n            verbose=2,\n            epochs=config[\"NUM_EPOCHS\"],\n            callbacks=cb_list,\n            batch_size=config[\"BATCH_SIZE\"],\n            workers=8,\n        )\n\n        oof_p = model.predict(val_x, batch_size=config[\"BATCH_SIZE\"], verbose=2)\n        oof_p = np.argmax(oof_p, axis=1)\n        true = np.concatenate([true, val_y])\n        oof = np.concatenate([oof, oof_p])\n\n        print(\"#\" * 25)\n        print(f\"### Evaluation Metrics\")\n        model.evaluate(val_x, val_y)\n\n        if fold_num == config[\"FOLD_END\"]:\n            compute_evaluation_metrics(model, val_x, val_y, decoder=decoder)\n            \n        del train_df, train_x, train_y, val_x, val_y\n        gc.collect()\n\n        if config[\"FLAG_WANDB\"]:\n            wandb.finish()\n            \n    # PRINT OVERALL RESULTS\n    print(\"#\" * 25)\n    print(f\"Overall Metrics\")\n    get_metrics(true, oof)\n    tf.keras.backend.clear_session()\n\n    if config[\"FLAG_GEN_TFLITE\"]:\n        tflite_conversion(model)\n        \n    \nif __name__ == \"__main__\":\n    cfg = my_app()\n    print(\"Total Time: \", time.time() - start_time)\n\n    # Dilated Convolutions\n    # conv_1 = tf.keras.layers.Conv1D(5, 1, strides=1, activation='silu')(x_conv)\n    # conv_3 = tf.keras.layers.Conv1D(5, 1, strides=3, activation='silu')(x_conv)\n    # conv_5 = tf.keras.layers.Conv1D(5, 1, strides=5, activation='silu')(x_conv)\n    # conv_15 = tf.keras.layers.Conv1D(5, 1, strides=15, activation='silu')(x_conv)\n    # conv_out = tf.keras.layers.concatenate([conv_1, conv_3, conv_5, conv_15], axis=1)\n    # conv_out = tf.reshape(conv_out, (-1, conv_out.shape[1] * conv_out.shape[2]))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile config.yaml\nCFG:\n    # General Params  ##########################################################################\n\n    DESCRIPTION: initial trials\n    LR: 6e-4\n    BATCH_SIZE: 512 # 512\n    NUM_EPOCHS: 100\n    NUM_SPLITS: 7\n    TARGET_METRIC: \"val_sparse_categorical_accuracy\"\n\n    FOLD_START: 1\n    FOLD_END: 1\n    \n    SEED: 42\n    \n    # Flags  ##################################################################################\n    \n    FLAG_DROP_Z: False\n    FLAG_GEN_TFLITE: True\n#     FLAG_GEN_TFLITE: False\n    \n    FLAG_CONCAT_FEATS: False # Use mean and standard deviation information captured in the dataset\n\n\n    FLAG_DEBUG: False\n#     FLAG_WANDB: True\n    FLAG_WANDB: False\n    # FLAG_DEBUG: True\n\n    # Model  ##################################################################################\n\n    # NUM_RESIDUAL_UNITS: 128\n    NUM_RESIDUAL_UNITS: 1024\n\n    # NUM_GRU_UNITS: 128\n    NUM_GRU_UNITS: 512\n    RATE_GRU_DROPOUT: 0.5\n    NUM_GRU_BLOCKS: 1 # 2 minimal value\n\n    MSD_DROP_TYPE: \"normal\"\n    MSD_DROPOUT: 0.5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 trainer.py","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}