{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ASL Sign Detection - PyTorch Lightning\n\n## Leave an upvote if you find it useful or fork it :) And also for the notebooks mentioned in the Sources section\n\n## Versions:\n- 11: Fix `reset_index` bug when setting up `train_df` and `val_df` in `ASLDataModule` in combination with `__getitem()__`, which caused unrealistic high val accuracies:\n```python\n    def setup(self, stage=None):\n        val_fold = self.hparams.val_fold\n        \n        train_df = self.df[self.df.fold != val_fold].reset_index(drop=True) # Removed .reset_index in version 11\n        val_df = self.df[self.df.fold == val_fold].reset_index(drop=True) # Removed .reset_index in version 11\n```\n\n```python\n    def __getitem__(self, idx):\n        # Use df_index as idx due to folds splitting\n        df_index = self.df.index.values[idx]\n\n        x = self.X[df_index]\n        y = self.y[df_index]\n```       \n- 14: Use lip instead of face features\n- 15: \n    - Use [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders) features\n    - Use TF Preprocessing from [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)\n\n## TODO: \n- [ ] Implement FeatureGen in PyTorch\n- [ ] Use ArcMarginProduct\n- [ ] Use 2D instead of 3D points\n- [x] Switch to using time-based features instead of only mean and std\n- [ ] Add mirror data augmentation\n\n## Sources:\n- ### [Mayukh Bhattacharyya](https://www.kaggle.com/mayukh18)'s [End-to-End Pytorch Training + Submission](https://www.kaggle.com/code/mayukh18/end-to-end-pytorch-training-submission)\n    - Great feature generation using PyTorch model\n    - Super nice PyTorch to TFLite Conversion\n- ### [Darien Schettler](https://www.kaggle.com/dschettler8845)'s [🤟 GISLR 🤟 - 📚Learn – 🔭EDA – 🤖Baseline](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline)\n    - Awesome EDA to understand training data\n    - Nice and simple baseline model\n- ### [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)\n    - Feature creation\n    - TF Preprocessing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Installs","metadata":{}},{"cell_type":"code","source":"!pip install onnx-tf tflite-runtime nb_black -q\n!pip install --upgrade -q wandb","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:57:27.894658Z","iopub.execute_input":"2023-03-07T08:57:27.895033Z","iopub.status.idle":"2023-03-07T08:57:51.118582Z","shell.execute_reply.started":"2023-03-07T08:57:27.895001Z","shell.execute_reply":"2023-03-07T08:57:51.117036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext lab_black","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:57:51.121074Z","iopub.execute_input":"2023-03-07T08:57:51.121464Z","iopub.status.idle":"2023-03-07T08:57:51.391695Z","shell.execute_reply.started":"2023-03-07T08:57:51.121425Z","shell.execute_reply":"2023-03-07T08:57:51.390661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import json\nimport math\nfrom pathlib import Path\n\nimport numpy as np\nimport onnx\nimport pandas as pd\nimport pytorch_lightning as pl\nimport tensorflow as tf\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchmetrics\nimport wandb\nfrom kaggle_secrets import UserSecretsClient\nfrom onnx_tf.backend import prepare\nfrom pytorch_lightning.callbacks import EarlyStopping\nfrom pytorch_lightning.callbacks import LearningRateMonitor\nfrom pytorch_lightning.loggers import WandbLogger\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom timm.optim import create_optimizer_v2\nfrom torchmetrics import MetricCollection","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:57:51.392954Z","iopub.execute_input":"2023-03-07T08:57:51.393302Z","iopub.status.idle":"2023-03-07T08:58:05.230157Z","shell.execute_reply.started":"2023-03-07T08:57:51.393259Z","shell.execute_reply":"2023-03-07T08:58:05.229098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WandB","metadata":{}},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_api\")\nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:05.233174Z","iopub.execute_input":"2023-03-07T08:58:05.233781Z","iopub.status.idle":"2023-03-07T08:58:07.940720Z","shell.execute_reply.started":"2023-03-07T08:58:05.233738Z","shell.execute_reply":"2023-03-07T08:58:07.939515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paths & Settings","metadata":{}},{"cell_type":"code","source":"KAGGLE_DIR = Path(\"/\") / \"kaggle\"\nINPUT_DIR = KAGGLE_DIR / \"input\"\nDATA_ROOT_DIR = INPUT_DIR / \"asl-signs\"\nNPY_DATA_DIR = INPUT_DIR / \"gislr-feature-data-on-the-shoulders\"\nMY_NPY_DATA_DIR = INPUT_DIR / \"asl-signs-features-npy\"\n\nTRAIN_CSV_PATH = MY_NPY_DATA_DIR / \"train_prepared.csv\"\nSIGN_TO_IDX_PATH = INPUT_DIR / \"asl-signs\" / \"sign_to_prediction_index_map.json\"\n\nwith open(SIGN_TO_IDX_PATH, \"r\") as f:\n    SIGN_TO_IDX = json.load(f)\n\nN_SPLITS = 5\nSEED = 2023\nROWS_PER_FRAME = 543\n\nIN_FEATURES = 5796","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:07.942660Z","iopub.execute_input":"2023-03-07T08:58:07.943322Z","iopub.status.idle":"2023-03-07T08:58:07.957427Z","shell.execute_reply.started":"2023-03-07T08:58:07.943280Z","shell.execute_reply":"2023-03-07T08:58:07.956486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(TRAIN_CSV_PATH)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:07.959122Z","iopub.execute_input":"2023-03-07T08:58:07.959521Z","iopub.status.idle":"2023-03-07T08:58:08.523767Z","shell.execute_reply.started":"2023-03-07T08:58:07.959485Z","shell.execute_reply":"2023-03-07T08:58:08.522562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TF Feature Generator\n\n## Source: [GISLR Feature Data: On the Shoulders](https://www.kaggle.com/code/roberthatch/gislr-feature-data-on-the-shoulders)","metadata":{}},{"cell_type":"code","source":"DROP_Z = False\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET + 21\nRIGHT_HAND_OFFSET = POSE_OFFSET + 33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [\n    61,\n    185,\n    40,\n    39,\n    37,\n    0,\n    267,\n    269,\n    270,\n    409,\n    291,\n    146,\n    91,\n    181,\n    84,\n    17,\n    314,\n    405,\n    321,\n    375,\n    78,\n    191,\n    80,\n    81,\n    82,\n    13,\n    312,\n    311,\n    310,\n    415,\n    95,\n    88,\n    178,\n    87,\n    14,\n    317,\n    402,\n    318,\n    324,\n    308,\n]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET + 21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET + 21))\n\npoint_landmarks = [\n    item\n    for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks]\n    for item in sublist\n]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nif DROP_Z:\n    INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 2)\nelse:\n    INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 3)\n\nFLAT_INPUT_SHAPE = (INPUT_SHAPE[0] + 2 * (SEGMENTS + 1)) * INPUT_SHAPE[1]","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:08.561728Z","iopub.execute_input":"2023-03-07T08:58:08.562094Z","iopub.status.idle":"2023-03-07T08:58:08.584002Z","shell.execute_reply.started":"2023-03-07T08:58:08.562049Z","shell.execute_reply":"2023-03-07T08:58:08.582809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(\n        tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis\n    ) / tf.reduce_sum(\n        tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis\n    )\n\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std = tf_nan_std(x, axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1] * 2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:08.585827Z","iopub.execute_input":"2023-03-07T08:58:08.586196Z","iopub.status.idle":"2023-03-07T08:58:08.603698Z","shell.execute_reply.started":"2023-03-07T08:58:08.586159Z","shell.execute_reply":"2023-03-07T08:58:08.602698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGenTF(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n\n    def call(self, x_in):\n        if DROP_Z:\n            x_in = x_in[:, :, 0:2]\n        x_list = [\n            tf.expand_dims(\n                tf_nan_mean(x_in[:, av_set[0] : av_set[0] + av_set[1], :], axis=1),\n                axis=1,\n            )\n            for av_set in averaging_sets\n        ]\n        x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n        x = tf.concat(x_list, 1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where(\n                ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0), 1, 0\n            )\n            p1 = tf.where(\n                ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0), 1, 0\n            )\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n\n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(\n            tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)),\n            [NUM_FRAMES, LANDMARKS],\n        )\n        x = tf.reshape(x, (1, INPUT_SHAPE[0] * INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:08.608790Z","iopub.execute_input":"2023-03-07T08:58:08.609065Z","iopub.status.idle":"2023-03-07T08:58:08.634133Z","shell.execute_reply.started":"2023-03-07T08:58:08.609040Z","shell.execute_reply":"2023-03-07T08:58:08.632970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class ASLDataFrameDataset(torch.utils.data.Dataset):\n    def __init__(self, df, in_features, transform=None):\n        self.df = df\n        self.transform = transform\n\n        print(\"Loading data...\")\n        self.X = np.load(NPY_DATA_DIR / \"feature_data.npy\")\n        self.y = np.load(NPY_DATA_DIR / \"feature_labels.npy\")\n        print(f\"X shape: {self.X.shape}, y shape: {self.y.shape}\")\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # Use df_index as idx due to folds splitting\n        df_index = self.df.index.values[idx]\n        x = self.X[df_index]\n        y = self.y[df_index]\n\n        x = torch.Tensor(x)\n        y = torch.Tensor([y]).long()\n\n        if self.transform:\n            x = self.transform(x)\n\n        return x, y","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.289069Z","iopub.execute_input":"2023-03-07T08:58:09.289556Z","iopub.status.idle":"2023-03-07T08:58:09.304448Z","shell.execute_reply.started":"2023-03-07T08:58:09.289507Z","shell.execute_reply":"2023-03-07T08:58:09.303097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lightning DataModule","metadata":{}},{"cell_type":"code","source":"class ASLDataModule(pl.LightningDataModule):\n    def __init__(\n        self,\n        batch_size: int,\n        data_csv_path: str,\n        in_features: int,\n        num_workers: int,\n        val_fold: float,\n    ):\n        super().__init__()\n\n        self.save_hyperparameters()\n\n        self.df = pd.read_csv(data_csv_path)\n\n        self.train_transform, self.val_transform = self._init_transforms()\n\n    def _init_transforms(self):\n        train_transform = None\n        val_transform = None\n\n        return train_transform, val_transform\n\n    def setup(self, stage=None):\n        val_fold = self.hparams.val_fold\n        train_df = self.df[self.df.fold != val_fold]\n        val_df = self.df[self.df.fold == val_fold]\n\n        if stage == \"fit\" or stage is None:\n            self.train_dataset = self._dataset(train_df, self.train_transform)\n            self.val_dataset = self._dataset(val_df, self.val_transform)\n\n    def _dataset(self, df, transform):\n        return ASLDataFrameDataset(df, self.hparams.in_features, transform=transform)\n\n    def train_dataloader(self):\n        return self._dataloader(self.train_dataset, train=True)\n\n    def val_dataloader(self):\n        return self._dataloader(self.val_dataset)\n\n    def _dataloader(self, dataset, train=False):\n        return torch.utils.data.DataLoader(\n            dataset,\n            batch_size=self.hparams.batch_size,\n            shuffle=train,\n            num_workers=self.hparams.num_workers,\n            drop_last=train,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.306254Z","iopub.execute_input":"2023-03-07T08:58:09.306958Z","iopub.status.idle":"2023-03-07T08:58:09.329743Z","shell.execute_reply.started":"2023-03-07T08:58:09.306922Z","shell.execute_reply":"2023-03-07T08:58:09.328728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Arc Margin","metadata":{}},{"cell_type":"code","source":"# From https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/blob/master/src/modeling/metric_learning.py\n# Added type annotations, device, and 16bit support\nclass ArcMarginProduct(nn.Module):\n    def __init__(\n        self,\n        in_features: int,\n        out_features: int,\n        scale: float,\n        margin: float,\n        easy_margin: bool,\n        ls_eps: float,\n    ):\n        super().__init__()\n\n        self.in_features = in_features\n        self.out_features = out_features\n        self.scale = scale\n        self.margin = margin\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(margin)\n        self.sin_m = math.sin(margin)\n        self.th = math.cos(math.pi - margin)\n        self.mm = math.sin(math.pi - margin) * margin\n\n    def forward(\n        self, input: torch.Tensor, label: torch.Tensor, device: str = \"cuda\"\n    ) -> torch.Tensor:\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n\n        # # Enable 16 bit precision\n        # cosine = cosine.to(torch.float32)\n\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n        # --------------------------- convert label to one-hot ---------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.scale\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.333117Z","iopub.execute_input":"2023-03-07T08:58:09.333587Z","iopub.status.idle":"2023-03-07T08:58:09.360212Z","shell.execute_reply.started":"2023-03-07T08:58:09.333559Z","shell.execute_reply":"2023-03-07T08:58:09.359301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\n## Source: [🤟 GISLR 🤟 - 📚Learn – 🔭EDA – 🤖Baseline](https://www.kaggle.com/code/dschettler8845/gislr-learn-eda-baseline)","metadata":{}},{"cell_type":"code","source":"class ASLLinearModel(torch.nn.Module):\n    def __init__(\n        self,\n        in_features: int,\n        first_out_features: int,\n        num_classes: int,\n        num_blocks: int,\n        drop_rate: float,\n    ):\n        super().__init__()\n\n        blocks = []\n        out_features = first_out_features\n        for idx in range(num_blocks):\n            if idx == num_blocks - 1:\n                out_features = num_classes\n\n            blocks.append(self._make_block(in_features, out_features, drop_rate))\n\n            in_features = out_features\n            out_features = out_features // 2\n\n        self.model = nn.Sequential(*blocks)\n        print(self.model)\n\n    def _make_block(self, in_features, out_features, drop_rate):\n        return nn.Sequential(\n            nn.Linear(in_features, out_features),\n            nn.BatchNorm1d(out_features),\n            nn.ReLU(),\n            nn.Dropout(drop_rate),\n        )\n\n    def forward(self, x):\n        return self.model(x)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.361546Z","iopub.execute_input":"2023-03-07T08:58:09.361992Z","iopub.status.idle":"2023-03-07T08:58:09.379114Z","shell.execute_reply.started":"2023-03-07T08:58:09.361955Z","shell.execute_reply":"2023-03-07T08:58:09.377996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lightning Module","metadata":{}},{"cell_type":"code","source":"class ASLModule(pl.LightningModule):\n    def __init__(\n        self,\n        drop_rate: float,\n        eta_min: float,\n        first_out_features: int,\n        learning_rate: float,\n        loss: str,\n        in_features: int,\n        max_epochs: int,\n        model_name: str,\n        num_blocks: int,\n        num_classes: int,\n        optimizer: str,\n        scheduler: str,\n        weight_decay: float,\n    ):\n        super().__init__()\n\n        self.save_hyperparameters()\n\n        self.model = self._init_model()\n\n        self.loss_fn = self._init_loss_fn()\n\n        self.metrics = self._init_metrics()\n\n    def _init_model(self):\n        if self.hparams.model_name == \"linear\":\n            return ASLLinearModel(\n                in_features=self.hparams.in_features,\n                first_out_features=self.hparams.first_out_features,\n                num_classes=self.hparams.num_classes,\n                num_blocks=self.hparams.num_blocks,\n                drop_rate=self.hparams.drop_rate,\n            )\n        else:\n            raise ValueError(f\"{self.hparams.model_name} is not a valid model name\")\n\n    def _init_loss_fn(self):\n        if self.hparams.loss == \"CELoss\":\n            return torch.nn.CrossEntropyLoss()\n        else:\n            raise ValueError(f\"{self.hparams.loss} is not a valid loss function\")\n\n    def _init_metrics(self):\n        metrics = {\n            \"acc\": torchmetrics.classification.MulticlassAccuracy(\n                num_classes=len(SIGN_TO_IDX)\n            ),\n        }\n        metric_collection = MetricCollection(metrics)\n\n        return torch.nn.ModuleDict(\n            {\n                \"train_metrics\": metric_collection.clone(prefix=\"train_\"),\n                \"val_metrics\": metric_collection.clone(prefix=\"val_\"),\n            }\n        )\n\n    def configure_optimizers(self):\n        optimizer = self._init_optimizer()\n\n        scheduler = self._init_scheduler(optimizer)\n\n        return {\n            \"optimizer\": optimizer,\n            \"lr_scheduler\": {\n                \"scheduler\": scheduler,\n                \"interval\": \"epoch\",\n            },\n        }\n\n    def _init_optimizer(self):\n        return create_optimizer_v2(\n            self.parameters(),\n            opt=self.hparams.optimizer,\n            lr=self.hparams.learning_rate,\n            weight_decay=self.hparams.weight_decay,\n        )\n\n    def _init_scheduler(self, optimizer):\n        if self.hparams.scheduler == \"CosineAnnealingLR\":\n            scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n                optimizer,\n                T_max=self.hparams.max_epochs,\n                eta_min=self.hparams.eta_min,\n            )\n        elif self.hparams.scheduler == \"StepLR\":\n            scheduler = torch.optim.lr_scheduler.StepLR(\n                optimizer,\n                step_size=self.hparams.max_epochs // 5,\n                gamma=0.95,\n            )\n        else:\n            raise ValueError(f\"Unknown scheduler: {self.hparams.scheduler}\")\n        return scheduler\n\n    def forward(self, x):\n        return self.model(x)\n\n    def training_step(self, batch):\n        return self._shared_step(batch, \"train\")\n\n    def validation_step(self, batch, batch_idx):\n        self._shared_step(batch, \"val\")\n\n    def predict_step(self, batch, batch_idx):\n        _, labels, logits = self._forward_pass(batch, stage=None)\n        preds = logits.sigmoid()\n        return preds, labels\n\n    def _shared_step(self, batch, stage):\n        x, y, y_pred = self._forward_pass(batch)\n\n        loss = self.loss_fn(y_pred, y)\n\n        self.metrics[f\"{stage}_metrics\"](y_pred, y)\n\n        self._log(stage, loss, batch_size=len(x))\n\n        return loss\n\n    def _forward_pass(self, batch):\n        x, y = batch\n        y = y.view(-1)\n        y_pred = self(x)\n\n        return x, y, y_pred\n\n    def _log(self, stage, loss, batch_size):\n        self.log(f\"{stage}_loss\", loss, batch_size=batch_size)\n        self.log_dict(self.metrics[f\"{stage}_metrics\"], batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.382100Z","iopub.execute_input":"2023-03-07T08:58:09.382370Z","iopub.status.idle":"2023-03-07T08:58:09.426053Z","shell.execute_reply.started":"2023-03-07T08:58:09.382345Z","shell.execute_reply":"2023-03-07T08:58:09.425089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"def load_logger_and_callbacks(\n    fast_dev_run, metrics, overfit_batches, patience, project, val_fold\n):\n    if fast_dev_run or overfit_batches > 0:\n        logger, callbacks = None, None\n    else:\n        logger, id_ = get_logger(metrics=metrics, project=project)\n        callbacks = get_callbacks(\n            id_=id_,\n            mode=list(metrics.values())[0],\n            monitor=list(metrics.keys())[0],\n            patience=patience,\n            val_fold=val_fold,\n        )\n\n    return logger, callbacks\n\n\ndef get_logger(metrics, project):\n    logger = WandbLogger(project=project)\n    id_ = logger.experiment.id\n\n    for metric, summary in metrics.items():\n        logger.experiment.define_metric(metric, summary=summary)\n\n    return logger, id_\n\n\ndef get_callbacks(id_, mode, monitor, patience, val_fold):\n    callbacks = [\n        EarlyStopping(monitor=monitor, mode=mode, patience=patience),\n        LearningRateMonitor(),\n    ]\n\n    return callbacks","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.427306Z","iopub.execute_input":"2023-03-07T08:58:09.427752Z","iopub.status.idle":"2023-03-07T08:58:09.445040Z","shell.execute_reply.started":"2023-03-07T08:58:09.427715Z","shell.execute_reply":"2023-03-07T08:58:09.444029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(\n    accelerator: str = \"gpu\",\n    batch_size: int = 256,\n    devices: int = 1,\n    drop_rate: float = 0.4,\n    eta_min: float = 1e-6,\n    fast_dev_run: bool = False,\n    first_out_features: int = 2048,\n    in_features: int = IN_FEATURES,\n    learning_rate: float = 3e-4,\n    loss: str = \"CELoss\",\n    max_epochs: int = 200,\n    model_name: str = \"linear\",\n    num_blocks: int = 3,\n    num_classes: int = 250,\n    num_workers: int = 2,\n    overfit_batches: int = 0,\n    optimizer: str = \"AdamW\",\n    patience: int = 20,\n    precision: int = 16,\n    project: str = \"asl-sign-detection-kaggle\",\n    scheduler: str = \"CosineAnnealingLR\",\n    swa: bool = False,\n    val_fold: float = 2.0,\n    weight_decay: float = 1e-6,\n):\n    pl.seed_everything(SEED, workers=True)\n\n    if fast_dev_run:\n        num_workers = 0\n\n    data_module = ASLDataModule(\n        batch_size=batch_size,\n        data_csv_path=TRAIN_CSV_PATH,\n        in_features=in_features,\n        num_workers=num_workers,\n        val_fold=val_fold,\n    )\n\n    module = ASLModule(\n        drop_rate=drop_rate,\n        eta_min=eta_min,\n        first_out_features=first_out_features,\n        in_features=in_features,\n        learning_rate=learning_rate,\n        loss=loss,\n        max_epochs=max_epochs,\n        model_name=model_name,\n        num_blocks=num_blocks,\n        num_classes=num_classes,\n        optimizer=optimizer,\n        scheduler=scheduler,\n        weight_decay=weight_decay,\n    )\n\n    logger, callbacks = load_logger_and_callbacks(\n        fast_dev_run=fast_dev_run,\n        metrics={\"val_loss\": \"min\", \"val_acc\": \"max\", \"val_f1\": \"max\"},\n        overfit_batches=overfit_batches,\n        patience=patience,\n        project=project,\n        val_fold=val_fold,\n    )\n\n    trainer = pl.Trainer(\n        accelerator=accelerator,\n        benchmark=True,\n        devices=devices,\n        callbacks=callbacks,\n        fast_dev_run=fast_dev_run,\n        logger=logger,\n        log_every_n_steps=5,\n        max_epochs=max_epochs,\n        num_sanity_val_steps=0,\n        overfit_batches=overfit_batches,\n        precision=precision,\n        strategy=\"ddp\" if devices > 1 else None,\n    )\n\n    trainer.fit(module, datamodule=data_module)\n\n    return module","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.446429Z","iopub.execute_input":"2023-03-07T08:58:09.446970Z","iopub.status.idle":"2023-03-07T08:58:09.472581Z","shell.execute_reply.started":"2023-03-07T08:58:09.446931Z","shell.execute_reply":"2023-03-07T08:58:09.471638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"module = train()","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:58:09.473896Z","iopub.execute_input":"2023-03-07T08:58:09.474913Z","iopub.status.idle":"2023-03-07T08:59:59.929102Z","shell.execute_reply.started":"2023-03-07T08:58:09.474872Z","shell.execute_reply":"2023-03-07T08:59:59.926876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Conversion\n\n## Source: [End-to-End Pytorch Training + Submission](https://www.kaggle.com/code/mayukh18/end-to-end-pytorch-training-submission)","metadata":{}},{"cell_type":"code","source":"onnx_asl_module_path = \"asl_module.onnx\"\n\ninput_sample = torch.rand((1, IN_FEATURES)).cuda()\nmodule.to_onnx(\n    file_path=onnx_asl_module_path,\n    input_sample=input_sample,\n    opset_version=12,\n    input_names=[\"input\"],\n    output_names=[\"output\"],\n    dynamic_axes={\"input\": {0: \"input\"}},\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T08:59:59.937414Z","iopub.execute_input":"2023-03-07T08:59:59.938889Z","iopub.status.idle":"2023-03-07T09:00:00.692160Z","shell.execute_reply.started":"2023-03-07T08:59:59.938840Z","shell.execute_reply":"2023-03-07T09:00:00.688932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf_model_path = \"tf_model\"\nonnx_asl_module = onnx.load(onnx_asl_module_path)\ntf_rep = prepare(onnx_asl_module)\ntf_rep.export_graph(tf_model_path)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:00:00.697548Z","iopub.execute_input":"2023-03-07T09:00:00.698236Z","iopub.status.idle":"2023-03-07T09:00:05.418031Z","shell.execute_reply.started":"2023-03-07T09:00:00.698188Z","shell.execute_reply":"2023-03-07T09:00:05.414085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Inference Model in Tensorflow\n\n## Source: [End-to-End Pytorch Training + Submission](https://www.kaggle.com/code/mayukh18/end-to-end-pytorch-training-submission)","metadata":{}},{"cell_type":"code","source":"class ASLInferModel(tf.Module):\n    def __init__(self, tf_model_path):\n        super().__init__()\n\n        self.feature_gen = FeatureGenTF()\n        self.model = tf.saved_model.load(tf_model_path)\n        self.feature_gen.trainable = False\n        self.model.trainable = False\n\n    @tf.function(\n        input_signature=[\n            tf.TensorSpec(shape=[None, 543, 3], dtype=tf.float32, name=\"inputs\")\n        ]\n    )\n    def call(self, input):\n        output_tensors = {}\n        features = self.feature_gen(tf.cast(input, dtype=tf.float32))\n\n        output_tensors[\"outputs\"] = self.model(**{\"input\": features})[\"output\"][0, :]\n\n        return output_tensors\n\n\nmytfmodel = ASLInferModel(tf_model_path)\ntf.saved_model.save(\n    mytfmodel, \"tf_infer_model\", signatures={\"serving_default\": mytfmodel.call}\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:00:05.419620Z","iopub.execute_input":"2023-03-07T09:00:05.420679Z","iopub.status.idle":"2023-03-07T09:00:06.393939Z","shell.execute_reply.started":"2023-03-07T09:00:05.420629Z","shell.execute_reply":"2023-03-07T09:00:06.392578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\n## Source: [End-to-End Pytorch Training + Submission](https://www.kaggle.com/code/mayukh18/end-to-end-pytorch-training-submission)","metadata":{}},{"cell_type":"code","source":"tf_infer_model_path = \"/kaggle/working/tf_infer_model\"\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\ntflite_model = converter.convert()\n\ntflite_model_path = \"model.tflite\"\n\n# Save the model\nwith open(tflite_model_path, \"wb\") as f:\n    f.write(tflite_model)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:00:06.395538Z","iopub.execute_input":"2023-03-07T09:00:06.396263Z","iopub.status.idle":"2023-03-07T09:00:08.291231Z","shell.execute_reply.started":"2023-03-07T09:00:06.396215Z","shell.execute_reply":"2023-03-07T09:00:08.289953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_relevant_data_subset(pq_path):\n    data_columns = [\"x\", \"y\", \"z\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:00:08.292881Z","iopub.execute_input":"2023-03-07T09:00:08.293485Z","iopub.status.idle":"2023-03-07T09:00:08.306978Z","shell.execute_reply.started":"2023-03-07T09:00:08.293446Z","shell.execute_reply":"2023-03-07T09:00:08.305916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:00:08.309296Z","iopub.execute_input":"2023-03-07T09:00:08.309674Z","iopub.status.idle":"2023-03-07T09:00:08.624360Z","shell.execute_reply.started":"2023-03-07T09:00:08.309640Z","shell.execute_reply":"2023-03-07T09:00:08.623041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{"execution":{"iopub.status.busy":"2023-03-07T09:00:08.626148Z","iopub.execute_input":"2023-03-07T09:00:08.626847Z","iopub.status.idle":"2023-03-07T09:00:10.997521Z","shell.execute_reply.started":"2023-03-07T09:00:08.626806Z","shell.execute_reply":"2023-03-07T09:00:10.996081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}