{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Google - Isolated Sign Language Recognition\n\nThe goal of this competition is to classify isolated American Sign Language (ASL) signs.\n\nThe landmarks were extracted from raw videos with the MediaPipe holistic model and are asked to predict the sign from this data.\n\n## Version 1\n- Based on https://www.kaggle.com/code/robikscube/sign-language-recognition-eda-twitch-stream\n\n## Version 2\n- Added remaining code\n\n## Version 3\n- Based on https://www.kaggle.com/code/medali1992/gislr-nn-arcface-baseline/notebook (Version 26)","metadata":{"_uuid":"00a1fb54-ef02-49e2-b4c8-6bedb26ee7fb","_cell_guid":"01400152-b59f-45f2-bdbe-d4795535ac9d","trusted":true}},{"cell_type":"markdown","source":"# pip Install","metadata":{"_uuid":"f7d6de90-dab5-49be-a40d-19ca18c7ae4e","_cell_guid":"cb242158-30bc-4aad-8a05-cfc1c903236a","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-06T09:52:37.415135Z","iopub.execute_input":"2023-03-06T09:52:37.415581Z","iopub.status.idle":"2023-03-06T09:52:37.422026Z","shell.execute_reply.started":"2023-03-06T09:52:37.415539Z","shell.execute_reply":"2023-03-06T09:52:37.420990Z"}}},{"cell_type":"code","source":"!pip install onnx_tf\n!pip install tflite-runtime\n!pip install -q --upgrade wandb\n\n# install nb_black for sutoformatting\n!pip install nb_black --quiet\n%load_ext lab_black","metadata":{"_uuid":"85ac61b4-f8d2-41bd-a6c4-733326244f59","_cell_guid":"b961df1a-d887-487b-9fc6-8642d594f590","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-06T10:12:12.383151Z","iopub.execute_input":"2023-03-06T10:12:12.383631Z","iopub.status.idle":"2023-03-06T10:12:59.377220Z","shell.execute_reply.started":"2023-03-06T10:12:12.383583Z","shell.execute_reply":"2023-03-06T10:12:59.375873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.style.use(\"seaborn-colorblind\")\n\n\nimport math\nimport random\nimport time\nfrom collections import OrderedDict\nimport tensorflow as tf\nfrom tqdm import tqdm\nimport json\nimport os\nimport gc\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split, GroupKFold, StratifiedGroupKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.optim.optimizer import Optimizer\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import (\n    CosineAnnealingWarmRestarts,\n    CosineAnnealingLR,\n    ReduceLROnPlateau,\n)\nfrom torchinfo import summary\n\nimport onnx\nimport onnx_tf\nfrom onnx_tf.backend import prepare\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nVERSION = 14\nDATA_DIR = \"/kaggle/input/asl-signs\"","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:30:54.198985Z","iopub.execute_input":"2023-03-06T10:30:54.199455Z","iopub.status.idle":"2023-03-06T10:30:54.223732Z","shell.execute_reply.started":"2023-03-06T10:30:54.199385Z","shell.execute_reply":"2023-03-06T10:30:54.222673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    score = accuracy_score(y_true, y_pred)\n    return score\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\ndef load_relevant_data_subset_with_imputation(pq_path):\n    data_columns = [\"x\", \"y\", \"z\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    data.replace(np.nan, 0, inplace=True)\n    n_frames = int(len(data) / CFG.rows_per_frame)\n    data = data.values.reshape(n_frames, CFG.rows_per_frame, len(data_columns))\n    return data.astype(np.float32)\n\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = [\"x\", \"y\"]\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / CFG.rows_per_frame)\n    data = data.values.reshape(n_frames, CFG.rows_per_frame, len(data_columns))\n    return data.astype(np.float32)\n\n\ndef read_dict(file_path):\n    path = os.path.expanduser(file_path)\n    with open(path, \"r\") as f:\n        dic = json.load(f)\n    return dic","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:30:58.141686Z","iopub.execute_input":"2023-03-06T10:30:58.142123Z","iopub.status.idle":"2023-03-06T10:30:58.166710Z","shell.execute_reply.started":"2023-03-06T10:30:58.142082Z","shell.execute_reply":"2023-03-06T10:30:58.164949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    num_workers = 2\n    apex = False\n    scheduler = \"CosineAnnealingLR\"  # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts','OneCycleLR']\n    epochs = 500\n    print_freq = 200\n    # CosineAnnealingLR params\n    cosanneal_params = {\"T_max\": 5, \"eta_min\": 3 * 1e-5, \"last_epoch\": -1}\n    # ReduceLROnPlateau params\n    reduce_params = {\n        \"mode\": \"max\",\n        \"factor\": 0.8,\n        \"patience\": 5,\n        \"eps\": 1e-6,\n        \"verbose\": True,\n    }\n    # CosineAnnealingWarmRestarts params\n    cosanneal_res_params = {\"T_0\": 3, \"eta_min\": 1e-6, \"T_mult\": 1, \"last_epoch\": -1}\n    # OneCycleLR params\n    onecycle_params = {\n        \"pct_start\": 0.1,\n        \"div_factor\": 1e1,\n        \"max_lr\": 1e-3,\n        \"steps_per_epoch\": 3,\n        \"epochs\": 3,\n    }\n    momentum = 0.9\n    model_name = \"NN_ArcFace\"\n    lr = 0.000333\n    weight_decay = 1e-4\n    gradient_accumulation_steps = 1\n    max_grad_norm = 1000\n    data_path = \"../input/asl-signs/\"\n    debug = False\n    arcface = False\n    use_aggregation_dataset = True\n    target_size = 250\n    rows_per_frame = 543\n    batch_size = 512\n    train = True\n    early_stop = True\n    target_col = \"label\"\n    scale = 30.0\n    margin = 0.50\n    easy_margin = False\n    ls_eps = 0.0\n    fc_dim = 512\n    early_stopping_steps = 5\n    grad_cam = False\n    seed = 42","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:31:01.974358Z","iopub.execute_input":"2023-03-06T10:31:01.974829Z","iopub.status.idle":"2023-03-06T10:31:02.000446Z","shell.execute_reply.started":"2023-03-06T10:31:01.974791Z","shell.execute_reply":"2023-03-06T10:31:01.998565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Directory Settings\n","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = f\"./{CFG.model_name}_version{VERSION}/\"\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\n\ndef init_logger(log_file=OUTPUT_DIR + \"train.log\"):\n    from logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n\nLOGGER = init_logger()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:31:06.070048Z","iopub.execute_input":"2023-03-06T10:31:06.070517Z","iopub.status.idle":"2023-03-06T10:31:06.088482Z","shell.execute_reply.started":"2023-03-06T10:31:06.070475Z","shell.execute_reply":"2023-03-06T10:31:06.086561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data\n","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(f\"{CFG.data_path}train.csv\")\nlabel_index = read_dict(f\"{CFG.data_path}sign_to_prediction_index_map.json\")\nindex_label = dict([(label_index[key], key) for key in label_index])\ntrain[\"label\"] = train[\"sign\"].map(lambda sign: label_index[sign])\ntrain[\"path\"] = DATA_DIR + \"/\" + train[\"path\"]\n\nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=4000, random_state=CFG.seed).reset_index(drop=True)\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:31:10.409445Z","iopub.execute_input":"2023-03-06T10:31:10.410123Z","iopub.status.idle":"2023-03-06T10:31:10.640060Z","shell.execute_reply.started":"2023-03-06T10:31:10.410065Z","shell.execute_reply":"2023-03-06T10:31:10.638686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation¶","metadata":{}},{"cell_type":"code","source":"DROP_Z = True\n\nNUM_FRAMES = 15\nSEGMENTS = 3\n\nLEFT_HAND_OFFSET = 468\nPOSE_OFFSET = LEFT_HAND_OFFSET + 21\nRIGHT_HAND_OFFSET = POSE_OFFSET + 33\n\n## average over the entire face, and the entire 'pose'\naveraging_sets = [[0, 468], [POSE_OFFSET, 33]]\n\nlip_landmarks = [\n    61,\n    185,\n    40,\n    39,\n    37,\n    0,\n    267,\n    269,\n    270,\n    409,\n    291,\n    146,\n    91,\n    181,\n    84,\n    17,\n    314,\n    405,\n    321,\n    375,\n    78,\n    191,\n    80,\n    81,\n    82,\n    13,\n    312,\n    311,\n    310,\n    415,\n    95,\n    88,\n    178,\n    87,\n    14,\n    317,\n    402,\n    318,\n    324,\n    308,\n]\nleft_hand_landmarks = list(range(LEFT_HAND_OFFSET, LEFT_HAND_OFFSET + 21))\nright_hand_landmarks = list(range(RIGHT_HAND_OFFSET, RIGHT_HAND_OFFSET + 21))\n\npoint_landmarks = [\n    item\n    for sublist in [lip_landmarks, left_hand_landmarks, right_hand_landmarks]\n    for item in sublist\n]\n\nLANDMARKS = len(point_landmarks) + len(averaging_sets)\nprint(LANDMARKS)\nif DROP_Z:\n    INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 2)\nelse:\n    INPUT_SHAPE = (NUM_FRAMES, LANDMARKS * 3)\n\nFLAT_INPUT_SHAPE = (INPUT_SHAPE[0] + 2 * (SEGMENTS + 1)) * INPUT_SHAPE[1]","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:31:13.063522Z","iopub.execute_input":"2023-03-06T10:31:13.064040Z","iopub.status.idle":"2023-03-06T10:31:13.096384Z","shell.execute_reply.started":"2023-03-06T10:31:13.063992Z","shell.execute_reply":"2023-03-06T10:31:13.094832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_nan_mean(x, axis=0):\n    return tf.reduce_sum(\n        tf.where(tf.math.is_nan(x), tf.zeros_like(x), x), axis=axis\n    ) / tf.reduce_sum(\n        tf.where(tf.math.is_nan(x), tf.zeros_like(x), tf.ones_like(x)), axis=axis\n    )\n\n\ndef tf_nan_std(x, axis=0):\n    d = x - tf_nan_mean(x, axis=axis)\n    return tf.math.sqrt(tf_nan_mean(d * d, axis=axis))\n\n\ndef flatten_means_and_stds(x, axis=0):\n    # Get means and stds\n    x_mean = tf_nan_mean(x, axis=0)\n    x_std = tf_nan_std(x, axis=0)\n\n    x_out = tf.concat([x_mean, x_std], axis=0)\n    x_out = tf.reshape(x_out, (1, INPUT_SHAPE[1] * 2))\n    x_out = tf.where(tf.math.is_finite(x_out), x_out, tf.zeros_like(x_out))\n    return x_out","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:31:17.565569Z","iopub.execute_input":"2023-03-06T10:31:17.566546Z","iopub.status.idle":"2023-03-06T10:31:17.584848Z","shell.execute_reply.started":"2023-03-06T10:31:17.566496Z","shell.execute_reply":"2023-03-06T10:31:17.583412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGen(tf.keras.layers.Layer):\n    def __init__(self):\n        super(FeatureGen, self).__init__()\n\n    def call(self, x_in):\n        #         print(right_hand_percentage(x))\n        x_list = [\n            tf.expand_dims(\n                tf_nan_mean(x_in[:, av_set[0] : av_set[0] + av_set[1], :], axis=1),\n                axis=1,\n            )\n            for av_set in averaging_sets\n        ]\n        x_list.append(tf.gather(x_in, point_landmarks, axis=1))\n        x = tf.concat(x_list, 1)\n\n        x_padded = x\n        for i in range(SEGMENTS):\n            p0 = tf.where(\n                ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) != 0), 1, 0\n            )\n            p1 = tf.where(\n                ((tf.shape(x_padded)[0] % SEGMENTS) > 0) & ((i % 2) == 0), 1, 0\n            )\n            paddings = [[p0, p1], [0, 0], [0, 0]]\n            x_padded = tf.pad(x_padded, paddings, mode=\"SYMMETRIC\")\n        x_list = tf.split(x_padded, SEGMENTS)\n        x_list = [flatten_means_and_stds(_x, axis=0) for _x in x_list]\n\n        x_list.append(flatten_means_and_stds(x, axis=0))\n\n        ## Resize only dimension 0. Resize can't handle nan, so replace nan with that dimension's avg value to reduce impact.\n        x = tf.image.resize(\n            tf.where(tf.math.is_finite(x), x, tf_nan_mean(x, axis=0)),\n            [NUM_FRAMES, LANDMARKS],\n        )\n        x = tf.reshape(x, (1, INPUT_SHAPE[0] * INPUT_SHAPE[1]))\n        x = tf.where(tf.math.is_nan(x), tf.zeros_like(x), x)\n        x_list.append(x)\n        x = tf.concat(x_list, axis=1)\n        return x\n\n\nfeature_converter = FeatureGen()\n\nX = np.load(\n    \"/kaggle/input/gislr-feature-data-on-the-shoulders/feature_data.npy\"\n).astype(np.float32)\ny = np.load(\n    \"/kaggle/input/gislr-feature-data-on-the-shoulders/feature_labels.npy\"\n).astype(np.uint8)\nprint(X.shape, y.shape)\n\nif DROP_Z:\n    X = np.reshape(X, [X.shape[0], -1, 3])\n    X = X[:, :, 0:2]\n    X = np.reshape(X, [X.shape[0], -1])\n    print(X.shape, y.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:32:26.731686Z","iopub.execute_input":"2023-03-06T10:32:26.732123Z","iopub.status.idle":"2023-03-06T10:33:10.274741Z","shell.execute_reply.started":"2023-03-06T10:32:26.732086Z","shell.execute_reply":"2023-03-06T10:33:10.273666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Tracking","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_key\")\n\nimport wandb\n\nwandb.login(key=wandb_api)\n\n\ndef class2dict(f):\n    return dict(\n        (name, getattr(f, name)) for name in dir(f) if not name.startswith(\"__\")\n    )\n\n\nrun = wandb.init(\n    project=\"GISLR Competition\",\n    name=f\"{CFG.model_name}_Version{VERSION}\",\n    config=class2dict(CFG),\n    group=CFG.model_name,\n    job_type=\"train\",\n)","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:49:08.394165Z","iopub.execute_input":"2023-03-06T10:49:08.394670Z","iopub.status.idle":"2023-03-06T10:49:39.659276Z","shell.execute_reply.started":"2023-03-06T10:49:08.394615Z","shell.execute_reply":"2023-03-06T10:49:39.657949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataSet","metadata":{}},{"cell_type":"code","source":"class Dataset(Dataset):\n    def __init__(self, X, y):\n        self.X = X\n        self.y = y\n\n    def __len__(self):\n        return len(self.y)\n\n    def __getitem__(self, i):\n        return self.X[i].astype(np.float32), self.y[i]","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:50:56.644449Z","iopub.execute_input":"2023-03-06T10:50:56.644924Z","iopub.status.idle":"2023-03-06T10:50:56.656464Z","shell.execute_reply.started":"2023-03-06T10:50:56.644878Z","shell.execute_reply":"2023-03-06T10:50:56.654850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    def __init__(\n        self,\n        in_features,\n        out_features,\n        scale=30.0,\n        margin=0.50,\n        easy_margin=False,\n        ls_eps=0.0,\n    ):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.scale = scale\n        self.margin = margin\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(margin)\n        self.sin_m = math.sin(margin)\n        self.th = math.cos(math.pi - margin)\n        self.mm = math.sin(math.pi - margin) * margin\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.scale\n\n        return output\n\n\n# https://github.com/haqishen/Google-Landmark-Recognition-2020-3rd-Place-Solution/blob/main/landmark-recognition-2020-third-place-submission.ipynb\nclass ArcMarginProduct_subcenter(nn.Module):\n    def __init__(self, in_features, out_features, k=3):\n        super().__init__()\n        self.weight = nn.Parameter(torch.FloatTensor(out_features * k, in_features))\n        self.reset_parameters()\n        self.k = k\n        self.out_features = out_features\n\n    def reset_parameters(self):\n        stdv = 1.0 / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n\n    def forward(self, features):\n        cosine_all = F.linear(F.normalize(features), F.normalize(self.weight))\n        cosine_all = cosine_all.view(-1, self.out_features, self.k)\n        cosine, _ = torch.max(cosine_all, dim=2)\n        return cosine","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:50:59.279565Z","iopub.execute_input":"2023-03-06T10:50:59.280263Z","iopub.status.idle":"2023-03-06T10:50:59.317644Z","shell.execute_reply.started":"2023-03-06T10:50:59.280201Z","shell.execute_reply":"2023-03-06T10:50:59.316273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ASLLinearModel(torch.nn.Module):\n    def __init__(\n        self,\n        in_features: int,\n        first_out_features: int,\n        num_classes: int,\n        num_blocks: int,\n        drop_rate: float,\n    ):\n        super(ASLLinearModel, self).__init__()\n\n        blocks = []\n        out_features = first_out_features\n        for idx in range(num_blocks):\n            blocks.append(self._make_block(in_features, out_features, drop_rate))\n\n            in_features = out_features\n            out_features = out_features // 2\n\n        self.model = nn.Sequential(*blocks)\n        self.final = ArcMarginProduct(\n            256,\n            num_classes,\n            scale=CFG.scale,\n            margin=CFG.margin,\n            easy_margin=False,\n            ls_eps=0.0,\n        )\n        self.fc_probs = nn.Linear(256, num_classes)\n        print(self.model)\n\n    def _make_block(self, in_features, out_features, drop_rate):\n        return nn.Sequential(\n            nn.Linear(in_features, out_features),\n            nn.BatchNorm1d(out_features),\n            nn.ReLU(),\n            nn.Dropout(drop_rate),\n        )\n\n    def forward(self, x, label):\n        feature = self.model(x)\n        if CFG.arcface:\n            arcface = self.final(feature, label)\n            probs = self.fc_probs(feature)\n            return probs, arcface\n        else:\n            probs = self.fc_probs(feature)\n            return probs","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:51:02.982935Z","iopub.execute_input":"2023-03-06T10:51:02.983396Z","iopub.status.idle":"2023-03-06T10:51:03.008452Z","shell.execute_reply.started":"2023-03-06T10:51:02.983351Z","shell.execute_reply":"2023-03-06T10:51:03.006953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Function","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return \"%dm %ds\" % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return \"%s (remain %s)\" % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    if CFG.apex:\n        scaler = GradScaler()\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (features, labels) in enumerate(train_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        features = features.to(device).float()\n        labels = labels.to(device).long()\n        batch_size = labels.size(0)\n        if CFG.apex:\n            with autocast():\n                if CFG.arcface:\n                    probs, arcface = model(features, labels)\n                    arcface_loss = nn.CrossEntropyLoss()(arcface, labels)\n                    loss = criterion(probs, labels)\n                else:\n                    y_preds = model(features, labels)\n                    loss = criterion(y_preds, labels)\n        else:\n            if CFG.arcface:\n                probs, arcface = model(features, labels)\n                arcface_loss = nn.CrossEntropyLoss()(arcface, labels)\n                loss = criterion(probs, labels)\n            else:\n                y_preds = model(features, labels)\n                loss = criterion(y_preds, labels)\n        # record loss\n        if CFG.arcface:\n            loss = 0.5 * loss + 0.5 * arcface_loss\n            losses.update(loss.item(), batch_size)\n        else:\n            losses.update(loss.item(), batch_size)\n\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        if CFG.apex:\n            scaler.scale(loss).backward()\n        else:\n            loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(\n            model.parameters(), CFG.max_grad_norm\n        )\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            if CFG.apex:\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader) - 1):\n            print(\n                \"Epoch: [{0}][{1}/{2}] \"\n                \"Elapsed {remain:s} \"\n                \"Loss: {loss.val:.4f}({loss.avg:.4f}) \"\n                \"Grad: {grad_norm:.4f} \"\n                \"LR: {lr:.6f}  \".format(\n                    epoch + 1,\n                    step,\n                    len(train_loader),\n                    remain=timeSince(start, float(step + 1) / len(train_loader)),\n                    loss=losses,\n                    grad_norm=grad_norm,\n                    lr=scheduler.get_lr()[0],\n                )\n            )\n        wandb.log(\n            {\n                f\"loss\": losses.val,\n                f\"lr\": scheduler.get_lr()[0],\n            }\n        )\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (features, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        features = features.to(device).float()\n        labels = labels.to(device).long()\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            if CFG.arcface:\n                y_preds, _ = model(features, labels)\n            else:\n                y_preds = model(features, labels)\n\n        preds.append(y_preds.softmax(1).to(\"cpu\").numpy())\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader) - 1):\n            print(\n                \"EVAL: [{0}/{1}] \"\n                \"Elapsed {remain:s} \"\n                \"Loss: {loss.val:.4f}({loss.avg:.4f}) \".format(\n                    step,\n                    len(valid_loader),\n                    loss=losses,\n                    remain=timeSince(start, float(step + 1) / len(valid_loader)),\n                )\n            )\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:51:06.493213Z","iopub.execute_input":"2023-03-06T10:51:06.493696Z","iopub.status.idle":"2023-03-06T10:51:06.570072Z","shell.execute_reply.started":"2023-03-06T10:51:06.493653Z","shell.execute_reply":"2023-03-06T10:51:06.568571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Loop¶\n","metadata":{}},{"cell_type":"code","source":"# Seed for producing results\nseed_torch(seed=45)\n\n# ====================================================\n# loader\n# ====================================================\ngroups = train[\"path\"].map(lambda x: x.split(\"/\")[5])\nsgkf = StratifiedGroupKFold(n_splits=5, random_state=42, shuffle=True)\nfor i, (train_index, valid_index) in enumerate(sgkf.split(X, y, groups)):\n    train_index = train_index\n    valid_index = valid_index\n    print(f\"Fold {i}:\")\n    print(f\"  Train index shape: {train_index.shape}\")\n    print(f\"         group={groups[train_index]}\")\n    print(f\"  Valid index shape:  {valid_index.shape}\")\n    print(f\"         group={groups[valid_index]}\")\n    break\nX_train, X_val, y_train, y_val = (\n    X[train_index],\n    X[valid_index],\n    y[train_index],\n    y[valid_index],\n)\ntrain_dataset = Dataset(X_train, y_train)\nvalid_dataset = Dataset(X_val, y_val)\n\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=CFG.batch_size,\n    shuffle=True,\n    num_workers=CFG.num_workers,\n    pin_memory=True,\n    drop_last=True,\n)\nvalid_loader = DataLoader(\n    valid_dataset,\n    batch_size=CFG.batch_size,\n    shuffle=False,\n    num_workers=CFG.num_workers,\n    pin_memory=True,\n    drop_last=False,\n)\n\n\n# ====================================================\n# scheduler\n# ====================================================\ndef get_scheduler(optimizer):\n    if CFG.scheduler == \"ReduceLROnPlateau\":\n        scheduler = ReduceLROnPlateau(optimizer, **CFG.reduce_params)\n    elif CFG.scheduler == \"CosineAnnealingLR\":\n        scheduler = CosineAnnealingLR(optimizer, **CFG.cosanneal_params)\n    elif CFG.scheduler == \"CosineAnnealingWarmRestarts\":\n        scheduler = CosineAnnealingWarmRestarts(optimizer, **CFG.reduce_params)\n    return scheduler\n\n\n# ====================================================\n# model & optimizer\n# ====================================================\nmodel = ASLLinearModel(\n    in_features=3864,\n    first_out_features=1024,\n    num_classes=250,\n    num_blocks=3,\n    drop_rate=0.4,\n)\nmodel.to(device)\n\noptimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\nscheduler = get_scheduler(optimizer)\n\n# ====================================================\n# loop\n# ====================================================\ncriterion = nn.CrossEntropyLoss()\nbest_score = 0\nfor epoch in range(CFG.epochs):\n    start_time = time.time()\n\n    # train\n    avg_loss = train_fn(\n        train_loader, model, criterion, optimizer, epoch, scheduler, device\n    )\n\n    # eval\n    avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n\n    if isinstance(scheduler, ReduceLROnPlateau):\n        scheduler.step(avg_val_loss)\n    elif isinstance(scheduler, CosineAnnealingLR):\n        scheduler.step()\n    elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n        scheduler.step()\n\n    score = get_score(y_val, preds.argmax(1))\n\n    elapsed = time.time() - start_time\n\n    LOGGER.info(\n        f\"Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s\"\n    )\n    LOGGER.info(f\"Epoch {epoch+1} - Score: {score:.4f}\")\n    wandb.log(\n        {\n            f\"epoch\": epoch + 1,\n            f\"avg_train_loss\": avg_loss,\n            f\"avg_val_loss\": avg_val_loss,\n            f\"score\": score,\n        }\n    )\n\n    if best_score < score:\n        best_score = score\n        LOGGER.info(f\"Epoch {epoch+1} - Save Best score: {best_score:.4f} Model\")\n        torch.save(\n            model.state_dict(),\n            OUTPUT_DIR + f\"{CFG.model_name}_best_score_version{VERSION}.pth\",\n        )\nLOGGER.info(f\"Our CV score is {best_score}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-06T10:51:11.131815Z","iopub.execute_input":"2023-03-06T10:51:11.132272Z","iopub.status.idle":"2023-03-06T14:04:22.892307Z","shell.execute_reply.started":"2023-03-06T10:51:11.132229Z","shell.execute_reply":"2023-03-06T14:04:22.889732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorflow Conversion¶\n","metadata":{}},{"cell_type":"code","source":"class Model_infe(ASLLinearModel):\n    def __init__(self):\n        super().__init__(\n            in_features=3864,\n            first_out_features=1024,\n            num_classes=250,\n            num_blocks=3,\n            drop_rate=0.4,\n        )\n\n    def forward(self, x):\n        feature = self.model(x)\n        probs = self.fc_probs(feature)\n        return probs\n\n\nmodel_infe = Model_infe()\nmodel_infe.load_state_dict(\n    torch.load(OUTPUT_DIR + f\"{CFG.model_name}_best_score_version{VERSION}.pth\"),\n    strict=False,\n)\nmodel_infe = model_infe.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-03-07T03:51:57.166085Z","iopub.execute_input":"2023-03-07T03:51:57.166967Z","iopub.status.idle":"2023-03-07T03:51:57.204858Z","shell.execute_reply.started":"2023-03-07T03:51:57.166919Z","shell.execute_reply":"2023-03-07T03:51:57.203057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_input = torch.rand((1, 3864)).to(device)\nonnx_model_path = \"asl_model.onnx\"\n\nmodel_infe.eval()\n\ntorch.onnx.export(\n    model_infe,  # PyTorch Model\n    sample_input,  # Input tensor\n    onnx_model_path,  # Output file (eg. 'output_model.onnx')\n    opset_version=12,  # Operator support version\n    input_names=[\"input\"],  # Input tensor name (arbitary)\n    output_names=[\"output\"],  # Output tensor name (arbitary)\n    dynamic_axes={\"input\": {0: \"input\"}},\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import onnx\nfrom onnx_tf.backend import prepare\n\ntf_model_path = \"/kaggle/working/tf_model\"\nonnx_model = onnx.load(onnx_model_path)\ntf_rep = prepare(onnx_model)\ntf_rep.export_graph(tf_model_path)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Inference Model in Tensorflow","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\n\nclass ASLInferModel(tf.Module):\n    def __init__(self):\n        super(ASLInferModel, self).__init__()\n        self.feature_gen = FeatureGen()\n        self.model = tf.saved_model.load(tf_model_path)\n        self.feature_gen.trainable = False\n        self.model.trainable = False\n\n    @tf.function(\n        input_signature=[\n            tf.TensorSpec(shape=[None, 543, 2], dtype=tf.float32, name=\"inputs\")\n        ]\n    )\n    def call(self, input):\n        output_tensors = {}\n        features = self.feature_gen(tf.cast(input, dtype=tf.float32))\n        output_tensors[\"outputs\"] = self.model(**{\"input\": features})[\"output\"][0, :]\n        return output_tensors\n\nmytfmodel = ASLInferModel()\ntf.saved_model.save(\n    mytfmodel,\n    \"/kaggle/working/tf_infer_model\",\n    signatures={\"serving_default\": mytfmodel.call},\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Convert the model\n\ntf_infer_model_path = \"/kaggle/working/tf_infer_model\"\nconverter = tf.lite.TFLiteConverter.from_saved_model(tf_infer_model_path)\ntflite_model = converter.convert()\n\ntflite_model_path = \"model.tflite\"\n\n# Save the model\nwith open(tflite_model_path, \"wb\") as f:\n    f.write(tflite_model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROWS_PER_FRAME = 543  # number of landmarks per frame\npq_path = \"/kaggle/input/asl-signs/train_landmark_files/53618/1001379621.parquet\"\n\nimport tflite_runtime.interpreter as tflite\n\ninterpreter = tflite.Interpreter(tflite_model_path)\ninterpreter.allocate_tensors()\n\nfound_signatures = list(interpreter.get_signature_list().keys())\n\n# if REQUIRED_SIGNATURE not in found_signatures:\n#     raise KernelEvalException('Required input signature not found.')\n\nprediction_fn = interpreter.get_signature_runner(\"serving_default\")\noutput = prediction_fn(inputs=load_relevant_data_subset(pq_path))\nsign = np.argmax(output[\"outputs\"])\n\nprint(sign, output[\"outputs\"].shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip submission.zip $tflite_model_path","metadata":{},"execution_count":null,"outputs":[]}]}