{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This code is a Google - Isolated Sign Language Recognition 4th place trainning code.   \nIt has been changed a few times since the last post to run on kaggle notebook.  \nThe model in the final submission is an ensemble of six models, with the ability to create these configurations by switching CFGs.  \nSince it takes about 12 hours to train a single model, it is recommended to run it on a fast machine or experiment with a reduced number of epochs to 30.  ","metadata":{}},{"cell_type":"code","source":"!pip install cleanlab","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:30.208821Z","iopub.execute_input":"2023-05-06T08:21:30.209529Z","iopub.status.idle":"2023-05-06T08:21:40.984934Z","shell.execute_reply.started":"2023-05-06T08:21:30.209499Z","shell.execute_reply":"2023-05-06T08:21:40.983821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\nfrom scipy.stats import truncnorm\nfrom torchvision.ops import StochasticDepth\nimport multiprocessing\nimport math\nimport random\nfrom sklearn.model_selection import GroupKFold\nimport os\n\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nimport socket\nimport torch\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\nfrom cleanlab.filter import find_label_issues\n\nfrom torch.utils.data import Dataset, DataLoader\nimport warnings\nwarnings.filterwarnings(action='ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-06T08:21:40.987166Z","iopub.execute_input":"2023-05-06T08:21:40.987527Z","iopub.status.idle":"2023-05-06T08:21:44.731558Z","shell.execute_reply.started":"2023-05-06T08:21:40.987492Z","shell.execute_reply":"2023-05-06T08:21:44.730654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## define keypoints ","metadata":{}},{"cell_type":"code","source":"n_class = 250\nROWS_PER_FRAME = 543\n\nface_keypoints = dict(\n    silhouette=[\n        10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288,\n        397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136,\n        172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109],\n\n    lipsUpperOuter=[61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291],\n    lipsLowerOuter=[146, 91, 181, 84, 17, 314, 405, 321, 375],\n    lipsUpperInner=[78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308],\n    lipsLowerInner=[95, 88, 178, 87, 14, 317, 402, 318, 324],\n\n    rightEyeUpper0=[246, 161, 160, 159, 158, 157, 173],\n    rightEyeLower0=[33, 7, 163, 144, 145, 153, 154, 155, 133],\n    rightEyeUpper1=[247, 30, 29, 27, 28, 56, 190],\n    rightEyeLower1=[130, 25, 110, 24, 23, 22, 26, 112, 243],\n    rightEyeUpper2=[113, 225, 224, 223, 222, 221, 189],\n    rightEyeLower2=[226, 31, 228, 229, 230, 231, 232, 233, 244],\n    rightEyeLower3=[143, 111, 117, 118, 119, 120, 121, 128, 245],\n\n    rightEyebrowUpper=[156, 70, 63, 105, 66, 107, 55, 193],\n    rightEyebrowLower=[35, 124, 46, 53, 52, 65],\n\n    rightEyeIris=[473, 474, 475, 476, 477],\n\n    leftEyeUpper0=[466, 388, 387, 386, 385, 384, 398],\n    leftEyeLower0=[263, 249, 390, 373, 374, 380, 381, 382, 362],\n    leftEyeUpper1=[467, 260, 259, 257, 258, 286, 414],\n    leftEyeLower1=[359, 255, 339, 254, 253, 252, 256, 341, 463],\n    leftEyeUpper2=[342, 445, 444, 443, 442, 441, 413],\n    leftEyeLower2=[446, 261, 448, 449, 450, 451, 452, 453, 464],\n    leftEyeLower3=[372, 340, 346, 347, 348, 349, 350, 357, 465],\n\n    leftEyebrowUpper=[383, 300, 293, 334, 296, 336, 285, 417],\n    leftEyebrowLower=[265, 353, 276, 283, 282, 295],\n\n    leftEyeIris=[468, 469, 470, 471, 472],\n\n    midwayBetweenEyes=[168],\n\n    noseTip=[1],\n    noseBottom=[2],\n    noseRightCorner=[98],\n    noseLeftCorner=[327],\n\n    rightCheek=[205],\n    leftCheek=[425],\n)\n\nhand_keypoints = dict(\n    thumb=[1, 2, 3, 4],\n    indexFinger=[5, 6, 7, 8],\n    middleFinger=[9, 10, 11, 12],\n    ringFinger=[13, 14, 15, 16],\n    pinky=[17, 18, 19, 20],\n    palmBase=[0]\n)\n\npose_keypoints = dict(\n    leftArm=[12, 14, 16],\n    rightArm=[11, 13, 15],\n    body=[11, 23, 24, 12],\n)\n\nface_start_index = 0\nleft_start_index = 468\npose_start_index = 489\nright_start_index = 522\n\n# [468 - 488] 21\nl_hand_indexes = []\nfor keypoint_name in hand_keypoints:\n    for index in hand_keypoints[keypoint_name]:\n        l_hand_indexes.append(index + left_start_index)\nl_hand_indexes = list(set(l_hand_indexes))\nl_hand_indexes.sort()\nl_hand_indexes = np.array(l_hand_indexes)\n\n# [522 - 542] 21\nr_hand_indexes = []\nfor keypoint_name in hand_keypoints:\n    for index in hand_keypoints[keypoint_name]:\n        r_hand_indexes.append(index + right_start_index)\nr_hand_indexes = list(set(r_hand_indexes))\nr_hand_indexes.sort()\nr_hand_indexes = np.array(r_hand_indexes)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.733049Z","iopub.execute_input":"2023-05-06T08:21:44.733620Z","iopub.status.idle":"2023-05-06T08:21:44.748670Z","shell.execute_reply.started":"2023-05-06T08:21:44.733589Z","shell.execute_reply":"2023-05-06T08:21:44.747326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## utility functions","metadata":{}},{"cell_type":"code","source":"def fix_seed(seed=0):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # random\n    random.seed(seed)\n    # Numpy\n    np.random.seed(seed)\n    # Pytorch\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\nclass AffineMatTools():\n    def __init__(self):\n        self.A_list = []\n\n    def composit(self):\n        M = np.eye(3)\n        for A in self.A_list:\n            M = A.dot(M)\n        M = M[:2]\n        return M\n\n    def composit3(self):\n        M = np.eye(3)\n        for A in self.A_list:\n            M = A.dot(M)\n        return M\n\n    def adjust_composit(self, w, h):\n        M = self.composit()\n        X = np.array(((0, 0, 1), (w - 1, 0, 1), (0, h - 1, 1), (w - 1, h - 1, 1))).T\n        Z = M.dot(X)\n        min_x, min_y = Z.min(axis=1)\n        max_x, max_y = Z.max(axis=1)\n        # print(max_x , min_x)\n        # print(max_y , min_y)\n        new_w = round(max_x - min_x + 1)\n        new_h = round(max_y - min_y + 1)\n        self.shift(-min_x, -min_y)\n        M = self.composit()\n        return M, new_w, new_h\n\n    def scale(self, x, y=None):\n        if y is None:\n            y = x\n        A = np.eye(3)\n        A[0, 0] = x\n        A[1, 1] = y\n        self.A_list.append(A)\n\n    def shift(self, x, y):\n        A = np.eye(3)\n        A[0, 2] = x\n        A[1, 2] = y\n        self.A_list.append(A)\n\n    def rotation_radian(self, r):\n        A = np.eye(3)\n        A[0, 0] = math.cos(r)\n        A[1, 1] = math.cos(r)\n        A[0, 1] = -math.sin(r)\n        A[1, 0] = math.sin(r)\n        self.A_list.append(A)\n\n    def rotation_degree(self, d):\n        r = d * (2 * math.pi) / 360\n        self.rotation_radian(r)\n\n    def skew_x_radian(self, r):\n        A = np.eye(3)\n        A[0, 1] = math.tan(r)\n        self.A_list.append(A)\n\n    def skew_y_radian(self, r):\n        A = np.eye(3)\n        A[1, 0] = math.tan(r)\n        self.A_list.append(A)\n\n    def skew_x_degree(self, d):\n        r = d * (2 * math.pi) / 360\n        self.skew_x_radian(r)\n\n    def skew_y_degree(self, d):\n        r = d * (2 * math.pi) / 360\n        self.skew_y_radian(r)\n\n    def transform(self, data):\n        data = data.copy()\n        shape = data.shape\n        data = data.reshape((-1, 2))\n        data = np.concatenate((data, np.ones((data.shape[0], 1))), axis=1)\n        M = self.composit3()\n        data = data @ M.T\n        data = (data / data[:, 2][:, None])[:, :2]\n        data = data.reshape(shape)\n        return data\n\n\ndef evaluate_topK(preds, targets, topK=5):\n    topK_idx = np.argsort(preds, axis=1)[:, ::-1][:, :topK]\n    acc_topKs = np.cumsum((topK_idx - targets[:, None]) == 0, axis=1)\n    acc_topK = np.mean(acc_topKs, axis=0)\n    return acc_topK, acc_topKs\n\n\ndef load_relevant_data_subset(pq_path):\n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(pq_path, columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n    return data.astype(np.float32)\n\n\ndef make_kfold(df, target=\"participant_id\", n_splits=7):\n\n    kf = GroupKFold(n_splits)\n    df[\"fold\"] = -1\n    folds = {}\n    for i, (trn_idx, val_idx) in enumerate(kf.split(df, df[target], df[target])):\n        df.loc[val_idx, \"fold\"] = i\n        print(len(df.loc[df[\"fold\"] == i][target].unique()))\n        target_keys = df.loc[df[\"fold\"] == i][target].unique()\n        for target_key in target_keys:\n            folds[target_key] = i\n\n    print(df[\"fold\"].value_counts())\n    return df\n\n\ndef load_df(group=\"participant_id\", n_fold=7):\n    df = pd.read_csv(TRAIN_FILE)\n    label_map = json.load(open(JSON_FILE, \"r\"))\n    df['label'] = df['sign'].map(label_map)\n    df = make_kfold(df, target=group, n_splits=n_fold)\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.751453Z","iopub.execute_input":"2023-05-06T08:21:44.751728Z","iopub.status.idle":"2023-05-06T08:21:44.773413Z","shell.execute_reply.started":"2023-05-06T08:21:44.751698Z","shell.execute_reply":"2023-05-06T08:21:44.772525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### models","metadata":{}},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    def __init__(self, in_features, out_features, s=30.0, m=0.5, easy_margin=False):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = torch.nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, x, label=None):\n        cosine = torch.nn.functional.linear(torch.nn.functional.normalize(\n            x), torch.nn.functional.normalize(self.weight)).float()\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n        if label is None:\n            output = cosine\n        else:\n            one_hot = torch.zeros(cosine.size(), device='cuda')\n            one_hot.scatter_(1, label.cuda().view(-1, 1).long(), 1)\n            # you can use torch.where if your torch.__version__ is 0.4\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output\n\n\nclass LRUnit1D(nn.Module):\n    def __init__(self, in_dim, stochastic_depth_prob=0.0, actf=torch.nn.ReLU()):\n        super(LRUnit1D, self).__init__()\n        self.layer0 = nn.Linear(in_dim, in_dim, bias=False)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.layer1 = nn.Linear(in_dim, in_dim, bias=False)\n        self.bn1 = nn.BatchNorm1d(in_dim)\n        self.actf = actf\n        self.stochastic_depth = StochasticDepth(stochastic_depth_prob, \"row\")\n\n    def forward(self, x):\n        h = self.actf(self.bn0(self.layer0(x)))\n        h = self.bn1(self.layer1(h))\n        h = self.stochastic_depth(h)\n        return x + h\n\n\nclass RSUnit1D(nn.Module):\n    def __init__(self, in_dim, kernel_size=3, padding=1,\n                 padding_mode='zeros', actf=torch.nn.ReLU()):\n        super(RSUnit1D, self).__init__()\n        self.layer0 = nn.Conv1d(in_dim, in_dim, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode, bias=False)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.layer1 = nn.Conv1d(in_dim, in_dim, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode, bias=False)\n        self.bn1 = nn.BatchNorm1d(in_dim)\n        self.actf = actf\n\n    def forward(self, x):\n        h = self.actf(self.bn0(self.layer0(x)))\n        h = self.bn1(self.layer1(h))\n        return x + h\n# %%\n\n\nclass BackboneVariableLen(nn.Module):\n    def __init__(self,\n                 in_dim=42,\n                 dim1=128,\n                 dim2=512,\n                 kernel_size=3, padding=1,\n                 negative_slope=0.1,\n                 ):\n        super(BackboneVariableLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.conv1 = nn.Conv1d(in_dim, dim1, kernel_size=kernel_size, padding=padding, bias=True)\n        self.conv2 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv3 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv4 = RSUnit1D(dim1, kernel_size=1, padding=0, actf=self.actf)\n        self.conv5 = RSUnit1D(dim1, kernel_size=1, padding=0, actf=self.actf)\n        self.conv6 = RSUnit1D(dim1, kernel_size=1, padding=0, actf=self.actf)\n        self.conv6a = nn.Conv1d(dim1, dim2, kernel_size=1, padding=0, bias=True)\n\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n    def forward(self, x):\n        m = x[:, 0]  # mask\n        h = x[:, 1:]  # feature\n\n        h = self.bn0(h)\n        h = self.conv1(h)\n        h = self.conv2(h)\n        h = self.conv3(h)\n        h = self.conv4(h)\n        h = self.conv5(h)\n        h = self.conv6(h)\n        h = self.conv6a(h)\n\n        # Since it is padded according to the maximum length in the batch,\n        # pooling as it is will have an adverse effect.\n        # So we mask it with the original sequence length,\n        # put in a negative value, and then do max_poolin.\n        m = torch.unsqueeze(m, 1)\n        mask = (m == 0).repeat(1, h.shape[1], 1)\n        h[mask] = -1e16\n\n        h = self.g_pool(h)\n        h = self.flatten(h)\n        return h\n\n\nclass ModelVariableLen(nn.Module):\n    def __init__(self,\n                 in_dim_H=42,\n                 in_dim_L=80,\n                 dim1_H=128,\n                 dim1_L=64,\n                 dim2=512,\n                 n_class=250,\n                 kernel_size=3, padding=1,\n                 arc_m=0.5,\n                 arc_s=15,\n                 easy_margin=False,\n                 stochastic_depth_prob=0.0,\n                 negative_slope=0.1,\n                 n_recyile=1\n                 ):\n        super(ModelVariableLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.hand_module = BackboneVariableLen(in_dim=in_dim_H, dim1=dim1_H, dim2=dim2, negative_slope=negative_slope)\n        self.other_module = BackboneVariableLen(in_dim=in_dim_L, dim1=dim1_L, dim2=dim2, negative_slope=negative_slope)\n\n        self.line1 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line2 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line3 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.end = ArcMarginProduct(dim2, n_class, easy_margin=easy_margin, s=arc_s, m=arc_m)\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n        self.n_recyile = n_recyile\n\n    def forward(self, hand, other):\n        hand = self.hand_module(hand)\n        other = self.other_module(other)\n        h = hand + other\n        for n in range(self.n_recyile):\n            h = self.line1(h)\n        for n in range(self.n_recyile):\n            h = self.line2(h)\n        for n in range(self.n_recyile):\n            h = self.line3(h)\n        h = self.end(h)\n        return h\n\n# %%\n\n\nclass BackboneFixedLen(nn.Module):\n    def __init__(self,\n                 in_dim=42,\n                 dim1=128,\n                 dim2=512,\n                 kernel_size=3, padding=1,\n                 negative_slope=0.1,\n                 ):\n        super(BackboneFixedLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.conv1 = nn.Conv1d(in_dim, dim1, kernel_size=3, padding=1, bias=True)\n        self.conv2 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv3 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv4 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv5 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv6 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv6a = nn.Conv1d(dim1, dim2, kernel_size=1, padding=0, bias=True)\n\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n    def forward(self, x):\n        h = self.bn0(x)\n        h = self.conv1(h)\n        h = self.conv2(h)\n        h = self.pool(h)\n        h = self.conv3(h)\n        h = self.pool(h)\n        h = self.conv4(h)\n        h = self.pool(h)\n        h = self.conv5(h)\n        h = self.conv6(h)\n        h = self.conv6a(h)\n        h = self.g_pool(h)\n        h = self.flatten(h)\n        return h\n\n\nclass ModelFixedLen(nn.Module):\n    def __init__(self,\n                 in_dim_H=42,\n                 in_dim_L=80,\n                 dim1_H=128,\n                 dim1_L=64,\n                 dim2=512,\n                 n_class=250,\n                 kernel_size=3, padding=1,\n                 arc_m=0.5,\n                 arc_s=15,\n                 easy_margin=False,\n                 stochastic_depth_prob=0.0,\n                 negative_slope=0.1,\n                 n_recyile=1\n                 ):\n        super(ModelFixedLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.hand_module = BackboneFixedLen(in_dim=in_dim_H, dim1=dim1_H, dim2=dim2, negative_slope=negative_slope)\n        self.other_module = BackboneFixedLen(in_dim=in_dim_L, dim1=dim1_L, dim2=dim2, negative_slope=negative_slope)\n\n        self.line1 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line2 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line3 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.end = ArcMarginProduct(dim2, n_class, easy_margin=easy_margin, s=arc_s, m=arc_m)\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n        self.n_recyile = n_recyile\n\n    def forward(self, hand, other):\n        hand = self.hand_module(hand)\n        other = self.other_module(other)\n        h = hand + other\n        for n in range(self.n_recyile):\n            h = self.line1(h)\n        for n in range(self.n_recyile):\n            h = self.line2(h)\n        for n in range(self.n_recyile):\n            h = self.line3(h)\n        h = self.end(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.774762Z","iopub.execute_input":"2023-05-06T08:21:44.775098Z","iopub.status.idle":"2023-05-06T08:21:44.813042Z","shell.execute_reply.started":"2023-05-06T08:21:44.775070Z","shell.execute_reply":"2023-05-06T08:21:44.812234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### dataset","metadata":{}},{"cell_type":"code","source":"class AslDataset(Dataset):\n\n    print(\"class init\")\n    feature_keys = ['r_hand',           # 21\n                    'lipsUpperOuter',   #\n                    'lipsLowerOuter',\n                    'lipsUpperInner',\n                    'lipsLowerInner']\n\n    def __init__(self,\n                 df,\n                 # data_list,\n                 aug_hand_param=None,\n                 frame_drop=0.0):\n        self.df = df\n        # self.data_list = data_list\n        self.aug_hand_param = aug_hand_param\n        self.frame_drop = frame_drop\n\n    def __len__(self):\n        return len(self.df)\n        # return len(self.data_list)\n\n    def get_data(self,\n                 data_dict,\n                 aug_hand_param=None,\n                 frame_drop=0.0):\n\n        # data_dict = data_dict.copy()\n\n        # Compare the number of frames and flip horizontally\n        length = data_dict[\"l_hand\"].shape[0]\n        l_num = length - np.isnan(data_dict[\"l_hand\"][:, 0, 0]).sum()\n        r_num = length - np.isnan(data_dict[\"r_hand\"][:, 0, 0]).sum()\n        if r_num < l_num:\n            data_dict = self.mirrored(data_dict)\n\n        # augumentation of hand\n        if aug_hand_param:\n            data_dict[\"r_hand\"] = self.apply_aug_hand(data_dict[\"r_hand\"], aug_hand_param)\n\n        # concat data\n        data = np.concatenate([data_dict[key] for key in AslDataset.feature_keys], axis=1)\n\n        # Delete frames randomly\n        data = self.apply_frame_drop(data, frame_drop)\n\n        # ignore z\n        data = data[:, :, :2]\n\n        x = torch.tensor(data)  # TxPx2\n\n        hand_mask = ~torch.isnan(x[:, 0, 0])  # T\n\n        # normalization\n        x = x - x[~torch.isnan(x)].mean(0, keepdims=True)\n        x = x / x[~torch.isnan(x)].std(0, keepdims=True)\n\n        x[torch.isnan(x)] = 0.0  # TxPx2\n        x = torch.reshape(x, (x.shape[0], -1))\n        x = torch.permute(x, (1, 0))  # 2P x T\n\n        # extract only where there is a hand, return all if there is no hand\n        if hand_mask.sum() > 0:\n            hand = x[:42, hand_mask]\n        else:\n            hand = x[:42]\n        other = x[42:]\n\n        return hand, other\n\n    def apply_aug_hand(self, hand, aug_hand_param, debug=False):\n        angle = random.gauss(0, aug_hand_param[\"angle\"] / 2)\n        scale = random.gauss(1, aug_hand_param[\"scale\"] / 2)\n        shift_x = random.gauss(0, aug_hand_param[\"shift_x\"] / 2)\n        shift_y = random.gauss(0, aug_hand_param[\"shift_y\"] / 2)\n\n        amt = AffineMatTools()\n        amt.scale(scale)\n        amt.rotation_degree(angle)\n        amt.shift(shift_x, shift_y)\n\n        aug_hand = hand - hand[:, 0][:, None]\n\n        aug_hand_z = aug_hand[:, :, 2][:, :, None]\n        aug_hand = aug_hand[:, :, :2]\n        aug_hand = amt.transform(aug_hand)\n        aug_hand = np.concatenate((aug_hand, aug_hand_z), axis=2)\n        aug_hand = aug_hand + hand[:, 0][:, None]\n\n        aug_hand = aug_hand.astype(np.float32)\n        return aug_hand\n\n    def mirrored(self, data_dict):\n        mirrored_dict = {}\n\n        def invert_x(tmp):\n            tmp = tmp.copy()\n            tmp[:, :, 0] = -tmp[:, :, 0]\n            return tmp\n        mirrored_dict[\"r_hand\"] = invert_x(data_dict[\"l_hand\"])\n        mirrored_dict[\"l_hand\"] = invert_x(data_dict[\"r_hand\"])\n        mirrored_dict[\"lipsUpperOuter\"] = invert_x(data_dict[\"lipsUpperOuter\"][:, ::-1])\n        mirrored_dict[\"lipsLowerOuter\"] = invert_x(data_dict[\"lipsLowerOuter\"][:, ::-1])\n        mirrored_dict[\"lipsUpperInner\"] = invert_x(data_dict[\"lipsUpperInner\"][:, ::-1])\n        mirrored_dict[\"lipsLowerInner\"] = invert_x(data_dict[\"lipsLowerInner\"][:, ::-1])\n        return mirrored_dict\n\n    def apply_frame_drop(self, data, frame_drop):\n        # Drop frames at random percentage frame_drop\n        if frame_drop > 0.0:\n            drop_mask = np.random.random(len(data)) >= frame_drop\n            dropped_data = data[drop_mask]\n\n            if len(dropped_data) >= 2:\n                data = dropped_data\n        return data\n\n    def __getitem__(self, index):\n        path = os.path.join(CFG.preproccessed_data, str(self.df.iloc[index][\"sequence_id\"]) + \".pickle\")\n        data_dict, y = joblib.load(path)\n        x0, x1 = self.get_data(data_dict,\n                               aug_hand_param=self.aug_hand_param,\n                               frame_drop=self.frame_drop)\n        y = torch.tensor(y)\n        return x0, x1, y\n\n\ndef make_data_dict(data):\n\n    # normalize by eyebrow position\n    midwayBetweenEyes = data[:, 168]\n    m = np.nanmean(midwayBetweenEyes, axis=0, keepdims=True)\n    data = data - m\n\n    l_hand = data[:, l_hand_indexes]\n    r_hand = data[:, r_hand_indexes]\n    lipsUpperOuter = data[:, face_keypoints[\"lipsUpperOuter\"]]\n    lipsLowerOuter = data[:, face_keypoints[\"lipsLowerOuter\"]]\n    lipsUpperInner = data[:, face_keypoints[\"lipsUpperInner\"]]\n    lipsLowerInner = data[:, face_keypoints[\"lipsLowerInner\"]]\n\n    data_dict = dict(l_hand=l_hand,\n                     r_hand=r_hand,\n                     lipsUpperOuter=lipsUpperOuter,\n                     lipsLowerOuter=lipsLowerOuter,\n                     lipsUpperInner=lipsUpperInner,\n                     lipsLowerInner=lipsLowerInner)\n\n    return data_dict\n\n\ndef preproccesing_data_dict(row):\n    x = load_relevant_data_subset(row[0])\n    data_dict = make_data_dict(x)\n    y = row[1]\n    name = os.path.basename(row[0]).replace(\".parquet\", \".pickle\")\n    # name = str(row[0].sequence_id) + \".pickle\"\n    data_path = os.path.join(CFG.preproccessed_data, name)\n    joblib.dump((data_dict, y), data_path)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.814308Z","iopub.execute_input":"2023-05-06T08:21:44.814675Z","iopub.status.idle":"2023-05-06T08:21:44.835555Z","shell.execute_reply.started":"2023-05-06T08:21:44.814646Z","shell.execute_reply":"2023-05-06T08:21:44.834571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train utilities","metadata":{}},{"cell_type":"code","source":"\ndef train(model, loader, optimizer,\n          scheduler=None,\n          loss_func=nn.CrossEntropyLoss(ignore_index=-1),\n          device=\"cuda\"):\n\n    model.train()\n    model.to(device)\n\n    mean_loss = 0\n    n = 0\n    #for (data1, data2, target) in tqdm(loader):\n    for (data1, data2, target) in (loader):\n        data1 = data1.to(device)\n        data2 = data2.to(device)\n        target = target.to(device)\n        optimizer.zero_grad()\n        pred = model(data1, data2)\n        loss = 0.1 * loss_func(pred, target)\n        loss.backward()\n        optimizer.step()\n\n        loss = loss.detach().cpu().numpy()\n\n        mean_loss += loss * len(data1)\n\n        n += len(data1)\n\n        if scheduler:\n            scheduler.step()\n    optimizer.zero_grad()\n\n    mean_loss = mean_loss / n\n    return mean_loss\n\n\ndef valid(model, loader,\n          loss_func=nn.CrossEntropyLoss(ignore_index=-1, reduction=\"none\"),\n          atc_func=nn.Softmax(),\n          device=\"cuda\"):\n\n    model.eval()\n    model.to(device)\n\n    mean_loss = 0\n    PREDS = []\n    TARGETS = []\n    LOSSES = []\n    n = 0\n    #for (data1, data2, target) in tqdm(loader):\n    for (data1, data2, target) in loader:\n        data1 = data1.to(device)\n        data2 = data2.to(device)\n        target = target.to(device)\n\n        with torch.inference_mode():\n            pred = model(data1, data2)\n        pred = pred.squeeze(1)\n        loss = 0.1 * loss_func(pred, target)\n        loss = torch.reshape(loss, (loss.shape[0], -1))\n        loss = loss.sum(dim=1)\n        loss = loss.detach().cpu().numpy()\n\n        mean_loss += loss.mean() * len(data1)\n        n += len(data1)\n\n        pred = atc_func(pred)\n        pred = pred.detach().cpu().numpy()\n        target = target.detach().cpu().numpy()\n        PREDS.append(pred)\n        TARGETS.append(target)\n        LOSSES.append(loss)\n\n    mean_loss = mean_loss / n\n\n    PREDS = np.concatenate(PREDS)\n    TARGETS = np.concatenate(TARGETS)\n    LOSSES = np.concatenate(LOSSES)\n\n    return mean_loss, PREDS, TARGETS, LOSSES\n\n\ndef pack_seq_variable_len(seq, padding=0):\n\n    # Pack sequences of different lengths.\n    # Embed a mask that indicates the sequence length at the 0th feature.\n    max_len = 1024\n    min_len = 8\n    length = [s.shape[1] for s in seq]\n    batch_size = len(seq)\n    num_landmark = seq[0].shape[0]\n    x_len = min(max(min_len, max(length)), max_len)\n    x = torch.zeros((batch_size, 1 + num_landmark, x_len))\n    for b in range(batch_size):\n        L = length[b]\n        L = min((L, max_len))\n        x[b, 1:, :L] = seq[b][:, :L]\n        x[b, 0, :L] = 1\n    length = torch.tensor(length)\n    return x, length\n\n\ndef collate_fn_variable_len(batch):\n    images1, images2, targets = list(zip(*batch))\n    images1, length1 = pack_seq_variable_len(images1)\n    images2, length2 = pack_seq_variable_len(images2)\n    targets = torch.stack(targets)\n    return images1, images2, targets\n\n\ndef pack_seq_fixed_len(seq, length):\n\n    batch_size = len(seq)\n    num_landmark = seq[0].shape[0]\n    x = torch.zeros((batch_size, num_landmark, length))\n    for b in range(batch_size):\n        data = seq[b]\n        data = torch.unsqueeze(data, 0)\n        data = torch.unsqueeze(data, 0)\n        data = torch.nn.functional.interpolate(data, size=(data.shape[2], length))\n        x[b] = data[0, 0]\n    return x\n\n\nclass CollateFnFixedLen():\n    def __init__(self, mm=96, ss=0, aa=64, bb=128):\n        super().__init__()\n        self.mm = mm\n        self.ss = ss\n        self.aa = aa\n        self.bb = bb\n\n    def __call__(self, batch):\n        if self.ss > 0:\n            length = int(np.round(truncnorm.rvs((self.aa - self.mm) / self.ss,\n                                                (self.bb - self.mm) / self.ss,\n                                                loc=self.mm, scale=self.ss)))\n        else:\n            length = self.mm\n\n        hand, lips, targets = list(zip(*batch))\n        images1 = pack_seq_fixed_len(hand, length)\n        images2 = pack_seq_fixed_len(lips, length)\n        targets = torch.stack(targets)\n        return images1, images2, targets\n\n\ndef lr_func(epoch):\n    if epoch < 1:\n        return 0.01\n    elif epoch < 2:\n        return 0.1\n    elif epoch < 6:\n        return 1\n    elif epoch < 10:\n        return 0.2\n    else:\n        return 0.2**2\n\n\n@torch.no_grad()\ndef update_bn(loader, model, device=None):\n    \"\"\"\n    The original update_bn does not support multiple inputs, so I modified it.\n    \"\"\"\n    momenta = {}\n    for module in model.modules():\n        if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):\n            module.running_mean = torch.zeros_like(module.running_mean)\n            module.running_var = torch.ones_like(module.running_var)\n            momenta[module] = module.momentum\n\n    if not momenta:\n        return\n\n    was_training = model.training\n    model.train()\n    for module in momenta.keys():\n        module.momentum = None\n        module.num_batches_tracked *= 0\n\n    for input0, input1, target in tqdm(loader):\n        if device:\n            input0 = input0.to(device)\n            input1 = input1.to(device)\n        model(input0, input1)\n\n    for bn_module in momenta.keys():\n        bn_module.momentum = momenta[bn_module]\n    model.train(was_training)","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.837010Z","iopub.execute_input":"2023-05-06T08:21:44.837332Z","iopub.status.idle":"2023-05-06T08:21:44.860523Z","shell.execute_reply.started":"2023-05-06T08:21:44.837303Z","shell.execute_reply":"2023-05-06T08:21:44.859476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_cleanlab(CFG, df, ):\n    if CFG.use_cleanlab:\n\n        \"\"\"\n        Removal of label noise by cleanlab requires posterior probabilities for each data.\n        Read csv which precomputed by 21-participant_id folds CV.\n        https://github.com/cleanlab/cleanlab\n        \"\"\"\n        \n        valid_df_21fold = pd.read_csv(\"/kaggle/input/asl-4th-place-data/valid_df_21fold.csv\")\n        filter_by = 'both'\n        is_noise = find_label_issues(\n            valid_df_21fold.label,\n            valid_df_21fold[pred_col_names].values,\n            filter_by=filter_by,\n        )\n        valid_df_21fold[\"is_noise\"] = is_noise\n        df = df.merge(valid_df_21fold[[\"sequence_id\", \"is_noise\"]], on=\"sequence_id\", how=\"left\")\n    else:\n        df[\"is_noise\"] = False\n    print(f\"noise num: {df.is_noise.sum()}\")\n    return df\n\n\ndef get_loader(CFG, df, i_fold):\n\n    if i_fold == -1:  # for make submit model\n\n        train_df = df.copy()\n        train_df = train_df[~ train_df.is_noise].reset_index(drop=True)\n\n        T1 = AslDataset(train_df,\n                        aug_hand_param=CFG.aug_hand_param,\n                        frame_drop=CFG.frame_drop,\n                        )\n        I1 = DataLoader(T1,\n                        batch_size=CFG.batchsize1,\n                        num_workers=CFG.num_workers,\n                        shuffle=True,\n                        drop_last=True,\n                        collate_fn=CFG.collate_fn1)\n        return I1\n    else:  # for validation\n        # data_list1 = []\n        # data_list2 = []\n\n        train_df = df[df.fold != i_fold].reset_index(drop=True)\n        valid_df = df[df.fold == i_fold].reset_index(drop=True)\n\n        train_df = train_df[~ train_df.is_noise].reset_index(drop=True)\n\n        T1 = AslDataset(train_df,\n                        aug_hand_param=CFG.aug_hand_param,\n                        frame_drop=CFG.frame_drop,\n                        )\n        T2 = AslDataset(valid_df)\n\n        I1 = DataLoader(T1, batch_size=CFG.batchsize1,\n                        num_workers=CFG.num_workers,\n                        shuffle=True,\n                        drop_last=True,\n                        collate_fn=CFG.collate_fn1)\n        I2 = DataLoader(T2,\n                        batch_size=CFG.batchsize2,\n                        num_workers=CFG.num_workers,\n                        shuffle=False,\n                        collate_fn=CFG.collate_fn2)\n        return I1, I2\n\n\ndef get_models(CFG):\n\n    model = CFG.Model(in_dim_H=CFG.in_dim_H,\n                      in_dim_L=CFG.in_dim_L,\n                      dim1_H=CFG.dim1_H,\n                      dim1_L=CFG.dim1_L,\n                      dim2=CFG.dim2,\n                      easy_margin=CFG.easy_margin,\n                      arc_s=CFG.arc_s,\n                      arc_m=CFG.arc_m,\n                      stochastic_depth_prob=CFG.stochastic_depth_prob,\n                      n_recyile=CFG.n_recyile\n                      )\n    model.to(CFG.device)\n    swa_model = torch.optim.swa_utils.AveragedModel(model)\n    swa_model.to(CFG.device)\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\n    org_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_func)\n    swa_scheduler = torch.optim.swa_utils.SWALR(optimizer, swa_lr=CFG.swa_lr)\n\n    return model, swa_model, optimizer, org_scheduler, swa_scheduler\n\n\ndef train_submit_model(CFG, df):\n\n    fix_seed(CFG.seed)\n    os.makedirs(CFG.result_path, exist_ok=True)\n\n    loss_func1 = torch.nn.CrossEntropyLoss(label_smoothing=CFG.label_smoothing)\n\n    I1 = get_loader(CFG, df, i_fold=-1)\n\n    model, swa_model, optimizer, org_scheduler, swa_scheduler = get_models(CFG)\n\n    for epoch in tqdm(range(1, CFG.EPOCHS + 1)):\n\n        if epoch < CFG.swa_start:\n            tr_loss = train(model, I1, optimizer, loss_func=loss_func1)\n            org_scheduler.step()\n            log = f\"{epoch:3d} {tr_loss:1.5f}\"\n            print(log)\n            with open(os.path.join(CFG.result_path, \"log.txt\"), \"a\") as f:\n                f.write(log + \"\\n\")\n        else:\n            tr_loss = train(model, I1, optimizer, loss_func=loss_func1)\n            swa_model.update_parameters(model)\n            swa_scheduler.step()\n\n    update_bn(I1, swa_model, device=CFG.device)\n    swa_model_path = os.path.join(CFG.result_path, f\"swa_{epoch}.pt\")\n    model.eval()\n    torch.save(swa_model.state_dict(), swa_model_path)\n\n\ndef validation(CFG, df):\n\n    os.makedirs(CFG.result_path, exist_ok=True)\n    loss_func1 = torch.nn.CrossEntropyLoss(label_smoothing=CFG.label_smoothing)\n    loss_func2 = torch.nn.CrossEntropyLoss(reduction=\"none\")\n    valid_dfs = []\n    for i_fold in range(CFG.n_fold):\n        fix_seed(CFG.seed)\n        valid_df = df[df.fold == i_fold].reset_index(drop=True)\n        I1, I2 = get_loader(CFG, df, i_fold=i_fold)\n        model, swa_model, optimizer, org_scheduler, swa_scheduler = get_models(CFG)\n\n        for epoch in tqdm(range(1, CFG.EPOCHS + 1)):\n\n            if epoch < CFG.swa_start:\n                tr_loss = train(model, I1, optimizer, loss_func=loss_func1)\n                org_scheduler.step()\n                log = f\"{i_fold:1d} {epoch:3d} {tr_loss:1.5f}\"\n                print(log)\n                with open(os.path.join(CFG.result_path, \"valid_log.txt\"), \"a\") as f:\n                    f.write(log + \"\\n\")\n            else:\n                tr_loss = train(model, I1, optimizer, loss_func=loss_func1)\n                swa_model.update_parameters(model)\n                swa_scheduler.step()\n\n            if epoch in CFG.save_epochs:\n                update_bn(I1, swa_model, device=CFG.device)\n                vl_loss, preds, targets, losses = valid(swa_model, I2, loss_func=loss_func2, device=CFG.device)\n                acc_topK, acc_topKs = evaluate_topK(preds, targets, topK=5)\n                log = f\"{i_fold:1d} {epoch:3d} {tr_loss:1.5f} {vl_loss:1.5f} {acc_topK[0]:.5f} {acc_topK[4]:.5f}\"\n                print(log)\n                with open(os.path.join(CFG.result_path, \"valid_log.txt\"), \"a\") as f:\n                    f.write(log + \"\\n\")\n\n                swa_model_path = os.path.join(CFG.result_path, f\"{i_fold}_swa_{epoch}.pt\")\n                torch.save(swa_model.state_dict(), swa_model_path)\n\n        swa_model_path = os.path.join(CFG.result_path, f\"{i_fold}_swa_{epoch}.pt\")\n        torch.save(swa_model.state_dict(), swa_model_path)\n        vl_loss, preds, targets, losses = valid(swa_model, I2, loss_func=loss_func2, device=CFG.device)\n        valid_df = df[df.fold == i_fold].reset_index(drop=True)\n        valid_df[pred_col_names] = preds\n        valid_df[\"loss\"] = losses\n        valid_dfs.append(valid_df)\n    valid_dfs = pd.concat(valid_dfs)\n    valid_dfs_path = os.path.join(CFG.result_path, \"valid_df.csv\")\n    valid_dfs.to_csv(valid_dfs_path)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T09:58:16.779629Z","iopub.execute_input":"2023-05-06T09:58:16.779981Z","iopub.status.idle":"2023-05-06T09:58:16.810621Z","shell.execute_reply.started":"2023-05-06T09:58:16.779952Z","shell.execute_reply":"2023-05-06T09:58:16.809774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Please select experimtal condition\n+ https://www.kaggle.com/competitions/asl-signs/discussion/406673\n\n","metadata":{}},{"cell_type":"code","source":"\n# experimet_mode = \"A0\"\nexperimet_mode = \"B0\"\n# experimet_mode = \"B1\"\n# experimet_mode = \"C0\"\n# experimet_mode = \"E\"\n# experimet_mode = \"F\"\n\n\nclass CFG:\n    output_path = '../working/'\n    preproccessed_data = '../preproccessed_data/'\n    n_fold = 3\n    topK = 5\n    EPOCHS = 30 # Originally 300 epochs, but reduced to 30.\n    batchsize1 = 32\n    batchsize2 = 64\n    device = \"cuda\"\n\n    num_workers = 2\n    lr = 0.01\n    weight_decay = 0.0005\n    swa_lr = 0.01\n    swa_start = 16\n\n    frame_drop = 0.3\n    label_smoothing = 0.5\n    arc_s = 47.0\n    arc_m = 0.0\n    easy_margin = True\n\n    aug_hand_param = {}\n    aug_hand_param[\"angle\"] = 7.0\n    aug_hand_param[\"scale\"] = 0.10\n    aug_hand_param[\"shift_x\"] = 0.016\n    aug_hand_param[\"shift_y\"] = 0.053\n\n    in_dim_H = 42\n    in_dim_L = 80\n\n    dim1_H = 128\n    dim1_L = 64\n    dim2 = 512\n\n    SEQ_LEN = 96\n    save_epochs = (20, 50, 100, 200, 300)\n\n    if experimet_mode == \"A0\":\n        # A0:\n        seed = 0\n        use_cleanlab = False\n        Model = ModelFixedLen\n        n_recyile = 3\n        stochastic_depth_prob = 0.5\n        collate_fn1 = CollateFnFixedLen(mm=SEQ_LEN, ss=32, aa=SEQ_LEN - 32, bb=SEQ_LEN + 32)\n        collate_fn2 = CollateFnFixedLen(mm=SEQ_LEN, ss=0)\n        exp_name = f\"ModelFixedLen_deep_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    elif experimet_mode == \"B0\":\n        # B0\n        seed = 5\n        use_cleanlab = True\n        Model = ModelFixedLen\n        n_recyile = 3\n        stochastic_depth_prob = 0.5\n        collate_fn1 = CollateFnFixedLen(mm=SEQ_LEN, ss=32, aa=SEQ_LEN - 32, bb=SEQ_LEN + 32)\n        collate_fn2 = CollateFnFixedLen(mm=SEQ_LEN, ss=0)\n        exp_name = f\"ModelFixedLen_deep_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    elif experimet_mode == \"B1\":\n        # B1\n        seed = 6\n        use_cleanlab = True\n        Model = ModelFixedLen\n        n_recyile = 3\n        stochastic_depth_prob = 0.5\n        collate_fn1 = CollateFnFixedLen(mm=SEQ_LEN, ss=32, aa=SEQ_LEN - 32, bb=SEQ_LEN + 32)\n        collate_fn2 = CollateFnFixedLen(mm=SEQ_LEN, ss=0)\n        exp_name = f\"ModelFixedLen_deep_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    elif experimet_mode == \"C0\":\n        # C0\n        seed = 5\n        use_cleanlab = False\n        Model = ModelFixedLen\n        n_recyile = 1\n        stochastic_depth_prob = 0.1\n        collate_fn1 = CollateFnFixedLen(mm=SEQ_LEN, ss=32, aa=SEQ_LEN - 32, bb=SEQ_LEN + 32)\n        collate_fn2 = CollateFnFixedLen(mm=SEQ_LEN, ss=0)\n        exp_name = f\"ModelFixedLen_shallow_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    elif experimet_mode == \"E\":\n        seed = 35\n        use_cleanlab = True\n        Model = ModelVariableLen\n        n_recyile = 3\n        stochastic_depth_prob = 0.5\n        collate_fn1 = collate_fn_variable_len\n        collate_fn2 = collate_fn_variable_len\n        exp_name = f\"ModelVariableLen_deep_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    elif experimet_mode == \"F\":\n        seed = 150\n        use_cleanlab = False\n        Model = ModelVariableLen\n        n_recyile = 1\n        stochastic_depth_prob = 0.1\n        collate_fn1 = collate_fn_variable_len\n        collate_fn2 = collate_fn_variable_len\n        exp_name = f\"ModelVariableLen_shallow_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    result_path = os.path.join(\"../working/\", exp_name)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.887211Z","iopub.execute_input":"2023-05-06T08:21:44.887634Z","iopub.status.idle":"2023-05-06T08:21:44.902842Z","shell.execute_reply.started":"2023-05-06T08:21:44.887604Z","shell.execute_reply":"2023-05-06T08:21:44.901844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"DEBUG=False\n\nINPUT_PATH = \"../input/asl-signs/\"\nLANDMARK_FILES_DIR = os.path.join(INPUT_PATH, \"train_landmark_files\")\nTRAIN_FILE = os.path.join(INPUT_PATH, \"train.csv\")\nJSON_FILE = os.path.join(INPUT_PATH, \"sign_to_prediction_index_map.json\")\npred_col_names = [f\"pred{c:03}\" for c in range(n_class)]\n\n\ndf = load_df(group=\"participant_id\", n_fold=CFG.n_fold)\n\nif DEBUG:\n    df=df[::100].reset_index(drop=True)\n    CFG.EPOCHS = 20\n    \ndf = apply_cleanlab(CFG, df)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:21:44.906514Z","iopub.execute_input":"2023-05-06T08:21:44.907476Z","iopub.status.idle":"2023-05-06T08:22:03.942712Z","shell.execute_reply.started":"2023-05-06T08:21:44.907448Z","shell.execute_reply":"2023-05-06T08:22:03.941775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = [(os.path.join(INPUT_PATH, row.path), row.label) for row in df.itertuples()]\nos.makedirs(CFG.preproccessed_data, exist_ok=True)\nwith multiprocessing.Pool(multiprocessing.cpu_count()) as pool:\n    imap = pool.imap(preproccesing_data_dict, rows)\n    data_list = list(tqdm(imap, total=len(rows)))\ndel data_list\n\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:22:03.944161Z","iopub.execute_input":"2023-05-06T08:22:03.944780Z","iopub.status.idle":"2023-05-06T08:39:32.657774Z","shell.execute_reply.started":"2023-05-06T08:22:03.944735Z","shell.execute_reply":"2023-05-06T08:39:32.656619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Execute a 3fold validation.\n### It will probably take about 24-30 hours.","metadata":{}},{"cell_type":"code","source":"# %% for 3 fold validation\n# validation(CFG, df)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-06T08:39:32.659675Z","iopub.execute_input":"2023-05-06T08:39:32.661169Z","iopub.status.idle":"2023-05-06T08:39:32.666822Z","shell.execute_reply.started":"2023-05-06T08:39:32.661138Z","shell.execute_reply":"2023-05-06T08:39:32.666110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create one of the models for submitting.\n### It will probably take 10 hours at 300 epoch. ","metadata":{}},{"cell_type":"code","source":"train_submit_model(CFG, df)","metadata":{"execution":{"iopub.status.busy":"2023-05-06T09:58:23.626973Z","iopub.execute_input":"2023-05-06T09:58:23.627391Z","iopub.status.idle":"2023-05-06T09:58:24.001552Z","shell.execute_reply.started":"2023-05-06T09:58:23.627357Z","shell.execute_reply":"2023-05-06T09:58:23.999513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}