{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This code is a Google - Isolated Sign Language Recognition 4th place trainning code.   \nIt has been changed a few times since the last post to run on kaggle notebook.  \nThe model in the final submission is an ensemble of six models, with the ability to create these configurations by switching CFGs.  \nSince it takes about 12 hours to train a single model, it is recommended to run it on a fast machine or experiment with a reduced number of epochs to 30.  ","metadata":{}},{"cell_type":"code","source":"!pip install cleanlab","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:31.984125Z","iopub.execute_input":"2023-07-12T06:29:31.984582Z","iopub.status.idle":"2023-07-12T06:29:44.743463Z","shell.execute_reply.started":"2023-07-12T06:29:31.984535Z","shell.execute_reply":"2023-07-12T06:29:44.742242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\nfrom scipy.stats import truncnorm\nfrom torchvision.ops import StochasticDepth\nimport multiprocessing\nimport math\nimport random\nfrom sklearn.model_selection import GroupKFold\nimport os\nimport json\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nimport socket\nimport torch\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\nfrom cleanlab.filter import find_label_issues\nfrom torch.utils.data import Dataset, DataLoader\nimport warnings\nwarnings.filterwarnings(action='ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-12T06:29:44.747579Z","iopub.execute_input":"2023-07-12T06:29:44.747927Z","iopub.status.idle":"2023-07-12T06:29:44.755806Z","shell.execute_reply.started":"2023-07-12T06:29:44.747896Z","shell.execute_reply":"2023-07-12T06:29:44.754747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## define keypoints ","metadata":{}},{"cell_type":"code","source":"n_class = 250\nROWS_PER_FRAME = 543\n\nface_keypoints = dict(\n    lipsUpperOuter=[61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291],\n    lipsLowerOuter=[146, 91, 181, 84, 17, 314, 405, 321, 375],\n    lipsUpperInner=[78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308],\n    lipsLowerInner=[95, 88, 178, 87, 14, 317, 402, 318, 324],\n\n)\n\nhand_keypoints = dict(\n    thumb=[1, 2, 3, 4],\n    indexFinger=[5, 6, 7, 8],\n    middleFinger=[9, 10, 11, 12],\n    ringFinger=[13, 14, 15, 16],\n    pinky=[17, 18, 19, 20],\n    palmBase=[0]\n)\n\nleft_start_index = 468\nright_start_index = 522\nl_hand_indexes = []\nfor keypoint_name in hand_keypoints:\n    for index in hand_keypoints[keypoint_name]:\n        l_hand_indexes.append(index + left_start_index)\nl_hand_indexes = list(set(l_hand_indexes))\nl_hand_indexes.sort()\nl_hand_indexes = np.array(l_hand_indexes)\nr_hand_indexes = []\nfor keypoint_name in hand_keypoints:\n    for index in hand_keypoints[keypoint_name]:\n        r_hand_indexes.append(index + right_start_index)\nr_hand_indexes = list(set(r_hand_indexes))\nr_hand_indexes.sort()\nr_hand_indexes = np.array(r_hand_indexes)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.757242Z","iopub.execute_input":"2023-07-12T06:29:44.757651Z","iopub.status.idle":"2023-07-12T06:29:44.772398Z","shell.execute_reply.started":"2023-07-12T06:29:44.757617Z","shell.execute_reply":"2023-07-12T06:29:44.771367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## utility functions","metadata":{}},{"cell_type":"code","source":"def data_load(group=\"participant_id\", n_fold=7):\n    df = pd.read_csv(TRAIN_FILE)\n    label_map = json.load(open(JSON_FILE, \"r\"))\n    df['label'] = df['sign'].map(label_map)\n    \n    kf = GroupKFold(n_fold)\n    df[\"fold\"] = -1\n    folds = {}\n    for i, (trn_idx, val_idx) in enumerate(kf.split(df, df[group], df[group])):\n        df.loc[val_idx, \"fold\"] = i\n        print(len(df.loc[df[\"fold\"] == i][group].unique()))\n        target_keys = df.loc[df[\"fold\"] == i][group].unique()\n        for target_key in target_keys:\n            folds[target_key] = i\n\n    print(df[\"fold\"].value_counts())\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.775107Z","iopub.execute_input":"2023-07-12T06:29:44.775521Z","iopub.status.idle":"2023-07-12T06:29:44.785805Z","shell.execute_reply.started":"2023-07-12T06:29:44.775489Z","shell.execute_reply":"2023-07-12T06:29:44.784861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### models","metadata":{}},{"cell_type":"code","source":"class lossfunc(nn.Module):\n    def __init__(self, in_features, out_features, s=30.0, m=0.5, easy_margin=False):\n        super(lossfunc, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.weight = torch.nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, x, label=None):\n        cosine = torch.nn.functional.linear(torch.nn.functional.normalize(\n            x), torch.nn.functional.normalize(self.weight)).float()\n        sine = torch.sqrt((1.0 - torch.pow(cosine, 2)).clamp(0, 1))\n        phi = cosine * self.cos_m - sine * self.sin_m\n\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n        if label is None:\n            output = cosine\n        else:\n            one_hot = torch.zeros(cosine.size(), device='cuda')\n            one_hot.scatter_(1, label.cuda().view(-1, 1).long(), 1)\n            # you can use torch.where if your torch.__version__ is 0.4\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output\n\n\nclass LRUnit1D(nn.Module):\n    def __init__(self, in_dim, stochastic_depth_prob=0.0, actf=torch.nn.ReLU()):\n        super(LRUnit1D, self).__init__()\n        self.model = nn.Sequential(\n            nn.Linear(in_dim, in_dim, bias=False),\n            nn.BatchNorm1d(in_dim),\n            nn.Linear(in_dim, in_dim, bias=False),\n            nn.BatchNorm1d(in_dim),\n        )\n        \n        self.actf = actf\n        self.stochastic_depth = StochasticDepth(stochastic_depth_prob, \"row\")\n    def forward(self, x):\n        h = self.model[0](x)\n        h = self.model[1](h)\n        h = self.actf(h)\n        h = self.model[2](h)\n        h = self.model[3](h)\n        h = self.stochastic_depth(h)\n        output = x + h\n        return output\n\n\nclass RSUnit1D(nn.Module):\n    def __init__(self, in_dim, kernel_size=3, padding=1,\n                 padding_mode='zeros', actf=torch.nn.ReLU()):\n        super(RSUnit1D, self).__init__()\n        self.model = nn.Sequential(\n            nn.Conv1d(in_dim, in_dim, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode, bias=False),\n            nn.BatchNorm1d(in_dim),\n            actf,\n            nn.Conv1d(in_dim, in_dim, kernel_size=kernel_size, padding=padding, padding_mode=padding_mode, bias=False),\n            nn.BatchNorm1d(in_dim)\n        )\n        \n        self.actf = actf\n    def forward(self, x):\n        h = self.model[0](x)\n        h = self.model[1](h)\n        h = self.actf(h)\n        h = self.model[2](h)\n        h = self.model[3](h)\n        output = x + h\n        return output\n\nclass BackboneFixedLen(nn.Module):\n    def __init__(self,\n                 in_dim=42,\n                 dim1=128,\n                 dim2=512,\n                 kernel_size=3, padding=1,\n                 negative_slope=0.1,\n                 ):\n        super(BackboneFixedLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.bn0 = nn.BatchNorm1d(in_dim)\n        self.conv1 = nn.Conv1d(in_dim, dim1, kernel_size=3, padding=1, bias=True)\n        self.conv2 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv3 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv4 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv5 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv6 = RSUnit1D(dim1, kernel_size=kernel_size, padding=padding, actf=self.actf)\n        self.conv6a = nn.Conv1d(dim1, dim2, kernel_size=1, padding=0, bias=True)\n\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n    def forward(self, x):\n        h = self.bn0(x)\n        h = self.conv1(h)\n        h = self.conv2(h)\n        h = self.pool(h)\n        h = self.conv3(h)\n        h = self.pool(h)\n        h = self.conv4(h)\n        h = self.pool(h)\n        h = self.conv5(h)\n        h = self.conv6(h)\n        h = self.conv6a(h)\n        h = self.g_pool(h)\n        h = self.flatten(h)\n        return h\n\n\nclass ModelFixedLen(nn.Module):\n    def __init__(self,\n                 in_dim_H=42,\n                 in_dim_L=80,\n                 dim1_H=128,\n                 dim1_L=64,\n                 dim2=512,\n                 n_class=250,\n                 kernel_size=3, padding=1,\n                 arc_m=0.5,\n                 arc_s=15,\n                 easy_margin=False,\n                 stochastic_depth_prob=0.0,\n                 negative_slope=0.1,\n                 n_recyile=1\n                 ):\n        super(ModelFixedLen, self).__init__()\n        self.actf = torch.nn.LeakyReLU(negative_slope=negative_slope)\n        self.hand_module = BackboneFixedLen(in_dim=in_dim_H, dim1=dim1_H, dim2=dim2, negative_slope=negative_slope)\n        self.other_module = BackboneFixedLen(in_dim=in_dim_L, dim1=dim1_L, dim2=dim2, negative_slope=negative_slope)\n\n        self.line1 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line2 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.line3 = LRUnit1D(dim2, stochastic_depth_prob, actf=self.actf)\n        self.end = lossfunc(dim2, n_class, easy_margin=easy_margin, s=arc_s, m=arc_m)\n        self.flatten = torch.nn.Flatten(1)\n        self.g_pool = torch.nn.AdaptiveMaxPool1d(1, return_indices=False)\n        self.pool = torch.nn.MaxPool1d(2)\n\n        self.n_recyile = n_recyile\n\n    def forward(self, hand, other):\n        hand = self.hand_module(hand)\n        other = self.other_module(other)\n        h = hand + other\n        for n in range(self.n_recyile):\n            h = self.line1(h)\n        for n in range(self.n_recyile):\n            h = self.line2(h)\n        for n in range(self.n_recyile):\n            h = self.line3(h)\n        h = self.end(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-07-12T08:31:52.500992Z","iopub.execute_input":"2023-07-12T08:31:52.501420Z","iopub.status.idle":"2023-07-12T08:31:52.538666Z","shell.execute_reply.started":"2023-07-12T08:31:52.501384Z","shell.execute_reply":"2023-07-12T08:31:52.537620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### dataset","metadata":{}},{"cell_type":"code","source":"class AffineMatTools():\n    def __init__(self):\n        self.A_list = []\n\n    def composit(self):\n        M = np.eye(3)\n        for A in self.A_list:\n            M = A.dot(M)\n        M = M[:2]\n        return M\n\n    def composit3(self):\n        M = np.eye(3)\n        for A in self.A_list:\n            M = A.dot(M)\n        return M\n\n    def adjust_composit(self, w, h):\n        M = self.composit()\n        X = np.array(((0, 0, 1), (w - 1, 0, 1), (0, h - 1, 1), (w - 1, h - 1, 1))).T\n        Z = M.dot(X)\n        min_x, min_y = Z.min(axis=1)\n        max_x, max_y = Z.max(axis=1)\n        # print(max_x , min_x)\n        # print(max_y , min_y)\n        new_w = round(max_x - min_x + 1)\n        new_h = round(max_y - min_y + 1)\n        self.shift(-min_x, -min_y)\n        M = self.composit()\n        return M, new_w, new_h\n\n    def scale(self, x, y=None):\n        if y is None:\n            y = x\n        A = np.eye(3)\n        A[0, 0] = x\n        A[1, 1] = y\n        self.A_list.append(A)\n\n    def shift(self, x, y):\n        A = np.eye(3)\n        A[0, 2] = x\n        A[1, 2] = y\n        self.A_list.append(A)\n\n    def rotation_radian(self, r):\n        A = np.eye(3)\n        A[0, 0] = math.cos(r)\n        A[1, 1] = math.cos(r)\n        A[0, 1] = -math.sin(r)\n        A[1, 0] = math.sin(r)\n        self.A_list.append(A)\n\n    def rotation_degree(self, d):\n        r = d * (2 * math.pi) / 360\n        self.rotation_radian(r)\n    def transform(self, data):\n        data = data.copy()\n        shape = data.shape\n        data = data.reshape((-1, 2))\n        data = np.concatenate((data, np.ones((data.shape[0], 1))), axis=1)\n        M = self.composit3()\n        data = data @ M.T\n        data = (data / data[:, 2][:, None])[:, :2]\n        data = data.reshape(shape)\n        return data\nclass AslDataset(Dataset):\n\n    print(\"class init\")\n    feature_keys = ['r_hand',           # 21\n                    'lipsUpperOuter',   #\n                    'lipsLowerOuter',\n                    'lipsUpperInner',\n                    'lipsLowerInner']\n\n    def __init__(self,\n                 df,\n                 # data_list,\n                 aug_hand_param=None,\n                 frame_drop=0.0):\n        self.df = df\n        # self.data_list = data_list\n        self.aug_hand_param = aug_hand_param\n        self.frame_drop = frame_drop\n\n    def __len__(self):\n        return len(self.df)\n        # return len(self.data_list)\n    \n    def apply_aug_hand(self, hand, aug_hand_param, debug=False):\n        angle = random.gauss(0, aug_hand_param[\"angle\"] / 2)\n        scale = random.gauss(1, aug_hand_param[\"scale\"] / 2)\n        shift_x = random.gauss(0, aug_hand_param[\"shift_x\"] / 2)\n        shift_y = random.gauss(0, aug_hand_param[\"shift_y\"] / 2)\n\n        amt = AffineMatTools()\n        amt.scale(scale)\n        amt.rotation_degree(angle)\n        amt.shift(shift_x, shift_y)\n\n        aug_hand = hand - hand[:, 0][:, None]\n\n        aug_hand_z = aug_hand[:, :, 2][:, :, None]\n        aug_hand = aug_hand[:, :, :2]\n        aug_hand = amt.transform(aug_hand)\n        aug_hand = np.concatenate((aug_hand, aug_hand_z), axis=2)\n        aug_hand = aug_hand + hand[:, 0][:, None]\n\n        aug_hand = aug_hand.astype(np.float32)\n        return aug_hand\n\n    def mirrored(self, data_dict):\n        mirrored_dict = {}\n\n        def invert_x(tmp):\n            tmp = tmp.copy()\n            tmp[:, :, 0] = -tmp[:, :, 0]\n            return tmp\n        mirrored_dict[\"r_hand\"] = invert_x(data_dict[\"l_hand\"])\n        mirrored_dict[\"l_hand\"] = invert_x(data_dict[\"r_hand\"])\n        mirrored_dict[\"lipsUpperOuter\"] = invert_x(data_dict[\"lipsUpperOuter\"][:, ::-1])\n        mirrored_dict[\"lipsLowerOuter\"] = invert_x(data_dict[\"lipsLowerOuter\"][:, ::-1])\n        mirrored_dict[\"lipsUpperInner\"] = invert_x(data_dict[\"lipsUpperInner\"][:, ::-1])\n        mirrored_dict[\"lipsLowerInner\"] = invert_x(data_dict[\"lipsLowerInner\"][:, ::-1])\n        return mirrored_dict\n\n    def apply_frame_drop(self, data, frame_drop):\n        # Drop frames at random percentage frame_drop\n        if frame_drop > 0.0:\n            drop_mask = np.random.random(len(data)) >= frame_drop\n            dropped_data = data[drop_mask]\n\n            if len(dropped_data) >= 2:\n                data = dropped_data\n        return data\n    def get_data(self,\n                 data_dict,\n                 aug_hand_param=None,\n                 frame_drop=0.0):\n\n        # data_dict = data_dict.copy()\n\n        # Compare the number of frames and flip horizontally\n        length = data_dict[\"l_hand\"].shape[0]\n        l_num = length - np.isnan(data_dict[\"l_hand\"][:, 0, 0]).sum()\n        r_num = length - np.isnan(data_dict[\"r_hand\"][:, 0, 0]).sum()\n        if r_num < l_num:\n            data_dict = self.mirrored(data_dict)\n\n        # augumentation of hand\n        if aug_hand_param:\n            data_dict[\"r_hand\"] = self.apply_aug_hand(data_dict[\"r_hand\"], aug_hand_param)\n\n        # concat data\n        data = np.concatenate([data_dict[key] for key in AslDataset.feature_keys], axis=1)\n\n        # Delete frames randomly\n        data = self.apply_frame_drop(data, frame_drop)\n\n        # ignore z\n        data = data[:, :, :2]\n\n        x = torch.tensor(data)  # TxPx2\n\n        hand_mask = ~torch.isnan(x[:, 0, 0])  # T\n\n        # normalization\n        x = x - x[~torch.isnan(x)].mean(0, keepdims=True)\n        x = x / x[~torch.isnan(x)].std(0, keepdims=True)\n\n        x[torch.isnan(x)] = 0.0  # TxPx2\n        x = torch.reshape(x, (x.shape[0], -1))\n        x = torch.permute(x, (1, 0))  # 2P x T\n\n        # extract only where there is a hand, return all if there is no hand\n        if hand_mask.sum() > 0:\n            hand = x[:42, hand_mask]\n        else:\n            hand = x[:42]\n        other = x[42:]\n\n        return hand, other    \n    def __getitem__(self, index):\n        path = os.path.join(Project.preproccessed_data, str(self.df.iloc[index][\"sequence_id\"]) + \".pickle\")\n        data_dict, y = joblib.load(path)\n        x0, x1 = self.get_data(data_dict,\n                               aug_hand_param=self.aug_hand_param,\n                               frame_drop=self.frame_drop)\n        y = torch.tensor(y)\n        return x0, x1, y\n\n\ndef preproccesing_data_dict(row):\n    \n    data_columns = ['x', 'y', 'z']\n    data = pd.read_parquet(row[0], columns=data_columns)\n    n_frames = int(len(data) / ROWS_PER_FRAME)\n    x = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns)).astype(np.float32)\n    # 根据眉毛位置进行标准化\n    midwayBetweenEyes = x[:, 168]\n    m = np.nanmean(midwayBetweenEyes, axis=0, keepdims=True)\n    x = x - m\n\n    # 提取手部、嘴唇上下外侧、嘴唇上下内侧等关键点数据\n    l_hand = x[:, l_hand_indexes]  # 左手\n    r_hand = x[:, r_hand_indexes]  # 右手\n    lipsUpperOuter = x[:, face_keypoints[\"lipsUpperOuter\"]]  # 嘴唇上部外侧\n    lipsLowerOuter = x[:, face_keypoints[\"lipsLowerOuter\"]]  # 嘴唇下部外侧\n    lipsUpperInner = x[:, face_keypoints[\"lipsUpperInner\"]]  # 嘴唇上部内侧\n    lipsLowerInner = x[:, face_keypoints[\"lipsLowerInner\"]]  # 嘴唇下部内侧\n\n    data_dict = dict(l_hand=l_hand,\n                     r_hand=r_hand,\n                     lipsUpperOuter=lipsUpperOuter,\n                     lipsLowerOuter=lipsLowerOuter,\n                     lipsUpperInner=lipsUpperInner,\n                     lipsLowerInner=lipsLowerInner)\n\n    y = row[1]\n    name = os.path.basename(row[0]).replace(\".parquet\", \".pickle\")\n    # name = str(row[0].sequence_id) + \".pickle\"\n    data_path = os.path.join(Project.preproccessed_data, name)\n    joblib.dump((data_dict, y), data_path)\nprint(1)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.826985Z","iopub.execute_input":"2023-07-12T06:29:44.827323Z","iopub.status.idle":"2023-07-12T06:29:44.871182Z","shell.execute_reply.started":"2023-07-12T06:29:44.827292Z","shell.execute_reply":"2023-07-12T06:29:44.870343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train utilities","metadata":{}},{"cell_type":"code","source":"\ndef train(model, loader, optimizer,\n          scheduler=None,\n          loss_func=nn.CrossEntropyLoss(ignore_index=-1),\n          device=\"cuda\"):\n\n    model.train()\n    model.to(device)\n\n    mean_loss = 0\n    n = 0\n    #for (data1, data2, target) in tqdm(loader):\n    for (data1, data2, target) in (loader):\n        data1 = data1.to(device)\n        data2 = data2.to(device)\n        target = target.to(device)\n        optimizer.zero_grad()\n        pred = model(data1, data2)\n        loss = 0.1 * loss_func(pred, target)\n        loss.backward()\n        optimizer.step()\n\n        loss = loss.detach().cpu().numpy()\n\n        mean_loss += loss * len(data1)\n\n        n += len(data1)\n\n        if scheduler:\n            scheduler.step()\n    optimizer.zero_grad()\n\n    mean_loss = mean_loss / n\n    return mean_loss\n\n\ndef pack_seq_fixed_len(seq, length):\n\n    batch_size = len(seq)\n    num_landmark = seq[0].shape[0]\n    x = torch.zeros((batch_size, num_landmark, length))\n    for b in range(batch_size):\n        data = seq[b]\n        data = torch.unsqueeze(data, 0)\n        data = torch.unsqueeze(data, 0)\n        data = torch.nn.functional.interpolate(data, size=(data.shape[2], length))\n        x[b] = data[0, 0]\n    return x\n\n\nclass CollateFnFixedLen():\n    def __init__(self, mm=96, ss=0, aa=64, bb=128):\n        super().__init__()\n        self.mm = mm\n        self.ss = ss\n        self.aa = aa\n        self.bb = bb\n\n    def __call__(self, batch):\n        if self.ss > 0:\n            length = int(np.round(truncnorm.rvs((self.aa - self.mm) / self.ss,\n                                                (self.bb - self.mm) / self.ss,\n                                                loc=self.mm, scale=self.ss)))\n        else:\n            length = self.mm\n\n        hand, lips, targets = list(zip(*batch))\n        images1 = pack_seq_fixed_len(hand, length)\n        images2 = pack_seq_fixed_len(lips, length)\n        targets = torch.stack(targets)\n        return images1, images2, targets\n\n\ndef lr_func(epoch):\n    if epoch < 1:\n        return 0.01\n    elif epoch < 2:\n        return 0.1\n    elif epoch < 6:\n        return 1\n    elif epoch < 10:\n        return 0.2\n    else:\n        return 0.2**2\n\n\n@torch.no_grad()\ndef renew_bn(loader, model, device=None):\n  \n    momenta = {}\n    for module in model.modules():\n        if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):\n            module.running_mean = torch.zeros_like(module.running_mean)\n            module.running_var = torch.ones_like(module.running_var)\n            momenta[module] = module.momentum\n\n    if not momenta:\n        return\n\n    was_training = model.training\n    model.train()\n    for module in momenta.keys():\n        module.momentum = None\n        module.num_batches_tracked *= 0\n\n    for input0, input1, target in tqdm(loader):\n        if device:\n            input0 = input0.to(device)\n            input1 = input1.to(device)\n        model(input0, input1)\n\n    for bn_module in momenta.keys():\n        bn_module.momentum = momenta[bn_module]\n    model.train(was_training)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.874393Z","iopub.execute_input":"2023-07-12T06:29:44.874662Z","iopub.status.idle":"2023-07-12T06:29:44.894803Z","shell.execute_reply.started":"2023-07-12T06:29:44.874640Z","shell.execute_reply":"2023-07-12T06:29:44.893795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef get_loader(Project, df, i_fold):\n\n    if i_fold == -1:  # for make submit model\n\n        train_df = df.copy()\n        #train_df = train_df[~ train_df.is_noise].reset_index(drop=True)\n\n        T1 = AslDataset(train_df,\n                        aug_hand_param=Project.aug_hand_param,\n                        frame_drop=Project.frame_drop,\n                        )\n        I1 = DataLoader(T1,\n                        batch_size=Project.batchsize1,\n                        num_workers=Project.num_workers,\n                        shuffle=True,\n                        drop_last=True,\n                        collate_fn=Project.collate_fn1)\n        return I1\n    \n\ndef get_models(Project):\n\n    model = Project.Model(in_dim_H=Project.in_dim_H,\n                      in_dim_L=Project.in_dim_L,\n                      dim1_H=Project.dim1_H,\n                      dim1_L=Project.dim1_L,\n                      dim2=Project.dim2,\n                      easy_margin=Project.easy_margin,\n                      arc_s=Project.arc_s,\n                      arc_m=Project.arc_m,\n                      stochastic_depth_prob=Project.stochastic_depth_prob,\n                      n_recyile=Project.n_recyile\n                      )\n    model.to(Project.device)\n    swa_model = torch.optim.swa_utils.AveragedModel(model)\n    swa_model.to(Project.device)\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=Project.lr, weight_decay=Project.weight_decay)\n    org_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_func)\n    swa_scheduler = torch.optim.swa_utils.SWALR(optimizer, swa_lr=Project.swa_lr)\n\n    return model, swa_model, optimizer, org_scheduler, swa_scheduler\n\n\ndef train_submit_model(Project, df):\n\n    #fix_seed(CFG.seed)\n    os.environ['PYTHONHASHSEED'] = str(Project.seed)\n    # random\n    random.seed(Project.seed)\n    # Numpy\n    np.random.seed(Project.seed)\n    # Pytorch\n    torch.manual_seed(Project.seed)\n    torch.cuda.manual_seed_all(Project.seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n    os.makedirs(Project.result_path, exist_ok=True)\n\n    loss_func1 = torch.nn.CrossEntropyLoss(label_smoothing=Project.label_smoothing)\n\n    I1 = get_loader(Project, df, i_fold=-1)\n\n    model, swa_model, optimizer, org_scheduler, swa_scheduler = get_models(Project)\n    print(model)\n    for epoch in tqdm(range(1,Project.EPOCHS + 1)):\n\n        if epoch < Project.swa_start:\n            tr_loss = train(model, I1, optimizer, loss_func=loss_func1)\n            org_scheduler.step()\n            log = f\"{epoch:3d} {tr_loss:1.5f}\"\n            print(log)\n            with open(os.path.join(Project.result_path, \"log.txt\"), \"a\") as f:\n                f.write(log + \"\\n\")\n        else:\n            tr_loss = train(model, I1, optimizer, loss_func=loss_func1)\n            swa_model.update_parameters(model)\n            swa_scheduler.step()\n\n    renew_bn(I1, swa_model, device=Project.device)\n    swa_model_path = os.path.join(Project.result_path, f\"swa_{epoch}.pt\")\n    model.eval()\n    torch.save(swa_model.state_dict(), swa_model_path)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.896408Z","iopub.execute_input":"2023-07-12T06:29:44.896820Z","iopub.status.idle":"2023-07-12T06:29:44.913479Z","shell.execute_reply.started":"2023-07-12T06:29:44.896790Z","shell.execute_reply":"2023-07-12T06:29:44.912614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Please select experimtal condition\n+ https://www.kaggle.com/competitions/asl-signs/discussion/406673\n\n","metadata":{}},{"cell_type":"code","source":"\nexperimet_mode = \"A0\"\n\nclass Project:\n    output_path = '../working/'\n    preproccessed_data = '../preproccessed_data/'\n    n_fold = 3\n    topK = 5\n    EPOCHS = 30 # Originally 300 epochs, but reduced to 30.\n    batchsize1 = 32\n    batchsize2 = 64\n    device = \"cuda\"\n\n    num_workers = 2\n    lr = 0.01\n    weight_decay = 0.0005\n    swa_lr = 0.01\n    swa_start = 16\n\n    frame_drop = 0.3\n    label_smoothing = 0.5\n    arc_s = 47.0\n    arc_m = 0.0\n    easy_margin = True\n\n    aug_hand_param = {}\n    aug_hand_param[\"angle\"] = 7.0\n    aug_hand_param[\"scale\"] = 0.10\n    aug_hand_param[\"shift_x\"] = 0.016\n    aug_hand_param[\"shift_y\"] = 0.053\n\n    in_dim_H = 42\n    in_dim_L = 80\n\n    dim1_H = 128\n    dim1_L = 64\n    dim2 = 512\n\n    SEQ_LEN = 96\n    save_epochs = (20, 50, 100, 200, 300)\n\n    seed = 0\n    use_cleanlab = False\n    Model = ModelFixedLen\n    n_recyile = 3\n    stochastic_depth_prob = 0.5\n    collate_fn1 = CollateFnFixedLen(mm=SEQ_LEN, ss=32, aa=SEQ_LEN - 32, bb=SEQ_LEN + 32)\n    collate_fn2 = CollateFnFixedLen(mm=SEQ_LEN, ss=0)\n    exp_name = f\"ModelFixedLen_deep_seed{seed}_cleanlab_{use_cleanlab}\"\n\n    result_path = os.path.join(\"../working/\", exp_name)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.914915Z","iopub.execute_input":"2023-07-12T06:29:44.915482Z","iopub.status.idle":"2023-07-12T06:29:44.933406Z","shell.execute_reply.started":"2023-07-12T06:29:44.915451Z","shell.execute_reply":"2023-07-12T06:29:44.932436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"DEBUG=False\n\nINPUT_PATH = \"../input/asl-signs/\"\nLANDMARK_FILES_DIR = os.path.join(INPUT_PATH, \"train_landmark_files\")\nTRAIN_FILE = os.path.join(INPUT_PATH, \"train.csv\")\nJSON_FILE = os.path.join(INPUT_PATH, \"sign_to_prediction_index_map.json\")\npred_col_names = [f\"pred{c:03}\" for c in range(n_class)]\n\n\ndf = data_load(group=\"participant_id\", n_fold=Project.n_fold)\n\nif DEBUG:\n    df=df[::100].reset_index(drop=True)\n    Project.EPOCHS = 20\n    \n#df = apply_cleanlab(CFG, df)\nprint(1)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:44.937000Z","iopub.execute_input":"2023-07-12T06:29:44.937376Z","iopub.status.idle":"2023-07-12T06:29:45.215274Z","shell.execute_reply.started":"2023-07-12T06:29:44.937343Z","shell.execute_reply":"2023-07-12T06:29:45.214258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = [(os.path.join(INPUT_PATH, row.path), row.label) for row in df.itertuples()]\nos.makedirs(Project.preproccessed_data, exist_ok=True)\nwith multiprocessing.Pool(multiprocessing.cpu_count()) as pool:\n    imap = pool.imap(preproccesing_data_dict, rows)\n    data_list = list(tqdm(imap, total=len(rows)))\ndel data_list\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:29:45.216787Z","iopub.execute_input":"2023-07-12T06:29:45.217188Z","iopub.status.idle":"2023-07-12T06:46:29.049762Z","shell.execute_reply.started":"2023-07-12T06:29:45.217152Z","shell.execute_reply":"2023-07-12T06:46:29.048481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Execute a 3fold validation.\n### It will probably take about 24-30 hours.","metadata":{}},{"cell_type":"code","source":"# %% for 3 fold validation\n# validation(CFG, df)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:46:29.054086Z","iopub.execute_input":"2023-07-12T06:46:29.055864Z","iopub.status.idle":"2023-07-12T06:46:29.061298Z","shell.execute_reply.started":"2023-07-12T06:46:29.055820Z","shell.execute_reply":"2023-07-12T06:46:29.060425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create one of the models for submitting.\n### It will probably take 10 hours at 300 epoch. ","metadata":{}},{"cell_type":"code","source":"train_submit_model(Project, df)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T06:46:29.063743Z","iopub.execute_input":"2023-07-12T06:46:29.064097Z","iopub.status.idle":"2023-07-12T08:25:19.871260Z","shell.execute_reply.started":"2023-07-12T06:46:29.064063Z","shell.execute_reply":"2023-07-12T08:25:19.869141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}