{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Introduction\nThis is notebook for submission. The discussion and introduction can be found on the first notebook
([Multi-head CNN for multi-label classification-pt1](https://www.kaggle.com/yunchenyang/multi-head-cnn-for-multi-label-classification-pt1))\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Source code\nThe codes are also available on [GitHub](https://github.com/yunchen-yang/artcv).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"utils","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nfrom torchvision.transforms import Compose, Resize, RandomResizedCrop, Normalize, ToTensor\nimport pandas as pd\nimport torch\nfrom sklearn.metrics import fbeta_score\n\n\ndef label_indexer_fine(labels_dataframe):\n split_labels_dict = dict(attribute_id=[], attr_tier1=[], attr_tier2=[], attr_tier3=[])\n for i in range(labels_dataframe.shape[0]):\n tem = [item.strip() for split_list in [item_.split(';')\n for item_ in labels_dataframe['attribute_name'][i].split('::')]\n for item in split_list]\n split_labels_dict['attribute_id'].append(labels_dataframe['attribute_id'][i])\n split_labels_dict['attr_tier1'].append(tem[0])\n split_labels_dict['attr_tier2'].append(tem[1])\n try:\n split_labels_dict['attr_tier3'].append(tem[2])\n except:\n split_labels_dict['attr_tier3'].append('None')\n split_labels = pd.DataFrame(split_labels_dict,\n columns=['attribute_id', 'attr_tier1', 'attr_tier2', 'attr_tier3'])\n\n tier1 = dict()\n tier2 = dict()\n counting_dict = dict()\n attr2indexing = dict()\n indexing2attr = dict()\n label_indexing_list = []\n for i_1, item1 in enumerate(sorted(list(set(list(split_labels['attr_tier1']))))):\n tier1[item1] = i_1\n tier2[item1] = dict()\n list_tem = sorted(list(set(list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1]))))\n for i_2, item2 in enumerate(list_tem):\n tier2[item1][item2] = i_2 + 1\n counting_dict[item1] = np.ones(len(list_tem), dtype='int')\n for idx in range(split_labels.shape[0]):\n tier1_idx = tier1[split_labels['attr_tier1'][idx]]\n tier2_idx = tier2[split_labels['attr_tier1'][idx]][split_labels['attr_tier2'][idx]]\n tier3_idx = counting_dict[split_labels['attr_tier1'][idx]][tier2_idx - 1]\n counting_dict[split_labels['attr_tier1'][idx]][tier2_idx - 1] += 1\n label_indexing_list.append([tier1_idx, tier2_idx, tier3_idx])\n attr2indexing[split_labels['attribute_id'][idx]] = [tier1_idx, tier2_idx, tier3_idx]\n indexing2attr[str([tier1_idx, tier2_idx, tier3_idx])] = split_labels['attribute_id'][idx]\n labels_indexing_df = split_labels.copy()\n labels_indexing_df['indexing'] = label_indexing_list\n return labels_indexing_df, attr2indexing, indexing2attr\n\n\ndef label_indexer_coarse(labels_dataframe):\n split_labels_dict = dict(attribute_id=[], attr_tier1=[], attr_tier2=[])\n for i in range(labels_dataframe.shape[0]):\n tem = [item.strip() for item in labels_dataframe['attribute_name'][i].split('::')]\n split_labels_dict['attribute_id'].append(labels_dataframe['attribute_id'][i])\n split_labels_dict['attr_tier1'].append(tem[0])\n split_labels_dict['attr_tier2'].append(tem[1])\n\n split_labels = pd.DataFrame(split_labels_dict,\n columns=['attribute_id', 'attr_tier1', 'attr_tier2'])\n\n tier1 = dict()\n tier2 = dict()\n attr2indexing = dict()\n indexing2attr = dict()\n label_indexing_list = []\n for i_1, item1 in enumerate(sorted(list(set(list(split_labels['attr_tier1']))))):\n assert len(list(set(list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1])))) \\\n == len(list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1]))\n tier1[item1] = i_1\n tier2[item1] = dict()\n list_tem = list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1])\n for i_2, item2 in enumerate(list_tem):\n tier2[item1][item2] = i_2\n for idx in range(split_labels.shape[0]):\n tier1_idx = tier1[split_labels['attr_tier1'][idx]]\n tier2_idx = tier2[split_labels['attr_tier1'][idx]][split_labels['attr_tier2'][idx]]\n label_indexing_list.append([tier1_idx, tier2_idx])\n attr2indexing[split_labels['attribute_id'][idx]] = [tier1_idx, tier2_idx]\n indexing2attr[str([tier1_idx, tier2_idx])] = split_labels['attribute_id'][idx]\n labels_indexing_df = split_labels.copy()\n labels_indexing_df['indexing'] = label_indexing_list\n return labels_indexing_df, attr2indexing, indexing2attr\n\n\ndef imgreader(img_id, ext, path, attr_ids, attr2indexing, length_list, dimension=256,\n task=('ml', 'ml', 'mc', 'ml', 'ml'), transform='val', grey_scale=False):\n file_path = f'{path}/{img_id}.{ext}'\n with open(file_path, 'rb') as f:\n img_ = Image.open(f)\n if grey_scale:\n img = img_.convert('L')\n else:\n img = img_.convert('RGB')\n\n transformer = {\n 'train': Compose([RandomResizedCrop(size=(dimension, dimension)),\n ToTensor(),\n Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]\n ),\n 'val': Compose([Resize(size=(dimension, dimension)),\n ToTensor(),\n Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n }\n x = transformer[transform](img)\n y_list = [attr2indexing[int(attr_id)] for attr_id in attr_ids.split()]\n y_dict = labels_list2array(y_list, length_list, task)\n return x, tuple(y_dict.values())\n\n\ndef imgreader_test(file_path, dimension=256, transform='val', grey_scale=False):\n with open(file_path, 'rb') as f:\n img_ = Image.open(f)\n if grey_scale:\n img = img_.convert('L')\n else:\n img = img_.convert('RGB')\n\n transformer = {\n 'train': Compose([RandomResizedCrop(size=(dimension, dimension)),\n ToTensor(),\n Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]\n ),\n 'val': Compose([Resize(size=(dimension, dimension)),\n ToTensor(),\n Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n }\n x = transformer[transform](img)\n\n return x\n\n\ndef counting_elements(labels_indexing_df):\n return [len(labels_indexing_df['indexing'][labels_indexing_df['attr_tier1']==catagory])\n for catagory in sorted(list(set(list(labels_indexing_df['attr_tier1']))))]\n\n\ndef labels_list2array(y_list, length_list, task):\n y_dict = dict()\n for i in range(len(task)):\n if task[i] != 'mc':\n y_dict[i] = torch.FloatTensor(np.zeros(length_list[i]))\n else:\n y_dict[i] = torch.LongTensor([0])\n\n for idx_list in y_list:\n if task[idx_list[0]] != 'mc':\n y_dict[idx_list[0]][idx_list[1]] = 1\n else:\n y_dict[idx_list[0]][0] = idx_list[1]+1\n\n return y_dict\n\n\ndef image_list_scan(data_info, indices):\n if indices is None:\n return list(data_info['id']), list(data_info['attribute_ids'])\n else:\n return list(data_info['id'][indices]), list(data_info['attribute_ids'][indices])\n\n\ndef f2score(ground_truth, pred, return_mean=True):\n f_beta = [fbeta_score(ground_truth[i,:], pred[i,:], beta=2) for i in range(ground_truth.shape[0])]\n if return_mean:\n return sum(f_beta)/len(f_beta)\n else:\n return f_beta\n\n \ndef regularized_pred(probs, thre, upper_bound=(3, 4, 17, 18), lower_bound=3,\n boundary=([0, 100], [100, 781], [786, 2706], [2706, 3474])):\n thres_array = np.ones((probs.shape[1]), dtype='float')\n pred = dict()\n for i in range(len(boundary)):\n thres_array[boundary[i][0]: boundary[i][1]] = thre[i]\n probs_tem = probs[:, boundary[i][0]: boundary[i][1]]/thre[i]\n mask_tem = np.zeros(probs_tem.shape, dtype='float')\n max_args = probs_tem.argsort(axis=-1)[:,::-1][:, :upper_bound[i]]\n for i_ in range(mask_tem.shape[0]):\n mask_tem[i_, :][max_args[i_, :]] = 1\n probs_tem *= mask_tem\n probs_tem[probs_tem>=1] = 1\n probs_tem[probs_tem<1] = 0\n pred[i] = probs_tem \n pred_array = np.concatenate((pred[0], pred[1], \n probs[:, boundary[1][1]: boundary[2][0]], pred[2], pred[3]), axis=-1)\n no_label = np.where(pred_array.max(axis=-1)==0)[0]\n if no_label.shape[0] != 0:\n for idx in no_label:\n _max_args = (probs[idx, :]/thres_array).argsort(axis=-1)[::-1][:lower_bound]\n pred_array[idx, :][_max_args] = 1\n return pred_array","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"datatool","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nimport sys\nimport math\nfrom glob import glob\n\n\nclass ImgDataset(Dataset):\n def __init__(self, x, y, path, attr2indexing, length_list, task,\n ext='png', dimension=256, transform='val', grey_scale=False):\n super().__init__()\n self.x = x\n self.y = y\n self.path = path\n self.attr2indexing =attr2indexing\n self.length_list = length_list\n self.task = task\n self.ext = ext\n self.dimension = dimension\n self.transform = transform\n self.grey_scale = grey_scale\n\n def __len__(self):\n return len(self.x)\n\n def __getitem__(self, index):\n img, ys = imgreader(self.x[index], self.ext, self.path, self.y[index],\n self.attr2indexing, self.length_list,\n self.dimension, self.task, self.transform, self.grey_scale)\n y0, y1, y2, y3, y4 = ys\n\n return img, y0, y1, y2, y3, y4\n\n\nclass ImgTestset(Dataset):\n def __init__(self, x, dimension=256, transform='val', grey_scale=False):\n super().__init__()\n self.x = x\n\n self.dimension = dimension\n self.transform = transform\n self.grey_scale = grey_scale\n\n def __len__(self):\n return len(self.x)\n\n def __getitem__(self, index):\n img = imgreader_test(self.x[index], self.dimension, self.transform, self.grey_scale)\n\n return img\n\n\nclass TrainValSet:\n def __init__(self, ext='png', path=None, indices=None, dimension=256, data_info_path=None, labels_info_path=None,\n task=('ml', 'ml', 'mc', 'ml', 'ml'), train_transform='train', train_val_split=0.7, seed=0,\n test_path=None, test_csv_path=None):\n super().__init__()\n self.ext = ext\n self.dimension = dimension\n self.indices = indices\n self.seed = seed\n self.train_transform = train_transform\n\n if path is None:\n self.path = f'{sys.path[0]}/train'\n else:\n self.path = path\n\n self.test_path = test_path\n\n if data_info_path is None:\n self.data_info_path = f'{sys.path[0]}/train.csv'\n else:\n self.data_info_path = data_info_path\n\n if labels_info_path is None:\n self.labels_info_path = f'{sys.path[0]}/labels.csv'\n else:\n self.labels_info_path = labels_info_path\n\n self.labels_info = pd.read_csv(self.labels_info_path)\n self.data_info = pd.read_csv(self.data_info_path)\n\n self.labels_indexing_df, self.attr2indexing, self.indexing2attr = label_indexer_coarse(self.labels_info)\n self.length_list = counting_elements(self.labels_indexing_df)\n self.X_all, self.Y_all = image_list_scan(self.data_info, indices=self.indices)\n self.task = task\n\n if self.test_path is not None:\n self.test_csv_path = test_csv_path\n if self.test_csv_path is not None:\n self.test_csv = pd.read_csv(self.test_csv_path)\n self.X_test = [f'{self.test_path}/{_filename}.{self.ext}' for _filename in list(self.test_csv['id'])]\n else:\n self.X_test = glob(f'{self.test_path}/*.{self.ext}', recursive=True)\n self.test = ImgTestset(self.X_test, dimension=256, transform='val', grey_scale=False)\n\n self.train_val_split = train_val_split\n\n if bool(self.train_val_split):\n self.all = ImgDataset(self.X_all, self.Y_all, self.path, self.attr2indexing, self.length_list,\n task=self.task, ext=self.ext, dimension=256, transform='val', grey_scale=False)\n assert(0 < self.train_val_split < 1)\n num_train = math.ceil(len(self.X_all) * self.train_val_split)\n np.random.seed(seed=self.seed)\n indices_array = np.random.permutation(len(self.X_all))\n self.X_train = [self.X_all[i] for i in indices_array[:num_train]]\n self.Y_train = [self.Y_all[i] for i in indices_array[:num_train]]\n self.train = ImgDataset(self.X_train, self.Y_train, self.path, self.attr2indexing, self.length_list,\n task=self.task, ext=self.ext, dimension=256,\n transform=self.train_transform, grey_scale=False)\n self.X_val = [self.X_all[i] for i in indices_array[num_train:]]\n self.Y_val = [self.Y_all[i] for i in indices_array[num_train:]]\n self.val = ImgDataset(self.X_val, self.Y_val, self.path, self.attr2indexing, self.length_list,\n task=self.task, ext=self.ext, dimension=256, transform='val', grey_scale=False)\n else:\n if self.test_path is not None:\n self.all = ImgDataset(self.X_all, self.Y_all, self.path, self.attr2indexing, self.length_list,\n task=self.task, ext=self.ext, dimension=256,\n transform=self.train_transform, grey_scale=False)\n else:\n self.all = ImgDataset(self.X_all, self.Y_all, self.path, self.attr2indexing, self.length_list,\n task=self.task, ext=self.ext, dimension=256, transform='val', grey_scale=False)\n ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"modules","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.models.resnet import ResNet\nimport torch\nfrom torch import nn as nn\nimport collections\nimport torch.nn.functional as F\n\n\nclass ResNet_CNN(ResNet):\n def __init__(self, block, layers, weight_path, freeze_layers, **kwargs):\n super().__init__(block, layers, **kwargs)\n self.weight_path = weight_path\n if type(freeze_layers) == bool and freeze_layers:\n self.freeze_layers = 4\n else:\n self.freeze_layers = freeze_layers\n if self.weight_path is not None:\n self.load_state_dict(torch.load(self.weight_path))\n del self.fc\n if bool(self.freeze_layers):\n self._freeze_layers()\n\n def forward(self, x):\n x = self.conv1(x)\n x = self.bn1(x)\n x = self.relu(x)\n x = self.maxpool(x)\n\n x = self.layer1(x) if self.freeze_layers != 1 else self.layer1(x).detach()\n x = self.layer2(x) if self.freeze_layers != 2 else self.layer2(x).detach()\n x = self.layer3(x) if self.freeze_layers != 3 else self.layer3(x).detach()\n x = self.layer4(x)\n\n x = self.avgpool(x)\n x = torch.flatten(x, 1)\n\n if self.freeze_layers != 4:\n return x\n else:\n return x.detach()\n\n def _freeze_layers(self):\n _bool = False\n for name, module in self.named_children():\n if name == f'layer{self.freeze_layers+1}' or _bool:\n _bool = True\n for p in module.parameters():\n p.requires_grad = _bool\n\n\nclass Classifier(nn.Module):\n def __init__(self, dim_in, dim_out, dim_hidden, n_layers, task='ml', use_batch_norm=True, dropout_rate=0.01):\n super().__init__()\n dims = [dim_in] + [dim_hidden]*(n_layers-1) + [dim_out]\n self.task = task\n self.use_batch_norm = use_batch_norm\n self.dropout_rate = dropout_rate\n self.classifier = nn.Sequential(collections.OrderedDict(\n [('Layer {}'.format(i), nn.Sequential(\n nn.Linear(n_in, n_out),\n nn.BatchNorm1d(n_out, momentum=.01, eps=0.001) if self.use_batch_norm else None,\n nn.ReLU() if i < len(dims)-2 else None,\n nn.Dropout(p=self.dropout_rate) if self.dropout_rate > 0 else None))\n for i, (n_in, n_out) in enumerate(zip(dims[:-1], dims[1:]))]))\n\n def get_logits(self, x):\n for layers in self.classifier:\n for layer in layers:\n if layer is not None:\n x = layer(x)\n return x\n\n def forward(self, x):\n if self.task == 'mc':\n return F.softmax(self.get_logits(x), dim=-1)\n elif self.task == 'ml':\n return torch.sigmoid(self.get_logits(x))\n else:\n raise ValueError(\"The task tag must be either 'ml' (multi-label) or 'mc' (multi-class)!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"groups information","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"_label_groups0 = None\n_label_groups1 = (3, 6, 3, 4, 7, 7, 11, 11, 11, 11, 11, 11, 11, 2, 4, 9, 9, 5, 3, 11, 11, 11, 11, 11, 11, 2, 12, 13, 7, 10, 12, \n 5, 3, 0, 10, 8, 5, 2, 7, 1, 4, 10, 13, 11, 12, 12, 12, 5, 11, 4, 10, 3, 9, 0, 4, 10, 10, 13, 14, 5, 1, 6, 9, \n 12, 12, 8, 3, 5, 12, 8, 1, 9, 10, 3, 3, 11, 3, 3, 9, 11, 6, 11, 11, 11, 11, 11, 9, 12, 9, 5, 8, 7, 8, 10, 6, 7, \n 4, 11, 13, 3, 5, 14, 5, 10, 10, 13, 13, 13, 13, 11, 9, 3, 9, 9, 11, 7, 6, 7, 11, 11, 11, 11, 8, 8, 0, 8, 6, 6, \n 8, 14, 13, 13, 2, 13, 12, 5, 4, 13, 14, 14, 4, 3, 10, 4, 4, 5, 10, 5, 5, 1, 12, 9, 9, 13, 2, 12, 9, 11, 11, 13, \n 13, 13, 13, 13, 13, 14, 8, 7, 9, 9, 2, 7, 4, 4, 10, 10, 10, 11, 11, 11, 13, 8, 11, 3, 3, 5, 11, 3, 3, 13, 11, \n 11, 11, 6, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 5, 14, 4, 4, 12, 11, 6, 11, 11, 11, 11, 11, 11, 11, \n 8, 10, 2, 12, 3, 5, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 13, 10, 10, 10, 10, 9, 14, 14, 14, 8, 8, 0, 8, 9, \n 9, 3, 13, 10, 4, 5, 6, 9, 10, 2, 11, 10, 0, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 6, 6, 6, 7, 0, 0, 7, 6, \n 11, 6, 10, 7, 4, 12, 11, 6, 6, 6, 6, 11, 11, 11, 11, 11, 11, 11, 11, 13, 11, 10, 7, 6, 4, 6, 13, 7, 2, 6, 7, \n 14, 6, 9, 10, 10, 3, 3, 3, 12, 12, 9, 3, 9, 11, 9, 9, 9, 4, 9, 8, 5, 11, 6, 8, 8, 2, 11, 7, 8, 11, 4, 11, 4, \n 14, 14, 3, 4, 4, 6, 4, 7, 6, 2, 5, 12, 2, 3, 12, 4, 1, 3, 3, 3, 9, 8, 11, 6, 6, 3, 14, 10, 0, 7, 6, 12, 3, 11, \n 13, 6, 13, 13, 13, 13, 13, 13, 13, 9, 13, 13, 13, 13, 13, 5, 12, 12, 13, 0, 13, 13, 0, 10, 10, 12, 3, 7, 6, 6, \n 3, 5, 12, 4, 10, 9, 5, 14, 14, 14, 14, 14, 0, 0, 0, 3, 7, 0, 4, 9, 11, 4, 5, 12, 12, 12, 6, 13, 4, 13, 0, 14, \n 1, 8, 14, 10, 5, 3, 11, 3, 11, 11, 11, 12, 14, 4, 11, 7, 3, 8, 5, 4, 10, 3, 11, 9, 0, 3, 3, 8, 9, 12, 13, 13, \n 13, 13, 13, 6, 13, 11, 13, 1, 13, 13, 13, 11, 13, 13, 14, 11, 11, 11, 4, 7, 9, 9, 12, 9, 3, 12, 2, 8, 8, 0, 9, \n 5, 11, 11, 12, 5, 5, 11, 11, 1, 2, 4, 7, 6, 7, 1, 1, 1, 12, 4, 8, 6, 7, 6, 11, 0, 6, 2, 3, 4, 5, 3, 3, 1, 3, \n 13, 13, 3, 3, 8, 14, 3, 3, 3, 6, 11, 11, 11, 11, 14, 3, 3, 12, 7, 6, 8, 3, 1, 4, 13, 13, 13, 9, 9, 2, 12, 15, \n 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, \n 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, \n 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, \n 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15)\n\n_label_groups3_40 = (27, 19, 17, 17, 7, 23, 8, 26, 23, 30, 18, 18, 18, 16, 16, 27, 16, 27, 34, 14, 14, 27, 2, 24, 27,\n 8, 31, 0, 9, 9, 27, 27, 34, 27, 27, 38, 32, 1, 1, 27, 20, 27, 20, 17, 33, 29, 8, 31, 36, 9, 25,\n 32, 21, 20, 9, 19, 38, 8, 20, 20, 7, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 22, 11, 2, 2, 2, 14, 2, 22, 2,\n 2, 12, 14, 11, 14, 2, 2, 2, 2, 2, 2, 2, 20, 11, 5, 1, 1, 27, 27, 8, 37, 34, 34, 34, 34, 34, 34, 37,\n 37, 37, 37, 37, 37, 37, 34, 34, 34, 34, 37, 24, 36, 34, 24, 30, 30, 24, 24, 11, 24, 30, 27, 6, 6,\n 9, 35, 35, 30, 30, 35, 35, 35, 19, 27, 27, 9, 38, 23, 34, 28, 38, 38, 38, 38, 38, 31, 5, 28, 32,\n 28, 32, 32, 38, 38, 24, 27, 28, 28, 28, 28, 24, 11, 11, 24, 11, 2, 12, 11, 14, 24, 11, 2, 11, 11,\n 11, 12, 11, 11, 11, 11, 11, 12, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 11, 11, 12, 11, 11, 11,\n 14, 11, 11, 11, 11, 6, 6, 6, 5, 5, 5, 9, 28, 1, 28, 5, 3, 19, 31, 23, 31, 25, 0, 28, 14, 28, 14,\n 27, 14, 27, 3, 23, 9, 9, 16, 13, 13, 8, 32, 8, 26, 5, 28, 28, 28, 20, 20, 33, 28, 33, 27, 20, 2,\n 20, 24, 33, 28, 20, 33, 28, 32, 38, 27, 27, 38, 38, 38, 26, 26, 20, 26, 36, 36, 30, 8, 8, 8, 22, 2,\n 36, 36, 22, 19, 20, 31, 31, 20, 27, 27, 27, 0, 31, 31, 31, 27, 14, 5, 8, 27, 26, 23, 30, 30, 30,\n 39, 36, 36, 10, 32, 3, 3, 24, 16, 23, 16, 14, 16, 12, 16, 24, 23, 16, 14, 11, 7, 23, 34, 34, 7, 7,\n 16, 16, 16, 19, 16, 37, 20, 38, 23, 23, 23, 38, 38, 38, 23, 23, 38, 38, 32, 38, 28, 23, 23, 23, 23,\n 38, 16, 17, 17, 35, 37, 4, 25, 25, 10, 13, 21, 4, 4, 10, 4, 4, 4, 4, 15, 5, 15, 7, 14, 14, 12, 25,\n 34, 23, 38, 38, 27, 27, 21, 34, 34, 14, 27, 14, 14, 27, 16, 1, 27, 24, 24, 12, 24, 24, 19, 23, 5,\n 32, 32, 32, 32, 32, 27, 27, 27, 27, 31, 16, 10, 33, 0, 27, 27, 27, 1, 1, 14, 15, 36, 39, 36, 30,\n 30, 30, 30, 30, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 20, 33, 2, 26, 24, 35, 38, 27, 5, 28,\n 21, 21, 21, 21, 21, 21, 8, 8, 36, 38, 38, 36, 36, 28, 38, 34, 32, 36, 28, 34, 27, 1, 1, 1, 1, 27,\n 1, 14, 1, 19, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 1, 1, 1, 1, 14, 14, 1, 27, 27, 1, 1, 1, 1, 14,\n 14, 38, 1, 1, 1, 1, 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 21, 39, 30, 36, 18, 34, 15, 15, 4, 25, 28,\n 19, 19, 0, 23, 23, 3, 31, 21, 27, 13, 18, 4, 25, 12, 12, 12, 34, 34, 27, 34, 34, 37, 27, 34, 26,\n 27, 25, 31, 8, 8, 34, 8, 15, 3, 32, 27, 8, 8, 8, 7, 28, 38, 28, 28, 28, 28, 28, 28, 28, 28, 28, 20,\n 20, 28, 33, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 34, 23, 23, 34, 34, 34, 34,\n 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 39, 30, 36, 23, 36, 36, 36, 36, 36, 36, 7, 32, 23,\n 23, 23, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 28, 32, 32, 32, 32, 32, 38, 14, 14, 32, 14, 32, 32,\n 32, 13, 32, 32, 16, 16, 14, 14, 3, 26, 31, 29, 22, 22, 12, 11, 22, 14, 14, 14, 14, 14, 14, 14, 14,\n 14, 14, 22, 2, 14, 14, 14, 14, 14, 14, 22, 3, 24, 12, 11, 31, 24, 11, 14, 24, 24, 34, 36, 24, 34,\n 24, 30, 30, 24, 24, 31, 24, 24, 24, 31, 11, 31, 6, 31, 3, 12, 0, 0, 26, 26, 0, 32, 2, 16, 2, 27,\n 27, 27, 1, 27, 27, 27, 27, 39, 39, 39, 13, 13, 30, 30, 23, 30, 27, 27, 2, 23, 2, 2, 2, 2, 8, 8, 4,\n 20, 0, 2, 2, 2, 23, 32, 38, 14, 12, 3, 17, 23, 5, 19, 16, 16, 1, 1, 16, 33, 36, 36, 13, 19, 12, 29,\n 29, 4, 31, 12, 12, 12, 12, 14, 14, 12, 11, 14, 12, 12, 12, 12, 14, 12, 14, 12, 14, 14, 28, 2, 28,\n 35, 33, 28, 32, 23, 35, 1, 38, 38, 38, 38, 38, 38, 32, 17, 23, 17, 38, 23, 23, 23, 14, 17, 35, 38,\n 17, 8, 23, 23, 32, 8, 8, 12, 32, 35, 0, 9, 6, 36, 36, 35, 35, 6, 7, 14, 2, 9, 33, 33, 32, 34, 34,\n 34, 34, 19, 27, 27, 27, 8, 8, 19, 36, 22, 30, 23, 30, 28, 34, 34, 22, 13, 23, 13, 16, 24, 24, 24,\n 31, 12, 22, 31, 27, 31, 10, 10, 13, 21, 10, 10, 10, 21, 21, 21, 10, 10, 10, 10, 10, 10, 16, 12, 14,\n 1, 16, 14, 16, 16, 1, 27, 21, 30, 30, 39, 27, 5, 31, 9, 9, 20, 30, 8, 12, 20, 26, 26, 26, 33, 35,\n 16, 32, 23, 12, 38, 31, 38, 38, 38, 18, 14, 38, 38, 13, 13, 13, 13, 13, 13, 13, 38, 38, 12, 13, 21,\n 13, 13, 25, 5, 1, 1, 2, 1, 1, 7, 16, 6, 34, 34, 1, 19, 14, 37, 37, 37, 16, 15, 26, 22, 8, 27, 30,\n 32, 27, 27, 0, 26, 3, 15, 35, 2, 0, 27, 6, 21, 8, 23, 38, 32, 32, 27, 34, 27, 11, 20, 20, 20, 33,\n 20, 27, 20, 8, 27, 14, 14, 38, 34, 14, 20, 27, 14, 14, 33, 11, 14, 24, 24, 14, 14, 14, 14, 14, 14,\n 14, 14, 38, 14, 14, 14, 27, 14, 21, 14, 14, 27, 19, 16, 19, 8, 32, 23, 34, 34, 34, 34, 7, 16, 14,\n 14, 14, 24, 27, 27, 6, 24, 36, 27, 33, 2, 22, 2, 2, 23, 23, 2, 22, 22, 22, 19, 22, 19, 19, 2, 2,\n 22, 22, 22, 22, 22, 19, 22, 11, 23, 11, 33, 28, 28, 28, 28, 33, 27, 36, 38, 32, 28, 36, 26, 26, 30,\n 20, 23, 33, 20, 27, 14, 27, 14, 27, 14, 14, 14, 21, 7, 7, 16, 12, 12, 28, 28, 14, 23, 14, 39, 39,\n 27, 28, 28, 36, 28, 38, 36, 32, 28, 28, 27, 7, 7, 16, 27, 14, 28, 26, 37, 37, 15, 32, 32, 9, 34,\n 12, 12, 2, 12, 12, 2, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,\n 12, 12, 12, 12, 12, 12, 12, 14, 12, 12, 12, 12, 12, 12, 11, 12, 28, 27, 27, 27, 27, 16, 14, 31, 28,\n 7, 7, 7, 5, 23, 20, 33, 20, 3, 24, 11, 23, 35, 26, 26, 26, 33, 37, 38, 1, 32, 32, 14, 19, 19, 19,\n 19, 19, 38, 39, 39, 36, 33, 16, 16, 23, 16, 28, 28, 26, 26, 26, 33, 33, 20, 28, 36, 30, 36, 36, 36,\n 36, 30, 30, 30, 36, 30, 30, 30, 36, 30, 30, 30, 30, 30, 30, 30, 36, 30, 23, 23, 23, 30, 30, 36, 26,\n 15, 31, 19, 24, 38, 27, 38, 34, 34, 34, 5, 39, 23, 23, 23, 12, 30, 12, 12, 32, 34, 27, 34, 34, 34,\n 34, 3, 34, 34, 34, 24, 34, 34, 16, 16, 16, 27, 12, 27, 12, 27, 27, 19, 19, 38, 39, 19, 19, 23, 6,\n 6, 6, 6, 6, 25, 9, 8, 31, 5, 5, 9, 5, 0, 23, 24, 2, 2, 32, 20, 2, 23, 2, 30, 34, 12, 24, 24, 20,\n 20, 24, 24, 30, 11, 39, 22, 3, 23, 38, 38, 5, 37, 15, 23, 21, 28, 27, 32, 32, 34, 34, 27, 27, 4,\n 13, 32, 32, 11, 9, 9, 22, 18, 8, 8, 19, 19, 27, 30, 14, 14, 14, 20, 31, 22, 31, 8, 5, 32, 6, 13,\n 29, 31, 0, 19, 19, 19, 32, 36, 30, 32, 8, 6, 29, 29, 31, 27, 27, 0, 5, 38, 32, 17, 23, 17, 17, 7,\n 17, 36, 36, 6, 13, 25, 6, 21, 6, 21, 21, 21, 32, 13, 21, 21, 10, 13, 13, 13, 13, 13, 13, 13, 13,\n 13, 13, 21, 13, 13, 21, 21, 6, 21, 21, 21, 6, 21, 21, 21, 21, 21, 21, 21, 21, 6, 6, 6, 23, 21, 6,\n 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 32, 23, 38, 2, 32, 32, 32, 32, 32, 32, 28, 28, 32,\n 32, 32, 38, 14, 38, 32, 32, 32, 32, 13, 32, 32, 28, 28, 32, 32, 32, 32, 28, 23, 28, 38, 38, 0, 26,\n 22, 13, 37, 5, 9, 27, 26, 27, 20, 30, 30, 27, 27, 5, 5, 24, 20, 2, 5, 34, 34, 19, 19, 23, 38, 38,\n 35, 1, 1, 1, 5, 26, 26, 26, 26, 8, 39, 23, 30, 30, 30, 36, 30, 30, 23, 30, 39, 27, 3, 3, 33, 18,\n 19, 28, 26, 23, 23, 23, 26, 26, 26, 19, 5, 35, 7, 0, 0, 3, 5, 25, 23, 21, 9, 0, 7, 14, 33, 14, 14,\n 33, 7, 10, 26, 26, 26, 5, 25, 23, 23, 27, 19, 19, 20, 13, 19, 19, 26, 38, 36, 30, 23, 30, 30, 23,\n 30, 38, 38, 15, 1, 27, 35, 23, 5, 27, 8, 17, 5, 22, 22, 22, 22, 22, 22, 19, 16, 27, 16, 27, 33, 27,\n 36, 7, 14, 7, 14, 23, 16, 3, 20, 8, 17, 19, 27, 30, 10, 39, 27, 19, 39, 26, 18, 32, 27, 7, 9, 3,\n 16, 14, 6, 13, 9, 5, 24, 24, 7, 11, 20, 20, 9, 17, 11, 11, 14, 11, 16, 16, 14, 14, 27, 14, 14, 14,\n 22, 22, 22, 16, 7, 14, 26, 25, 25, 25, 35, 17, 5, 2, 24, 34, 2, 2, 2, 20, 2, 30, 34, 2, 2, 2, 2, 2,\n 2, 20, 20, 30, 30, 2, 24, 24, 24, 15, 38, 33, 23, 23, 23, 23, 23, 23, 23, 33, 33, 33, 33, 33, 33,\n 33, 12, 33, 33, 33, 33, 33, 33, 33, 16, 16, 16, 23, 16, 16, 23, 16, 1, 16, 16, 33, 25, 23, 4, 10,\n 21, 21, 25, 21, 4, 10, 4, 25, 25, 25, 25, 25, 25, 25, 25, 25, 21, 21, 32, 38, 38, 24, 24, 24, 24,\n 20, 20, 24, 11, 20, 38, 34, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,\n 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,\n 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40)\n\n_label_groups3 = (36, 37, 21, 21, 8, 28, 9, 32, 28, 25, 23, 23, 23, 26, 26, 22, 26, 22, 26, 22, 22, 26, 10, 42, 33, 9,\n 34, 5, 19, 4, 33, 33, 42, 33, 33, 36, 41, 1, 1, 18, 29, 27, 29, 21, 29, 21, 9, 34, 7, 29, 3, 41, 15,\n 29, 4, 20, 36, 9, 29, 37, 8, 10, 11, 10, 10, 10, 10, 10, 32, 10, 10, 10, 10, 10, 12, 12, 12, 12, 10,\n 10, 10, 11, 12, 11, 12, 10, 37, 25, 10, 10, 10, 10, 10, 5, 4, 6, 6, 37, 37, 9, 42, 42, 42, 42, 42,\n 42, 42, 42, 42, 42, 42, 42, 42, 37, 42, 42, 42, 42, 42, 10, 39, 42, 39, 39, 39, 10, 30, 5, 16, 37,\n 37, 13, 13, 7, 43, 43, 45, 45, 43, 43, 43, 27, 23, 23, 4, 47, 28, 42, 24, 47, 47, 47, 47, 47, 34,\n 4, 48, 41, 24, 41, 41, 47, 47, 48, 27, 33, 24, 48, 48, 10, 5, 5, 10, 5, 10, 11, 11, 12, 10, 5, 5, 5,\n 5, 5, 11, 5, 5, 5, 5, 5, 11, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 11, 5, 5, 11, 5, 11, 5, 12, 5, 5, 5, 5, 13,\n 13, 13, 4, 1, 1, 29, 48, 1, 48, 1, 5, 27, 34, 28, 32, 3, 5, 48, 22, 37, 14, 37, 14, 37, 3, 28, 25, 4,\n 16, 14, 14, 9, 41, 9, 32, 0, 19, 25, 25, 25, 25, 25, 24, 25, 25, 25, 10, 19, 19, 25, 19, 19, 19, 19,\n 41, 23, 37, 37, 47, 48, 43, 38, 45, 29, 38, 45, 45, 39, 9, 9, 9, 16, 33, 48, 48, 16, 27, 29, 34, 34,\n 29, 33, 33, 33, 5, 34, 34, 34, 17, 12, 1, 9, 30, 45, 28, 31, 31, 7, 49, 48, 48, 15, 41, 3, 3, 10, 26,\n 28, 26, 12, 26, 18, 26, 10, 28, 26, 12, 5, 8, 42, 44, 42, 8, 8, 26, 26, 26, 27, 26, 18, 29, 47, 28,\n 28, 28, 47, 47, 47, 28, 28, 47, 47, 41, 47, 24, 28, 28, 28, 28, 47, 30, 21, 21, 40, 29, 15, 15, 28,\n 15, 14, 15, 15, 15, 15, 15, 15, 15, 15, 4, 4, 13, 16, 12, 12, 11, 13, 42, 28, 47, 36, 17, 25, 13, 42,\n 42, 22, 22, 22, 22, 37, 1, 1, 26, 10, 10, 10, 10, 10, 27, 28, 4, 41, 41, 9, 41, 41, 17, 17, 17, 17,\n 34, 26, 12, 25, 5, 22, 12, 22, 1, 1, 12, 15, 35, 49, 39, 39, 39, 45, 45, 45, 49, 49, 49, 49, 49, 49,\n 49, 49, 49, 49, 49, 29, 29, 25, 32, 10, 35, 47, 33, 34, 48, 13, 13, 13, 13, 13, 13, 9, 9, 48, 47, 47,\n 48, 48, 24, 48, 44, 48, 48, 6, 44, 22, 6, 6, 6, 6, 18, 1, 12, 6, 20, 6, 6, 6, 6, 6, 6, 6, 1, 6, 6, 18,\n 18, 6, 6, 18, 6, 18, 18, 6, 18, 18, 6, 6, 6, 1, 12, 12, 18, 6, 6, 18, 6, 18, 18, 6, 18, 7, 6, 6, 6, 6,\n 6, 1, 15, 49, 39, 45, 23, 44, 21, 21, 15, 3, 48, 20, 20, 5, 28, 28, 3, 32, 13, 23, 23, 23, 15, 15, 11,\n 11, 11, 42, 37, 42, 42, 37, 42, 37, 42, 32, 46, 5, 34, 9, 9, 42, 9, 48, 48, 26, 42, 9, 9, 9, 0, 24,\n 47, 25, 25, 48, 48, 47, 24, 0, 25, 25, 29, 29, 41, 25, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,\n 24, 24, 24, 44, 28, 28, 42, 44, 44, 42, 42, 42, 44, 42, 37, 44, 44, 44, 44, 44, 44, 44, 49, 39, 39,\n 28, 39, 39, 39, 39, 39, 39, 49, 41, 28, 28, 28, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 24, 41, 41,\n 41, 41, 41, 47, 22, 22, 41, 12, 41, 41, 41, 41, 41, 41, 16, 16, 12, 12, 3, 32, 34, 8, 16, 16, 11, 5,\n 16, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 16, 5, 10, 11, 11, 34, 10,\n 5, 12, 10, 10, 42, 39, 34, 42, 39, 39, 39, 10, 10, 34, 10, 10, 10, 34, 5, 34, 13, 34, 2, 2, 17, 17,\n 32, 32, 5, 41, 10, 26, 2, 37, 18, 18, 18, 18, 18, 26, 37, 0, 0, 0, 23, 23, 31, 31, 31, 31, 37, 17,\n 10, 28, 10, 16, 12, 10, 9, 9, 3, 29, 5, 10, 10, 10, 37, 41, 36, 22, 11, 29, 21, 28, 4, 27, 30, 18, 18,\n 18, 18, 18, 35, 35, 35, 27, 11, 19, 19, 15, 32, 11, 11, 11, 11, 22, 22, 11, 11, 12, 11, 11, 11, 11,\n 12, 11, 12, 11, 12, 12, 19, 25, 25, 25, 25, 25, 41, 28, 43, 1, 43, 47, 43, 43, 43, 43, 41, 21, 28, 21,\n 36, 28, 28, 28, 22, 21, 43, 36, 21, 9, 28, 28, 41, 9, 9, 11, 41, 35, 27, 4, 13, 35, 35, 35, 35, 13,\n 25, 12, 10, 25, 29, 25, 41, 42, 42, 42, 42, 27, 27, 12, 27, 9, 9, 27, 39, 16, 39, 28, 39, 48, 44, 44,\n 16, 14, 28, 14, 26, 10, 10, 10, 32, 11, 33, 32, 33, 34, 15, 15, 14, 15, 15, 15, 40, 15, 13, 15, 15,\n 14, 15, 15, 15, 15, 26, 11, 12, 1, 26, 12, 26, 26, 6, 37, 13, 31, 31, 49, 17, 4, 34, 29, 29, 29, 39,\n 9, 11, 29, 32, 32, 32, 33, 35, 26, 41, 28, 5, 43, 34, 36, 36, 36, 36, 22, 36, 36, 36, 14, 14, 14, 36,\n 36, 36, 36, 36, 11, 14, 14, 14, 14, 14, 7, 1, 1, 10, 1, 1, 8, 26, 13, 44, 44, 18, 33, 22, 36, 33, 36,\n 30, 25, 38, 33, 9, 37, 39, 41, 23, 23, 5, 38, 29, 8, 35, 37, 5, 33, 13, 13, 9, 28, 47, 41, 41, 37, 42,\n 42, 5, 29, 29, 29, 25, 29, 33, 19, 9, 22, 22, 22, 22, 22, 22, 29, 22, 22, 22, 22, 22, 22, 10, 10, 12,\n 12, 12, 12, 12, 12, 12, 12, 47, 12, 12, 12, 25, 12, 13, 12, 12, 23, 27, 26, 20, 9, 41, 28, 42, 44, 44,\n 42, 42, 16, 22, 12, 12, 10, 37, 37, 13, 10, 35, 33, 27, 10, 33, 10, 10, 28, 28, 12, 10, 16, 16, 27,\n 16, 27, 27, 10, 10, 33, 33, 33, 33, 33, 33, 33, 38, 28, 38, 25, 25, 25, 25, 25, 25, 23, 45, 47, 41,\n 24, 45, 32, 45, 45, 25, 28, 25, 25, 23, 22, 11, 22, 22, 22, 0, 22, 14, 0, 0, 0, 11, 11, 24, 24, 0, 28,\n 22, 49, 49, 23, 24, 25, 39, 25, 47, 48, 48, 25, 24, 27, 31, 9, 16, 16, 12, 48, 38, 36, 42, 21, 41, 9,\n 4, 44, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,\n 11, 11, 11, 11, 11, 11, 11, 11, 12, 11, 11, 11, 11, 11, 11, 11, 16, 48, 22, 22, 22, 22, 26, 12, 34,\n 25, 8, 8, 8, 4, 28, 29, 33, 29, 3, 10, 5, 28, 43, 38, 38, 38, 25, 36, 36, 18, 41, 30, 12, 27, 27, 27,\n 27, 27, 47, 49, 49, 28, 45, 26, 28, 28, 30, 25, 24, 49, 45, 45, 25, 25, 29, 24, 45, 45, 45, 45, 45,\n 45, 45, 42, 31, 39, 31, 39, 39, 45, 31, 39, 31, 31, 31, 31, 45, 45, 45, 28, 28, 28, 39, 31, 45, 45,\n 21, 34, 27, 33, 43, 37, 36, 42, 42, 42, 4, 45, 28, 28, 28, 11, 39, 45, 45, 41, 42, 37, 42, 42, 42, 42,\n 3, 42, 42, 42, 42, 42, 42, 30, 30, 30, 23, 11, 30, 11, 23, 23, 27, 27, 47, 49, 20, 33, 28, 7, 7, 7, 7,\n 7, 13, 4, 9, 34, 4, 4, 29, 4, 5, 28, 10, 10, 10, 41, 29, 10, 28, 12, 39, 42, 11, 10, 10, 29, 29, 32,\n 32, 39, 5, 49, 12, 29, 28, 43, 43, 4, 8, 15, 28, 13, 24, 25, 48, 48, 42, 42, 30, 30, 30, 40, 41, 41,\n 5, 29, 29, 17, 23, 9, 9, 20, 20, 33, 39, 22, 22, 22, 29, 32, 33, 32, 9, 34, 41, 13, 14, 13, 34, 5, 20,\n 20, 20, 3, 45, 31, 37, 9, 13, 19, 19, 34, 23, 30, 5, 4, 36, 41, 21, 28, 21, 21, 8, 21, 45, 35, 13, 14,\n 28, 13, 13, 13, 13, 15, 15, 41, 14, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,\n 15, 15, 13, 13, 13, 13, 13, 15, 15, 14, 15, 15, 13, 15, 15, 13, 13, 13, 28, 14, 13, 13, 13, 13, 13,\n 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 41, 28, 47, 41, 41, 41, 41, 41, 41, 41, 24, 24, 41, 41,\n 41, 36, 22, 41, 41, 41, 41, 37, 41, 41, 41, 24, 24, 41, 41, 41, 41, 48, 28, 25, 47, 47, 5, 32, 33, 14,\n 14, 4, 4, 37, 32, 18, 29, 31, 31, 33, 33, 4, 4, 37, 29, 33, 4, 25, 44, 20, 20, 28, 47, 47, 40, 6, 6,\n 6, 4, 19, 19, 19, 19, 19, 49, 28, 39, 39, 39, 39, 31, 31, 28, 31, 49, 37, 3, 3, 29, 36, 27, 25, 38,\n 28, 28, 28, 38, 38, 38, 27, 1, 35, 8, 5, 5, 2, 34, 8, 37, 13, 29, 5, 16, 22, 22, 16, 12, 12, 16, 23,\n 49, 49, 49, 34, 15, 28, 28, 27, 20, 20, 29, 14, 27, 27, 38, 47, 45, 31, 28, 31, 39, 28, 31, 48, 47, 4,\n 1, 26, 40, 37, 4, 12, 9, 21, 4, 33, 33, 33, 33, 33, 33, 27, 26, 18, 26, 30, 33, 23, 48, 8, 12, 39, 22,\n 28, 26, 29, 29, 9, 21, 27, 23, 31, 15, 49, 37, 20, 49, 45, 26, 41, 33, 8, 25, 3, 0, 12, 13, 14, 25, 7,\n 10, 10, 38, 42, 29, 29, 29, 21, 5, 5, 12, 5, 16, 2, 12, 12, 25, 12, 12, 12, 16, 16, 33, 16, 8, 22, 38,\n 13, 14, 13, 43, 21, 4, 10, 10, 42, 10, 22, 10, 29, 10, 39, 42, 16, 10, 10, 10, 10, 36, 29, 29, 45, 10,\n 10, 10, 10, 10, 38, 36, 29, 28, 28, 28, 28, 28, 28, 28, 33, 29, 18, 18, 18, 29, 25, 11, 25, 29, 29,\n 29, 29, 29, 29, 1, 30, 30, 28, 30, 1, 28, 26, 1, 26, 1, 25, 15, 28, 15, 15, 15, 15, 15, 15, 15, 15,\n 15, 14, 15, 15, 15, 15, 15, 15, 15, 15, 13, 30, 41, 19, 43, 10, 10, 10, 10, 29, 29, 32, 5, 29, 47, 44,\n 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,\n 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,\n 50, 50, 50, 50, 50)\n\n_label_groups4 = (27, 4, 18, 12, 0, 8, 14, 14, 14, 4, 12, 0, 9, 9, 3, 9, 6, 15, 7, 22, 22, 12, 15, 1, 2, 9, 11, 1, 23,\n 6, 0, 6, 0, 6, 8, 3, 4, 27, 24, 18, 0, 17, 9, 22, 15, 15, 0, 14, 9, 21, 0, 5, 16, 15, 0, 17, 26, 25,\n 14, 13, 26, 6, 15, 11, 28, 18, 27, 13, 17, 1, 10, 22, 27, 23, 29, 9, 7, 25, 25, 10, 25, 4, 2, 4, 28,\n 25, 10, 19, 7, 28, 20, 12, 9, 18, 17, 13, 9, 29, 25, 14, 5, 23, 27, 23, 3, 3, 7, 24, 11, 19, 19, 10,\n 25, 18, 3, 7, 23, 24, 27, 26, 26, 29, 21, 27, 14, 28, 12, 28, 12, 27, 27, 11, 29, 27, 25, 12, 15, 16,\n 0, 25, 29, 23, 19, 12, 18, 8, 18, 25, 11, 5, 12, 4, 2, 4, 24, 14, 24, 0, 25, 12, 16, 9, 29, 19, 15,\n 15, 14, 6, 13, 13, 3, 13, 8, 9, 19, 18, 13, 18, 17, 25, 5, 27, 11, 10, 20, 28, 10, 7, 9, 9, 23, 6, 15,\n 2, 13, 25, 17, 9, 8, 14, 14, 14, 1, 4, 16, 22, 11, 15, 9, 12, 9, 27, 25, 2, 12, 2, 2, 25, 25, 0, 13,\n 28, 5, 18, 11, 17, 7, 4, 11, 25, 25, 10, 10, 10, 18, 17, 13, 13, 13, 13, 26, 6, 10, 1, 10, 23, 13, 13,\n 8, 13, 22, 7, 15, 4, 19, 19, 2, 2, 4, 0, 16, 16, 29, 22, 9, 9, 24, 10, 10, 5, 9, 11, 11, 17, 5, 12, 9,\n 10, 9, 5, 16, 22, 25, 16, 16, 13, 28, 15, 16, 8, 26, 12, 13, 17, 27, 27, 27, 7, 10, 13, 19, 15, 25,\n 18, 18, 8, 27, 26, 28, 10, 22, 14, 18, 4, 7, 5, 26, 17, 11, 29, 9, 2, 15, 12, 8, 22, 8, 3, 12, 12, 26,\n 17, 28, 9, 27, 26, 8, 1, 17, 10, 9, 4, 9, 17, 0, 0, 12, 15, 27, 12, 3, 7, 18, 9, 26, 11, 11, 0, 24,\n 16, 24, 12, 24, 18, 11, 7, 16, 5, 26, 27, 18, 5, 24, 10, 17, 25, 1, 4, 0, 17, 1, 3, 29, 4, 4, 23, 14,\n 2, 4, 29, 0, 0, 0, 3, 29, 18, 19, 25, 19, 17, 1, 28, 29, 27, 11, 6, 25, 27, 26, 9, 16, 8, 1, 9, 24, 8,\n 7, 24, 25, 8, 7, 18, 15, 10, 10, 23, 12, 4, 4, 9, 5, 26, 12, 2, 5, 12, 14, 1, 5, 26, 18, 4, 24, 4, 2,\n 4, 17, 22, 18, 1, 23, 0, 0, 5, 24, 13, 10, 11, 22, 0, 25, 18, 7, 19, 10, 27, 16, 4, 24, 5, 27, 15, 12,\n 24, 14, 14, 14, 10, 15, 4, 4, 6, 17, 23, 0, 2, 12, 16, 17, 4, 10, 12, 27, 10, 0, 8, 14, 8, 12, 23, 0,\n 10, 11, 18, 27, 8, 7, 27, 10, 16, 19, 27, 8, 10, 8, 0, 10, 23, 6, 8, 18, 0, 0, 3, 10, 4, 26, 10, 11,\n 8, 20, 28, 29, 8, 9, 18, 0, 17, 14, 14, 22, 22, 8, 27, 19, 5, 0, 19, 10, 22, 24, 19, 18, 5, 9, 22, 12,\n 15, 2, 5, 11, 24, 27, 16, 10, 7, 11, 18, 12, 6, 23, 15, 27, 27, 17, 10, 23, 8, 28, 27, 9, 9, 28, 4, 4,\n 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 19, 15, 14, 12, 29, 18, 12, 17, 24, 18, 10, 25, 15, 15,\n 28, 27, 27, 5, 10, 19, 4, 11, 13, 11, 17, 28, 1, 17, 12, 1, 14, 14, 15, 15, 16, 22, 16, 12, 5, 10, 10,\n 16, 22, 2, 9, 0, 10, 20, 18, 16, 10, 18, 25, 14, 9, 25, 16, 8, 27, 9, 24, 18, 15, 9, 16, 16, 21, 8,\n 10, 27, 17, 24, 6, 25, 13, 24, 13, 13, 13, 13, 18, 29, 21, 15, 25, 14, 7, 19, 18, 18, 27, 24, 17, 28,\n 25, 8, 0, 22, 23, 26, 8, 27, 29, 11, 10, 25, 10, 29, 15, 29, 13, 0, 19, 9, 27, 8, 26, 2, 1, 19, 21,\n 21, 28, 9, 24, 22, 13, 17, 8, 27, 12, 16, 22, 18, 27, 7, 16, 27, 25, 13, 16, 5, 18, 6, 26, 18, 18, 7,\n 0, 3, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models.resnet as resnet\nimport collections\nimport torch.nn.functional as F\n\n\nBLOCK = {'18': 'BasicBlock',\n '34': 'BasicBlock',\n '50': 'Bottleneck',\n '101': 'Bottleneck',\n '152': 'Bottleneck'}\n\n\nLAYERS = {'18': [2, 2, 2, 2],\n '34': [3, 4, 6, 3],\n '50': [3, 4, 6, 3],\n '101': [3, 4, 23, 3],\n '152': [3, 8, 36, 3]}\n\n\nclass ArtCV(nn.Module):\n def __init__(self, tag='18', num_labels=(100, 681, 6, 1920, 768),\n classifier_layers=(1, 1, 1, 1, 1), classifier_hidden=(2048, 2048, 2048, 2048, 2048),\n task=('ml', 'ml', 'mc', 'ml', 'ml'), weights=(1, 1, 1, 1, 1),\n use_batch_norm=True, dropout_rate=0.1,\n weight_path=None, freeze_cnn=False,\n focal_loss=False, focal_loss_mc=False, alpha_t_mc=True,\n alpha=(0.25, 0.25, 0.25, 0.25), alpha_mc=(0.25, 0.75, 0.75, 0.75, 0.75, 0.75),\n gamma_mc=2, gamma=(2, 2, 2, 2), alpha_t=True, alpha_group=(1, 1), gamma_group=2,\n hierarchical=False, label_groups=(0, 1, 1, 1, 1, 1), group_classifier_kwargs=dict(),\n weight_group=None,\n hierarchical_ml=(False, False, False, False),\n label_groups0=None, label_groups1=None, label_groups3=None, label_groups4=None,\n group_classifier_kwargs0=dict(), group_classifier_kwargs1=dict(), group_classifier_kwargs3=dict(),\n group_classifier_kwargs4=dict(), weight_group_ml=(None, None, None, None)):\n super().__init__()\n self.tag = tag\n self.num_labels = num_labels\n self.classifier_layers = classifier_layers\n self.classifier_hidden = classifier_hidden\n self.task = task\n self.weights = weights\n self.use_batch_norm = use_batch_norm\n self.dropout_rate = dropout_rate\n self.weight_path = weight_path\n self.freeze_cnn = freeze_cnn\n self.focal_loss = focal_loss\n self.focal_loss_mc = focal_loss_mc\n self.hierarchical = hierarchical\n self.hierarchical_ml = hierarchical_ml\n\n if self.focal_loss:\n self.alpha = alpha\n self.gamma = gamma\n self.alpha_t = alpha_t\n if self.focal_loss_mc:\n self.alpha_mc =alpha_mc\n self.gamma_mc = gamma_mc\n self.alpha_t_mc = alpha_t_mc\n if self.hierarchical:\n self.alpha_group = alpha_group\n self.gamma_group = gamma_group\n\n self.cnn = ResNet_CNN(getattr(resnet, BLOCK[tag]), LAYERS[tag],\n weight_path=self.weight_path, freeze_layers=self.freeze_cnn)\n\n self.classifiers = nn.ModuleDict(\n collections.OrderedDict(\n [('classifier{}'.format(i), Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n dim_out=self.num_labels[i],\n dim_hidden=self.classifier_hidden[i],\n n_layers=self.classifier_layers[i],\n task=self.task[i],\n use_batch_norm=self.use_batch_norm,\n dropout_rate=self.dropout_rate))\n for i in range(len(self.num_labels))]))\n\n if self.hierarchical:\n self.label_groups = np.array(label_groups)\n self.num_groups = len(np.unique(self.label_groups))\n self.group_classifier_kwargs = {'dim_hidden': self.classifier_hidden[2],\n 'n_layers': self.classifier_layers[2],\n 'task': self.task[2],\n 'use_batch_norm': self.use_batch_norm,\n 'dropout_rate': self.dropout_rate}\n self.group_classifier_kwargs.update(group_classifier_kwargs)\n self.group_classifier = Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n dim_out=self.num_groups, **self.group_classifier_kwargs)\n self.weight_group = weight_group if weight_group is not None else self.weights[2]\n self.group_idx_list = torch.nn.ParameterList([torch.nn.Parameter(\n torch.tensor((self.label_groups == i).astype(np.uint8), dtype=torch.bool), requires_grad=False)\n for i in range(self.num_groups)])\n\n if self.hierarchical_ml[0]:\n self.label_groups0 = np.array(label_groups0) if label_groups0 is not None else np.array(_label_groups0)\n self.num_groups0 = len(np.unique(self.label_groups0))\n self.group_classifier_kwargs0 = {'dim_hidden': self.classifier_hidden[0],\n 'n_layers': self.classifier_layers[0],\n 'task': self.task[0],\n 'use_batch_norm': self.use_batch_norm,\n 'dropout_rate': self.dropout_rate}\n self.group_classifier_kwargs0.update(group_classifier_kwargs0)\n self.group_classifier0 = Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n dim_out=self.num_groups0, **self.group_classifier_kwargs0)\n self.weight_group0 = weight_group_ml[0] if weight_group_ml[0] is not None else self.weights[0]\n self.group_idx_list0 = torch.nn.ParameterList([torch.nn.Parameter(\n torch.tensor((self.label_groups0 == i).astype(np.uint8), dtype=torch.bool), requires_grad=False)\n for i in range(self.num_groups0)])\n\n if self.hierarchical_ml[1]:\n self.label_groups1 = np.array(label_groups1) if label_groups1 is not None else np.array(_label_groups1)\n self.num_groups1 = len(np.unique(self.label_groups1))\n self.group_classifier_kwargs1 = {'dim_hidden': self.classifier_hidden[1],\n 'n_layers': self.classifier_layers[1],\n 'task': self.task[1],\n 'use_batch_norm': self.use_batch_norm,\n 'dropout_rate': self.dropout_rate}\n self.group_classifier_kwargs1.update(group_classifier_kwargs1)\n self.group_classifier1 = Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n dim_out=self.num_groups1, **self.group_classifier_kwargs1)\n self.weight_group1 = weight_group_ml[1] if weight_group_ml[1] is not None else self.weights[1]\n self.group_idx_list1 = torch.nn.ParameterList([torch.nn.Parameter(\n torch.tensor((self.label_groups1 == i).astype(np.uint8), dtype=torch.bool), requires_grad=False)\n for i in range(self.num_groups1)])\n\n if self.hierarchical_ml[2]:\n self.label_groups3 = np.array(label_groups3) if label_groups3 is not None else np.array(_label_groups3)\n self.num_groups3 = len(np.unique(self.label_groups3))\n self.group_classifier_kwargs3 = {'dim_hidden': self.classifier_hidden[3],\n 'n_layers': self.classifier_layers[3],\n 'task': self.task[3],\n 'use_batch_norm': self.use_batch_norm,\n 'dropout_rate': self.dropout_rate}\n self.group_classifier_kwargs3.update(group_classifier_kwargs3)\n self.group_classifier3 = Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n dim_out=self.num_groups3, **self.group_classifier_kwargs3)\n self.weight_group3 = weight_group_ml[2] if weight_group_ml[2] is not None else self.weights[3]\n self.group_idx_list3 = torch.nn.ParameterList([torch.nn.Parameter(\n torch.tensor((self.label_groups3 == i).astype(np.uint8), dtype=torch.bool), requires_grad=False)\n for i in range(self.num_groups3)])\n\n if self.hierarchical_ml[3]:\n self.label_groups4 = np.array(label_groups4) if label_groups4 is not None else np.array(_label_groups4)\n self.num_groups4 = len(np.unique(self.label_groups4))\n self.group_classifier_kwargs4 = {'dim_hidden': self.classifier_hidden[4],\n 'n_layers': self.classifier_layers[4],\n 'task': self.task[4],\n 'use_batch_norm': self.use_batch_norm,\n 'dropout_rate': self.dropout_rate}\n self.group_classifier_kwargs4.update(group_classifier_kwargs4)\n self.group_classifier4 = Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n dim_out=self.num_groups4, **self.group_classifier_kwargs4)\n self.weight_group4 = weight_group_ml[3] if weight_group_ml[3] is not None else self.weights[4]\n self.group_idx_list4 = torch.nn.ParameterList([torch.nn.Parameter(\n torch.tensor((self.label_groups4 == i).astype(np.uint8), dtype=torch.bool), requires_grad=False)\n for i in range(self.num_groups4)])\n\n def inference(self, x):\n return self.cnn(x)\n\n def get_probs_mc(self, x_features):\n y_pred2 = self.classifiers['classifier2'](x_features)\n\n if self.hierarchical:\n y_group2 = self.group_classifier(x_features)\n y_weighted2 = torch.zeros_like(y_pred2)\n for i, group_idx in enumerate(self.group_idx_list):\n y_weighted2[:, group_idx] = y_pred2[:, group_idx] / \\\n (y_pred2[:, group_idx].sum(dim=-1, keepdim=True) + 1e-8) * \\\n y_group2[:, [i]]\n return y_weighted2.view(-1, self.num_labels[2]), y_group2.view(-1, self.num_groups)\n else:\n return y_pred2.view(-1, self.num_labels[2])\n\n def get_probs(self, x):\n x_features = self.inference(x)\n y_pred0 = self.classifiers['classifier0'](x_features)\n y_pred1 = self.classifiers['classifier1'](x_features)\n y_pred3 = self.classifiers['classifier3'](x_features)\n y_pred4 = self.classifiers['classifier4'](x_features)\n if self.hierarchical_ml[0]:\n y_group0 = self.group_classifier0(x_features)\n y_probs0 = torch.zeros_like(y_pred0)\n for i, group_idx in enumerate(self.group_idx_list0):\n y_probs0[:, group_idx] = y_pred0[:, group_idx] / \\\n (y_pred0[:, group_idx].sum(dim=-1, keepdim=True) + 1e-8) * \\\n y_group0[:, [i]]\n else:\n y_probs0 = y_pred0.view(-1, self.num_labels[0])\n y_group0 = None\n\n if self.hierarchical_ml[1]:\n y_group1 = self.group_classifier1(x_features)\n y_probs1 = torch.zeros_like(y_pred1)\n for i, group_idx in enumerate(self.group_idx_list1):\n y_probs1[:, group_idx] = y_pred1[:, group_idx] / \\\n (y_pred1[:, group_idx].sum(dim=-1, keepdim=True) + 1e-8) * \\\n y_group1[:, [i]]\n else:\n y_probs1 = y_pred1.view(-1, self.num_labels[1])\n y_group1 = None\n\n if self.hierarchical_ml[2]:\n y_group3 = self.group_classifier3(x_features)\n y_probs3 = torch.zeros_like(y_pred3)\n for i, group_idx in enumerate(self.group_idx_list3):\n y_probs3[:, group_idx] = y_pred3[:, group_idx] / \\\n (y_pred3[:, group_idx].sum(dim=-1, keepdim=True) + 1e-8) * \\\n y_group3[:, [i]]\n else:\n y_probs3 = y_pred3.view(-1, self.num_labels[3])\n y_group3 = None\n\n if self.hierarchical_ml[3]:\n y_group4 = self.group_classifier4(x_features)\n y_probs4 = torch.zeros_like(y_pred4)\n for i, group_idx in enumerate(self.group_idx_list4):\n y_probs4[:, group_idx] = y_pred4[:, group_idx] / \\\n (y_pred4[:, group_idx].sum(dim=-1, keepdim=True) + 1e-8) * \\\n y_group4[:, [i]]\n else:\n y_probs4 = y_pred4.view(-1, self.num_labels[4])\n y_group4 = None\n\n if self.hierarchical:\n y_probs2, y_group2 = self.get_probs_mc(x_features)\n else:\n y_probs2 = self.get_probs_mc(x_features)\n y_group2 = None\n\n return y_probs0, y_probs1, y_probs2, y_probs3, y_probs4, (y_group0, y_group1, y_group2, y_group3, y_group4)\n\n def get_loss_mc(self, x, y2, reduction='sum'):\n if self.hierarchical:\n y_pred2, y_group2 = self.get_probs_mc(self.inference(x))\n if self.focal_loss_mc:\n loss_group = focal_loss_mc(y_group2,\n torch.tensor(list(map(lambda x: self.label_groups[x],\n y2.view(-1)))).view(-1),\n num_classes=self.num_groups, alpha=self.alpha_group,\n gamma=self.gamma_group, alpha_t=self.alpha_t_mc)\n else:\n loss_group = F.cross_entropy(y_group2,\n torch.tensor(list(map(lambda x: self.label_groups[x],\n y2.view(-1)))).view(-1), reduction='none')\n else:\n y_pred2 = self.get_probs_mc(self.inference(x))\n if self.focal_loss_mc:\n loss2 = focal_loss_mc(y_pred2, y2.view(-1), num_classes=self.num_labels[2],\n alpha=self.alpha_mc, gamma=self.gamma_mc, alpha_t=self.alpha_t_mc)\n else:\n loss2 = F.cross_entropy(y_pred2, y2.view(-1), reduction='none')\n\n if self.hierarchical:\n if reduction == 'sum':\n return loss2 * self.weights[2] + loss_group * self.weight_group\n elif reduction == 'none':\n return loss2 * self.weights[2], loss_group * self.weight_group\n else:\n return loss2 * self.weights[2]\n\n def get_loss(self, x, y0, y1, y2, y3, y4):\n y_pred0, y_pred1, y_pred2, y_pred3, y_pred4, y_groups_tuples = self.get_probs(x)\n y_group0, y_group1, y_group2, y_group3, y_group4 = y_groups_tuples\n if self.focal_loss_mc:\n if self.hierarchical:\n loss_group2 = focal_loss_mc(y_group2,\n torch.tensor(list(map(lambda x: self.label_groups[x],\n y2.view(-1)))).view(-1),\n num_classes=self.num_groups, alpha=self.alpha_group,\n gamma=self.gamma_group, alpha_t=self.alpha_t_mc)\n else:\n loss_group2 = None\n loss2 = focal_loss_mc(y_pred2, y2.view(-1), num_classes=self.num_labels[2],\n alpha=self.alpha_mc, gamma=self.gamma_mc, alpha_t=self.alpha_t_mc)\n else:\n if self.hierarchical:\n loss_group2 = F.cross_entropy(y_group2,\n torch.tensor(list(map(lambda x: self.label_groups[x],\n y2.view(-1)))).view(-1), reduction='none')\n else:\n loss_group2 = None\n loss2 = F.cross_entropy(y_pred2, y2.view(-1), reduction='none')\n\n if self.focal_loss:\n loss0 = torch.mean(focal_loss_ml(y_pred0, y0, alpha=self.alpha[0], gamma=self.gamma[0],\n alpha_t=self.alpha_t), dim=1)\n if self.hierarchical_ml[0]:\n loss_group0 = torch.mean(focal_loss_ml(y_group0,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(\n i, num_classes=self.num_groups0),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((x * (\n torch.tensor(\n self.label_groups0)+1)-1),\n dim=-1)[1:],\n y0.long())))),\n dim=0).float(),\n alpha=self.alpha[0], gamma=self.gamma[0],\n alpha_t=self.alpha_t), dim=1)\n else:\n loss_group0 = None\n\n loss1 = torch.mean(focal_loss_ml(y_pred1, y1, alpha=self.alpha[1], gamma=self.gamma[1],\n alpha_t=self.alpha_t), dim=1)\n if self.hierarchical_ml[1]:\n loss_group1 = torch.mean(focal_loss_ml(y_group1,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(\n i, num_classes=self.num_groups1),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((x * (\n torch.tensor(\n self.label_groups1)+1)-1),\n dim=-1)[1:],\n y1.long())))),\n dim=0).float(),\n alpha=self.alpha[1], gamma=self.gamma[1],\n alpha_t=self.alpha_t), dim=1)\n else:\n loss_group1 = None\n\n loss3 = torch.mean(focal_loss_ml(y_pred3, y3, alpha=self.alpha[2], gamma=self.gamma[2],\n alpha_t=self.alpha_t), dim=1)\n if self.hierarchical_ml[2]:\n loss_group3 = torch.mean(focal_loss_ml(y_group3,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(\n i, num_classes=self.num_groups3),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((x * (\n torch.tensor(\n self.label_groups3)+1)-1),\n dim=-1)[1:],\n y3.long())))),\n dim=0).float(),\n alpha=self.alpha[2], gamma=self.gamma[2],\n alpha_t=self.alpha_t), dim=1)\n else:\n loss_group3 = None\n\n loss4 = torch.mean(focal_loss_ml(y_pred4, y4, alpha=self.alpha[3], gamma=self.gamma[3],\n alpha_t=self.alpha_t), dim=1)\n if self.hierarchical_ml[3]:\n loss_group4 = torch.mean(focal_loss_ml(y_group4,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(\n i, num_classes=self.num_groups4),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((x * (\n torch.tensor(\n self.label_groups4)+1)-1),\n dim=-1)[1:],\n y4.long())))),\n dim=0).float(),\n alpha=self.alpha[3], gamma=self.gamma[3],\n alpha_t=self.alpha_t), dim=1)\n else:\n loss_group4 = None\n\n else:\n loss0 = torch.mean(F.binary_cross_entropy(y_pred0, y0, reduction='none'), dim=1)\n if self.hierarchical_ml[0]:\n loss_group0 = torch.mean(F.binary_cross_entropy(y_group0,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(i,\n num_classes=\n self.num_groups0),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((\n x * (\n torch.tensor(\n self.label_groups0)\n + 1) - 1),\n dim=-1)[1:],\n y0.long()))\n )), dim=0).float(),\n reduction='none'), dim=1)\n else:\n loss_group0 = None\n\n loss1 = torch.mean(F.binary_cross_entropy(y_pred1, y1, reduction='none'), dim=1)\n if self.hierarchical_ml[1]:\n loss_group1 = torch.mean(F.binary_cross_entropy(y_group1,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(i,\n num_classes=\n self.num_groups1),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((\n x * (\n torch.tensor(\n self.label_groups1)\n + 1) - 1),\n dim=-1)[1:],\n y1.long()))\n )), dim=0).float(),\n reduction='none'), dim=1)\n else:\n loss_group1 = None\n\n loss3 = torch.mean(F.binary_cross_entropy(y_pred3, y3, reduction='none'), dim=1)\n if self.hierarchical_ml[2]:\n loss_group3 = torch.mean(F.binary_cross_entropy(y_group3,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(i,\n num_classes=\n self.num_groups3),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((\n x * (\n torch.tensor(\n self.label_groups3)\n + 1) - 1),\n dim=-1)[1:],\n y3.long()))\n )), dim=0).float(),\n reduction='none'), dim=1)\n else:\n loss_group3 = None\n\n loss4 = torch.mean(F.binary_cross_entropy(y_pred4, y4, reduction='none'), dim=1)\n if self.hierarchical_ml[3]:\n loss_group4 = torch.mean(F.binary_cross_entropy(y_group4,\n torch.cat(list(map(lambda i:\n torch.sum(\n F.one_hot(i,\n num_classes=\n self.num_groups4),\n dim=0).view(1, -1),\n list(map(lambda x: torch.unique((\n x * (\n torch.tensor(\n self.label_groups4)\n + 1) - 1),\n dim=-1)[1:],\n y4.long()))\n )), dim=0).float(),\n reduction='none'), dim=1)\n else:\n loss_group4 = None\n\n return loss0, loss1, loss2, loss3, loss4, (loss_group0, loss_group1, loss_group2, loss_group3, loss_group4)\n\n def forward(self, x, y0, y1, y2, y3, y4, reduction='sum'):\n loss0, loss1, loss2, loss3, loss4, loss_groups_tuple = self.get_loss(x, y0, y1, y2, y3, y4)\n loss_group0, loss_group1, loss_group2, loss_group3, loss_group4 = loss_groups_tuple\n\n loss_groups = 0\n if loss_group0 is not None:\n loss_groups += loss_group0 * self.weight_group0\n\n if loss_group1 is not None:\n loss_groups += loss_group1 * self.weight_group1\n\n if loss_group3 is not None:\n loss_groups += loss_group3 * self.weight_group3\n\n if loss_group4 is not None:\n loss_groups += loss_group4 * self.weight_group4\n\n if loss_group2 is not None:\n loss_groups += loss_group2 * self.weight_group\n\n if reduction == 'sum':\n return loss0 * self.weights[0] + loss1 * self.weights[1] + loss2 * self.weights[2] + \\\n loss3 * self.weights[3] + loss4 * self.weights[4] + loss_groups\n elif reduction == 'none':\n return loss0 * self.weights[0], loss1 * self.weights[1], loss2 * self.weights[2], \\\n loss3 * self.weights[3], loss4 * self.weights[4], loss_groups\n\n def get_concat_probs(self, x, return_hier_pred=False):\n if return_hier_pred:\n y_pred0, y_pred1, y_pred2, y_pred3, y_pred4, y_groups_tuples = self.get_probs(x)\n _, _, y_group2, _, _ = y_groups_tuples\n return torch.cat((y_pred0, y_pred1,\n F.one_hot(y_pred2.argmax(axis=-1), num_classes=self.num_labels[2]).float()[:, 1:],\n y_pred3, y_pred4), dim=1), \\\n F.one_hot(y_group2.argmax(axis=-1), num_classes=self.num_groups)\n else:\n y_pred0, y_pred1, y_pred2, y_pred3, y_pred4, _ = self.get_probs(x)\n\n return torch.cat((y_pred0, y_pred1,\n F.one_hot(y_pred2.argmax(axis=-1), num_classes=self.num_labels[2]).float()[:, 1:],\n y_pred3, y_pred4), dim=1)\n\n\ndef focal_loss_ml(inputs, targets, alpha=0.25, gamma=2, alpha_t=False):\n BCE_loss = F.binary_cross_entropy(inputs, targets, reduction='none')\n pt = torch.exp(-BCE_loss)\n if alpha_t:\n return (-alpha * (targets * 2 -1) + targets) * (1-pt)**gamma * BCE_loss\n else:\n return alpha * (1-pt)**gamma * BCE_loss\n\n\ndef focal_loss_mc(inputs, targets,\n num_classes, alpha=(0.25, 0.75, 0.75, 0.75, 0.75, 0.75), gamma=2, alpha_t=False):\n targets_one_hot = F.one_hot(targets, num_classes=num_classes)\n pt = inputs * targets_one_hot\n one_sub_pt = 1 - pt\n log_pt = targets_one_hot * torch.log(inputs + 1e-6)\n if alpha_t:\n return torch.sum((-torch.tensor(alpha))*one_sub_pt**gamma*log_pt, dim=-1)\n else:\n return torch.sum((-1)*one_sub_pt**gamma*log_pt, dim=-1)\n\n\ndef get_weight_mat(y, ratio=10, base=10):\n return (torch.ones(y.shape) - y) * (torch.sum(y, dim=-1).view(-1, 1) + base) \\\n * ratio / (y.shape[1] - torch.sum(y, dim=-1).view(-1, 1)).detach()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"trainer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport sys\nimport copy\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import fbeta_score\n\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import RandomSampler, SequentialSampler\nfrom tqdm import trange\n\n\nclass Trainer:\n def __init__(self, model, dataset, use_cuda=True,\n shuffle=True, epochs=100, extra_epochs_mc=1, train_mc_step=1, \n batch_size_train=64, batch_size_val=64, batch_size_all=64, batch_size_test=64,\n head_log=True,\n monitor_frequency=False, compute_acc=True, printout=False,\n dataloader_train_kwargs=dict(), dataloader_val_kwargs=dict(),\n dataloader_all_kwargs=dict(),\n dataloader_test_kwargs=dict()):\n self.model = model\n self.dataset = dataset\n self.train_val = bool(self.dataset.train_val_split)\n self.use_cuda = use_cuda\n if self.use_cuda and torch.cuda.is_available():\n torch.set_default_tensor_type(torch.cuda.FloatTensor)\n self.epochs = epochs\n self.extra_epochs_mc = extra_epochs_mc\n self.train_mc_step = train_mc_step\n self.head_log = head_log\n self.running_loss = []\n if self.head_log:\n self.loss_log = dict()\n self.loss_log['Head 0'] = []\n self.loss_log['Head 1'] = []\n self.loss_log['Head 2'] = []\n self.loss_log['Head 3'] = []\n self.loss_log['Head 4'] = []\n if self.model.hierarchical:\n self.loss_log['Head group classifiers'] = []\n\n if self.train_val:\n if shuffle:\n train_sampler = RandomSampler(dataset.train)\n val_sampler = RandomSampler(dataset.val)\n else:\n train_sampler = SequentialSampler(dataset.train)\n val_sampler = SequentialSampler(dataset.val)\n self.dataloader_train_kwargs = copy.deepcopy(dataloader_train_kwargs)\n self.dataloader_train_kwargs.update({'batch_size': batch_size_train, 'sampler': train_sampler})\n self.dataloader_train = DataLoader(self.dataset.train, **self.dataloader_train_kwargs)\n self.dataloader_val_kwargs = copy.deepcopy(dataloader_val_kwargs)\n self.dataloader_val_kwargs.update({'batch_size': batch_size_val, 'sampler': val_sampler})\n self.dataloader_val = DataLoader(self.dataset.val, **self.dataloader_val_kwargs)\n self.loss_history_train = []\n self.loss_history_val = []\n self.accuracy_history_train = []\n self.accuracy_history_val = []\n\n else:\n sampler = RandomSampler(dataset.all)\n self.dataloader_train_kwargs = copy.deepcopy(dataloader_train_kwargs)\n self.dataloader_train_kwargs.update({'batch_size': batch_size_train, 'sampler': sampler})\n self.dataloader_train = DataLoader(self.dataset.all, **self.dataloader_train_kwargs)\n self.loss_history_train = []\n self.accuracy_history_train = []\n\n all_sampler = SequentialSampler(dataset.all)\n self.dataloader_all_kwargs = copy.deepcopy(dataloader_all_kwargs)\n self.dataloader_all_kwargs.update({'batch_size': batch_size_all, 'sampler': all_sampler})\n self.dataloader_all = DataLoader(self.dataset.all, **self.dataloader_all_kwargs)\n\n if self.dataset.test_path is not None:\n test_sampler = SequentialSampler(dataset.test)\n self.dataloader_test_kwargs = copy.deepcopy(dataloader_test_kwargs)\n self.dataloader_test_kwargs.update({'batch_size': batch_size_test, 'sampler': test_sampler})\n self.dataloader_test = DataLoader(self.dataset.test, **self.dataloader_test_kwargs)\n\n self.use_cuda = use_cuda and torch.cuda.is_available()\n if self.use_cuda:\n self.model.cuda()\n self.monitor_frequency = monitor_frequency\n self.compute_acc = compute_acc\n self.printout = printout\n\n def before_iter(self):\n pass\n\n def after_iter(self):\n pass\n\n def train_mc(self, lr, parameters=None, betas=(0.9, 0.999), eps=1e-8, weight_decay=0,\n grad_clip=False, max_norm=1e-5):\n epochs = self.extra_epochs_mc\n self.model.classifiers['classifier2'].train()\n if self.model.hierarchical:\n self.model.group_classifier.train()\n params = filter(lambda x: x.requires_grad, self.model.parameters()) \\\n if parameters is None else parameters\n optim = torch.optim.Adam(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n for epoch in range(epochs):\n for x, _, _, y2, _, _ in self.dataloader_train:\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y2 = y2.cuda()\n loss = torch.mean(self.model.get_loss_mc(x, y2))\n optim.zero_grad()\n loss.backward()\n if grad_clip:\n torch.nn.utils.clip_grad_norm_(params, max_norm=max_norm)\n optim.step()\n \n def train(self, parameters=None, lr=1e-1, mc_lr=1e-1, betas=(0.9, 0.999), eps=1e-8, weight_decay=0,\n reduce_lr=False, step=5, gamma=0.8,\n reduce_lr_mc=False, step_mc=5, gamma_mc=0.8,\n grad_clip=False, max_norm=1e-5,\n train_mc_kwargs=dict()):\n epochs = self.epochs\n self.model.train()\n params = filter(lambda x: x.requires_grad, self.model.parameters()) \\\n if parameters is None else parameters\n optim = torch.optim.Adam(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n if bool(self.extra_epochs_mc):\n epoch_count_mc = 0\n with trange(epochs, desc='Training progress: ', file=sys.stdout) as progbar:\n for epoch_idx in progbar:\n self.before_iter()\n progbar.update(1)\n running_loss = 0\n if self.head_log:\n head_loss0 = 0\n head_loss1 = 0\n head_loss2 = 0\n head_loss3 = 0\n head_loss4 = 0\n if self.model.hierarchical:\n head_loss_group = 0\n for data_tensors in self.dataloader_train:\n data_tensor_tuples = [data_tensors]\n if self.head_log:\n if self.model.hierarchical:\n loss0, loss1, loss2, loss3, loss4, loss_group = self.loss(*data_tensor_tuples,\n head_log=self.head_log)\n loss = torch.mean(loss0 + loss1 + loss2 + loss3 + loss4 + loss_group)\n head_loss_group += torch.mean(loss_group).item()\n else:\n loss0, loss1, loss2, loss3, loss4, _ = self.loss(*data_tensor_tuples,\n head_log=self.head_log)\n loss = torch.mean(loss0 + loss1 + loss2 + loss3 + loss4)\n head_loss0 += torch.mean(loss0).item()\n head_loss1 += torch.mean(loss1).item()\n head_loss2 += torch.mean(loss2).item()\n head_loss3 += torch.mean(loss3).item()\n head_loss4 += torch.mean(loss4).item()\n else:\n loss = self.loss(*data_tensor_tuples)\n running_loss += loss.item()\n optim.zero_grad()\n loss.backward()\n if grad_clip:\n torch.nn.utils.clip_grad_norm_(params, max_norm=max_norm)\n optim.step()\n self.running_loss.append(running_loss/len(self.dataloader_train))\n if self.head_log:\n self.loss_log['Head 0'].append(head_loss0/len(self.dataloader_train))\n self.loss_log['Head 1'].append(head_loss1/len(self.dataloader_train))\n self.loss_log['Head 2'].append(head_loss2/len(self.dataloader_train))\n self.loss_log['Head 3'].append(head_loss3/len(self.dataloader_train))\n self.loss_log['Head 4'].append(head_loss4/len(self.dataloader_train))\n if self.model.hierarchical:\n self.loss_log['Head group classifiers'].append(head_loss_group / len(self.dataloader_train))\n\n if (epoch_idx+1) % step == 0 & reduce_lr:\n for p in optim.param_groups:\n p['lr'] *= gamma\n\n if bool(self.monitor_frequency):\n if (epoch_idx + 1) % self.monitor_frequency == 0:\n current_loss_train = self.compute_loss(tag='train')\n self.loss_history_train.append(current_loss_train)\n if self.compute_acc:\n current_accuracy_train = self.compute_accuracy(tag='train')\n self.accuracy_history_train.append(current_accuracy_train)\n if self.train_val:\n current_loss_val = self.compute_loss(tag='val')\n self.loss_history_val.append(current_loss_val)\n if self.compute_acc:\n current_accuracy_val = self.compute_accuracy(tag='val')\n self.accuracy_history_val.append(current_accuracy_val)\n if self.printout:\n print(\"After %i epochs, loss is %f and prediction accuracy is %f.\"\n % (epoch_idx, current_loss_train, current_accuracy_train))\n if bool(self.extra_epochs_mc) & (epoch_idx + 1) % self.train_mc_step == 0:\n self.train_mc(lr=mc_lr, **train_mc_kwargs)\n epoch_count_mc += 1\n if epoch_count_mc % step_mc == 0 & reduce_lr_mc:\n mc_lr *= gamma_mc\n \n self.after_iter()\n\n def loss(self, data_tensors, head_log=False):\n x, y0, y1, y2, y3, y4 = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y0 = y0.cuda()\n y1 = y1.cuda()\n y2 = y2.cuda()\n y3 = y3.cuda()\n y4 = y4.cuda()\n if head_log:\n loss0, loss1, loss2, loss3, loss4, loss_group = self.model(x, y0, y1, y2, y3, y4, reduction='none')\n return loss0, loss1, loss2, loss3, loss4, loss_group\n\n else:\n loss = torch.mean(self.model(x, y0, y1, y2, y3, y4, reduction='sum'))\n return loss\n\n @torch.no_grad()\n def plot_running_loss(self, epochs_override=None):\n len_ticks = len(self.running_loss)\n if epochs_override is None:\n x_axis = np.linspace(1, self.epochs, len_ticks)\n else:\n x_axis = np.linspace(1, epochs_override, len_ticks)\n plt.figure()\n plt.plot(x_axis, self.running_loss)\n plt.xlabel('Number of epochs')\n plt.ylabel('Estimated loss')\n plt.show()\n\n @torch.no_grad()\n def plot_head_loss(self, epochs_override=None):\n if not self.head_log:\n pass\n else:\n len_ticks = len(self.running_loss)\n if epochs_override is None:\n x_axis = np.linspace(1, self.epochs, len_ticks)\n else:\n x_axis = np.linspace(1, epochs_override, len_ticks)\n plt.figure()\n for key in self.loss_log.keys():\n log_array = np.array(self.loss_log[key])\n plt.plot(x_axis, log_array/log_array[0], label=key)\n plt.legend(loc='lower left', bbox_to_anchor=(1, 0.25, 0.5, 0.5))\n plt.xlabel('Number of epochs')\n plt.ylabel('Estimated loss')\n plt.show()\n\n @torch.no_grad()\n def compute_loss(self, tag):\n self.model.eval()\n loss_sum = 0\n if tag == 'train':\n _dataloader = self.dataloader_train\n elif tag == 'val':\n _dataloader = self.dataloader_val\n elif tag == 'all':\n _dataloader = self.dataloader_all\n else:\n raise ValueError('Invalid tag!')\n _dataset = _dataloader.dataset\n for data_tensors in _dataloader:\n x, y0, y1, y2, y3, y4 = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y0 = y0.cuda()\n y1 = y1.cuda()\n y2 = y2.cuda()\n y3 = y3.cuda()\n y4 = y4.cuda()\n loss = self.model(x, y0, y1, y2, y3, y4)\n loss_sum += torch.sum(loss).item()\n loss_mean = loss_sum / len(_dataset)\n self.model.train()\n return loss_mean\n\n @torch.no_grad()\n def loss_history_plot(self, epochs_override=None):\n len_ticks = len(self.loss_history_train)\n if epochs_override is None:\n x_axis = np.linspace(1, self.epochs, len_ticks)\n else:\n x_axis = np.linspace(1, epochs_override, len_ticks)\n plt.figure()\n if self.train_val:\n assert (len(self.loss_history_train) == len(self.loss_history_val))\n plt.plot(x_axis, self.loss_history_train, label='Training set')\n plt.plot(x_axis, self.loss_history_val, label='Validation set')\n plt.legend()\n else:\n plt.plot(x_axis, self.loss_history_train)\n\n plt.xlabel('Number of epochs')\n plt.ylabel('Loss')\n plt.show()\n\n @torch.no_grad()\n def get_probs(self, tag, return_hier_pred=False, return_probs_only=False):\n self.model.eval()\n predictions_tem = []\n if return_hier_pred:\n hier_pred_tem = []\n if tag == 'test':\n _dataloader = self.dataloader_test\n for data_tensors in _dataloader:\n x = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y_concat_prob = self.model.get_concat_probs(x)\n predictions_tem += [y_concat_prob]\n predictions_array = torch.cat(predictions_tem).detach().cpu().numpy()\n return predictions_array\n else:\n ground_truth = []\n if tag == 'train':\n _dataloader = self.dataloader_train\n elif tag == 'val':\n _dataloader = self.dataloader_val\n elif tag == 'all':\n _dataloader = self.dataloader_all\n else:\n raise ValueError('Invalid tag!')\n for data_tensors in _dataloader:\n if not return_probs_only:\n x, y0, y1, y2, y3, y4 = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y0 = y0.cuda()\n y1 = y1.cuda()\n y2 = y2.cuda()\n y3 = y3.cuda()\n y4 = y4.cuda()\n ground_truth += [torch.cat((y0.long(),\n y1.long(),\n F.one_hot(y2, num_classes=6).squeeze()[:, 1:].long(),\n y3.long(),\n y4.long()), dim=1)]\n else:\n x = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n if return_hier_pred:\n y_concat_prob, y_group_pred = self.model.get_concat_probs(x, return_hier_pred=return_hier_pred)\n hier_pred_tem += [y_group_pred]\n else:\n y_concat_prob = self.model.get_concat_probs(x, return_hier_pred=return_hier_pred)\n predictions_tem += [y_concat_prob]\n predictions_array = torch.cat(predictions_tem).detach().cpu().numpy()\n if return_hier_pred:\n hier_pred = torch.cat(hier_pred_tem).detach().cpu().numpy()\n self.model.train()\n if not return_probs_only:\n if return_hier_pred:\n return torch.cat(ground_truth).detach().cpu().numpy(), predictions_array, hier_pred\n else:\n return torch.cat(ground_truth).detach().cpu().numpy(), predictions_array\n else:\n if return_hier_pred:\n return predictions_array, hier_pred\n else:\n return predictions_array\n\n @torch.no_grad()\n def make_predictions(self, tag, return_pred_only=False,\n thre=(0.08, 0.08, 0.08, 0.08), upper_bound=(3, 4, 17, 18), lower_bound=3,\n boundary=([0, 100], [100, 781], [786, 2706], [2706, 3474])):\n self.model.eval()\n predictions_tem = []\n if tag == 'test':\n _dataloader = self.dataloader_test\n for data_tensors in _dataloader:\n x = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y_concat_pred = regularized_pred(self.model.get_concat_probs(x).detach().cpu().numpy(),\n thre=thre,\n upper_bound=upper_bound, lower_bound=lower_bound, boundary=boundary)\n predictions_tem += [y_concat_pred]\n predictions_array = np.concatenate(predictions_tem)\n return predictions_array\n else:\n ground_truth = []\n if tag == 'train':\n _dataloader = self.dataloader_train\n elif tag == 'val':\n _dataloader = self.dataloader_val\n elif tag == 'all':\n _dataloader = self.dataloader_all\n else:\n raise ValueError('Invalid tag!')\n for data_tensors in _dataloader:\n if not return_pred_only:\n x, y0, y1, y2, y3, y4 = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y0 = y0.cuda()\n y1 = y1.cuda()\n y2 = y2.cuda()\n y3 = y3.cuda()\n y4 = y4.cuda()\n ground_truth += [torch.cat((y0.long(),\n y1.long(),\n F.one_hot(y2, num_classes=6).squeeze()[:, 1:].long(),\n y3.long(),\n y4.long()), dim=1)]\n else:\n x = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y_concat_pred = regularized_pred(self.model.get_concat_probs(x).detach().cpu().numpy(),\n thre=thre,\n upper_bound=upper_bound, lower_bound=lower_bound, boundary=boundary)\n predictions_tem += [y_concat_pred]\n predictions_array = np.concatenate(predictions_tem)\n self.model.train()\n if not return_pred_only:\n return torch.cat(ground_truth).detach().cpu().numpy(), predictions_array\n else:\n return predictions_array\n\n @torch.no_grad()\n def compute_hier_acc(self, tag):\n ground_truth = []\n hier_pred_tem = []\n if tag == 'train':\n _dataloader = self.dataloader_train\n elif tag == 'val':\n _dataloader = self.dataloader_val\n elif tag == 'all':\n _dataloader = self.dataloader_all\n else:\n raise ValueError('Invalid tag!')\n for data_tensors in _dataloader:\n x, _, _, y2, _, _ = data_tensors\n if self.use_cuda and torch.cuda.is_available():\n x = x.cuda()\n y2 = y2.cuda()\n gt_oh = F.one_hot(y2, num_classes=6).squeeze()[:, 1:].long()\n ground_truth += [torch.sum(gt_oh, dim=-1)]\n if self.model.hierarchical:\n _, y_group_pred = self.model.get_concat_probs(x, return_hier_pred=True)\n hier_pred_tem += [y_group_pred]\n else:\n y_concat_prob = self.model.get_concat_probs(x, return_hier_pred=False)\n hier_pred_tem += [torch.sum(y_concat_prob[:, 781:786], dim=-1)]\n if self.model.hierarchical:\n return np.mean(torch.cat(ground_truth).detach().cpu().numpy() ==\n np.where(torch.cat(hier_pred_tem).detach().cpu().numpy() == 1)[1])\n else:\n return np.mean(torch.cat(ground_truth).detach().cpu().numpy() ==\n torch.cat(hier_pred_tem).detach().cpu().numpy())\n\n @torch.no_grad()\n def compute_accuracy(self, tag,\n thre=(0.08, 0.08, 0.08, 0.08), upper_bound=(3, 4, 17, 18), lower_bound=3,\n boundary=([0, 100], [100, 781], [786, 2706], [2706, 3474])):\n y_true, y_pred = self.make_predictions(tag=tag, thre=thre,\n upper_bound=upper_bound, lower_bound=lower_bound, boundary=boundary)\n f_beta = [fbeta_score(y_true[i, :], y_pred[i, :], beta=2) for i in range(y_true.shape[0])]\n return sum(f_beta) / len(f_beta)\n\n @torch.no_grad()\n def accuracy_history_plot(self, epochs_override=None):\n len_ticks = len(self.accuracy_history_train)\n if epochs_override is None:\n x_axis = np.linspace(1, self.epochs, len_ticks)\n else:\n x_axis = np.linspace(1, epochs_override, len_ticks)\n plt.figure()\n if self.train_val:\n assert (len(self.accuracy_history_train) == len(self.accuracy_history_val))\n plt.plot(x_axis, self.accuracy_history_train, label='Training set')\n plt.plot(x_axis, self.accuracy_history_val, label='Validation set')\n plt.legend()\n else:\n plt.plot(x_axis, self.accuracy_history_train)\n\n plt.xlabel('Number of epochs')\n plt.ylabel('Accuracy')\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The inference section will use trained model to predict labels in test set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's define the dataset, model and trainer.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"_root_path = '/kaggle/input/imet-2020-fgvc7'\npath = f'{_root_path}/train'\ntest_path = f'{_root_path}/test'\ndata_info_path = f'{_root_path}/train.csv'\nlabels_info_path = f'{_root_path}/labels.csv'\ntest_csv_path = f'{_root_path}/sample_submission.csv'\n\ndataset = TrainValSet(train_val_split=False, train_transform='val',\n path=path, data_info_path=data_info_path, labels_info_path=labels_info_path,\n test_path=test_path, test_csv_path=test_csv_path)\n\ntag='50'\nweight_path = f'/kaggle/input/resnet{tag}/resnet{tag}.pth'\nmodel = ArtCV(tag=tag, weight_path=weight_path, freeze_cnn=True, dropout_rate=0, weights=(1, 1, 10, 1, 1),\n classifier_layers=(2,2,2,2,2), focal_loss=False, \n alpha=(0.25, 0.25, 0.25, 0.25), alpha_mc=(0.25, 0.75, 0.75, 0.75, 0.75, 0.75),\n gamma_mc=2, gamma=(2, 2, 2, 2), alpha_t=True,\n hierarchical=True, label_groups=(0, 1, 2, 1, 1, 1), alpha_group=(1, 1), gamma_group=2, weight_group=5,\n hierarchical_ml=(False, True, True, True), label_groups3=_label_groups3_40)\n\ntrainer = Trainer(model, dataset, batch_size_train=256, batch_size_val=256, batch_size_all=64,\n epochs=15, extra_epochs_mc=0, head_log=True, compute_acc=False,\n monitor_frequency=False,\n dataloader_train_kwargs={'num_workers':2}, dataloader_val_kwargs={'num_workers':2},\n dataloader_all_kwargs={'num_workers':2})\n\nfile_name = 'best0526/frozen_hier_all_no_crop_resnet50_2layer_12_epochs_reduced_lr.model.pkl'\nsave_path = f'/kaggle/input/{file_name}'\nmodel.load_state_dict(torch.load(save_path))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's evaluate the results.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_array=trainer.make_predictions(tag='test', thre=(0.1, 0.1, 0.1, 0.1), upper_bound=(3, 4, 10, 10), lower_bound=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(f'{_root_path}/sample_submission.csv')\nfor i, one_hot in enumerate(predictions_array):\n ids = np.where(one_hot)[0]\n submission.iloc[i].attribute_ids = ' '.join([str(x) for x in ids])\n\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Acknowledge\n
If you find this notebook helpful, please upvote.\n
Team member: \n
[yunchenyang](https://www.kaggle.com/yunchenyang), email: yunchenyang@hotmail.com; \n
[ytisserant](https://www.kaggle.com/ytisserant), email: ytisserant@gmail.com","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}