{"cells":[{"metadata":{},"cell_type":"markdown","source":"## 1. Dataset Visualization\n### 1) Goal\nBefore constructing the classification model, let's visualize the dataset and define the task in the first place. <br>\n<b>Step 1:</b> Load and check the labels and training data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nroot_path = '/kaggle/input/imet-2020-fgvc7/'\nlabels = pd.read_csv(root_path+'labels.csv')\ndata = pd.read_csv(root_path+'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"labels.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<b>Step 2:</b> Split the attribute names into catagories and types.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def _split_attr_names(attr_name_str):\n    return [item.strip() for item in attr_name_str.split('::')] \n\ndef _split_labels(labels_dataframe):\n    split_labels_dict = dict(attribute_id=[], attr_tier1=[], attr_tier2=[])\n    for i in range(labels_dataframe.shape[0]):\n        tem = _split_attr_names(labels['attribute_name'][i])\n        split_labels_dict['attribute_id'].append(labels['attribute_id'][i])\n        split_labels_dict['attr_tier1'].append(tem[0])\n        split_labels_dict['attr_tier2'].append(tem[1]) \n    split_labels = pd.DataFrame(split_labels_dict, \n                                columns=['attribute_id', 'attr_tier1', 'attr_tier2'])\n    return split_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"split_labels = _split_labels(labels)\nsplit_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"catagories = sorted(list(set(list(split_labels['attr_tier1']))))\nprint(len(catagories))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are in total 5 catagories: %s, %s, %s, %s, %s.'\n     %(catagories[0],\n       catagories[1],\n       catagories[2],\n       catagories[3],\n       catagories[4]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<b>Step 3:</b> Define the types of classification tasks.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def label_indexer_coarse(labels_dataframe):\n    split_labels_dict = dict(attribute_id=[], attr_tier1=[], attr_tier2=[])\n    for i in range(labels_dataframe.shape[0]):\n        tem = [item.strip() for item in labels_dataframe['attribute_name'][i].split('::')] \n        split_labels_dict['attribute_id'].append(labels_dataframe['attribute_id'][i])\n        split_labels_dict['attr_tier1'].append(tem[0])\n        split_labels_dict['attr_tier2'].append(tem[1])\n        \n    split_labels = pd.DataFrame(split_labels_dict, \n                                columns=['attribute_id', 'attr_tier1', 'attr_tier2'])\n    \n    tier1 = dict()\n    tier2 = dict()\n    attr2indexing = dict()\n    indexing2attr = dict()\n    label_indexing_list = []\n    for i_1, item1 in enumerate(sorted(list(set(list(split_labels['attr_tier1']))))):\n        assert len(list(set(list(split_labels['attr_tier2'][split_labels['attr_tier1']==item1])))) \\\n        == len(list(split_labels['attr_tier2'][split_labels['attr_tier1']==item1]))\n        tier1[item1] = i_1\n        tier2[item1] = dict()\n        list_tem = list(split_labels['attr_tier2'][split_labels['attr_tier1']==item1])\n        for i_2, item2 in enumerate(list_tem):\n            tier2[item1][item2] = i_2 + 1\n    for idx in range(split_labels.shape[0]):\n        tier1_idx = tier1[split_labels['attr_tier1'][idx]]\n        tier2_idx = tier2[split_labels['attr_tier1'][idx]][split_labels['attr_tier2'][idx]]\n        label_indexing_list.append([tier1_idx, tier2_idx])\n        attr2indexing[split_labels['attribute_id'][idx]] = [tier1_idx, tier2_idx]\n        indexing2attr[str([tier1_idx, tier2_idx])] = split_labels['attribute_id'][idx]\n    labels_indexing_df = split_labels.copy()\n    labels_indexing_df['indexing'] = label_indexing_list\n    return labels_indexing_df, attr2indexing, indexing2attr\nlabels_indexing_df, attr2indexing, indexing2attr = label_indexer_coarse(labels)\n\ncount_array = np.zeros((data.shape[0], 6), dtype='int')\nfor i, attr_ids in enumerate(data['attribute_ids']):\n    cata_indexing = [labels_indexing_df['indexing'][int(attr_id)][0] \n                     for attr_id in data['attribute_ids'][i].split()]\n    for i_ in range(count_array.shape[1]):\n        count_array[i, i_] = cata_indexing.count(i_)\nfor i_tier1, tier1 in enumerate(sorted(list(set(list(split_labels['attr_tier1']))))):\n    print('Catagory: %s; minimum count of tagging per item: %s; maximum count of tagging per item: %s' \n          %(tier1, np.amin(count_array, axis=0)[i_tier1], np.amax(count_array, axis=0)[i_tier1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check the sparsity of dataset in multi-label classification tasks.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"label_count_array = np.zeros((len(labels['attribute_id'])), dtype='int')\nfor i in range(len(data['attribute_ids'])):\n    for label in [int(attr_id) for attr_id in data['attribute_ids'][i].split()]:\n        label_count_array[label] += 1\nprint('The average percentage of positive labels in catagory country is %s' %np.mean(label_count_array[0: 100]/len(data)))\nprint('The average percentage of positive labels in catagory culture is %s' %np.mean(label_count_array[100: 781]/len(data)))\nprint('The average percentage of positive labels in catagory medium is %s' %np.mean(label_count_array[786: 2706]/len(data)))\nprint('The average percentage of positive labels in catagory tags is %s' %np.mean(label_count_array[2706: 3474]/len(data)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check the percentage of each class in the multi-class classification task.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"_percent = np.array([1-np.sum(label_count_array[781: 786]/len(data)), label_count_array[781: 786][0]/len(data),\n label_count_array[781: 786][1]/len(data), label_count_array[781: 786][2]/len(data), \n label_count_array[781: 786][3]/len(data), label_count_array[781: 786][4]/len(data)])\nprint('Unknown dimension: %s; %s: %s; %s: %s; %s: %s; %s: %s; %s: %s' \n      %(_percent[0], \n        split_labels['attr_tier2'][781], _percent[1],\n        split_labels['attr_tier2'][782], _percent[2], \n        split_labels['attr_tier2'][783], _percent[3],\n        split_labels['attr_tier2'][784], _percent[4],\n        split_labels['attr_tier2'][785], _percent[5]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally, let's check the minimum and maximum amount of attributes per item.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nprint('The minimum amount of attributes of the images: %s'\n      %min([len(data['attribute_ids'][i].split()) for i in range(len(data['attribute_ids']))]))\nprint('The maximum amount of attributes of the images: %s'\n      %max([len(data['attribute_ids'][i].split()) for i in range(len(data['attribute_ids']))]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2) Tasks\nAs the counting results shown, an object can only have one single label in catagory \"dimension\", but it may have multiple labels in another 4 catagories.<br>\nHence, we define the classification task of catagory \"dimension\" as multi-class classification and the tasks of the other 4 catagories as multi-label classification. <br>\nWe decided to train a multi-head CNN model to complete this task. An pretrained ResNet architecture will co-train with 5 classifier heads. The second classifier will perform the multi-class classification (In total 6 classes; 0 means no label being eligible). The other 4 classifiers will perform multi-label classification.  ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 2. Model constructing and training\nlet's start constructing the classification model.\n### 1) Library and codes\nThe codes are also available on GitHub (https://github.com/yunchen-yang/artcv).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### Utils","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nfrom torchvision.transforms import Compose, Resize, RandomResizedCrop, Normalize, ToTensor\nimport pandas as pd\nimport torch\nfrom sklearn.metrics import fbeta_score\n\n\ndef label_indexer_fine(labels_dataframe):\n    split_labels_dict = dict(attribute_id=[], attr_tier1=[], attr_tier2=[], attr_tier3=[])\n    for i in range(labels_dataframe.shape[0]):\n        tem = [item.strip() for split_list in [item_.split(';')\n                                               for item_ in labels_dataframe['attribute_name'][i].split('::')]\n               for item in split_list]\n        split_labels_dict['attribute_id'].append(labels_dataframe['attribute_id'][i])\n        split_labels_dict['attr_tier1'].append(tem[0])\n        split_labels_dict['attr_tier2'].append(tem[1])\n        try:\n            split_labels_dict['attr_tier3'].append(tem[2])\n        except:\n            split_labels_dict['attr_tier3'].append('None')\n    split_labels = pd.DataFrame(split_labels_dict,\n                                columns=['attribute_id', 'attr_tier1', 'attr_tier2', 'attr_tier3'])\n\n    tier1 = dict()\n    tier2 = dict()\n    counting_dict = dict()\n    attr2indexing = dict()\n    indexing2attr = dict()\n    label_indexing_list = []\n    for i_1, item1 in enumerate(sorted(list(set(list(split_labels['attr_tier1']))))):\n        tier1[item1] = i_1\n        tier2[item1] = dict()\n        list_tem = sorted(list(set(list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1]))))\n        for i_2, item2 in enumerate(list_tem):\n            tier2[item1][item2] = i_2 + 1\n        counting_dict[item1] = np.ones(len(list_tem), dtype='int')\n    for idx in range(split_labels.shape[0]):\n        tier1_idx = tier1[split_labels['attr_tier1'][idx]]\n        tier2_idx = tier2[split_labels['attr_tier1'][idx]][split_labels['attr_tier2'][idx]]\n        tier3_idx = counting_dict[split_labels['attr_tier1'][idx]][tier2_idx - 1]\n        counting_dict[split_labels['attr_tier1'][idx]][tier2_idx - 1] += 1\n        label_indexing_list.append([tier1_idx, tier2_idx, tier3_idx])\n        attr2indexing[split_labels['attribute_id'][idx]] = [tier1_idx, tier2_idx, tier3_idx]\n        indexing2attr[str([tier1_idx, tier2_idx, tier3_idx])] = split_labels['attribute_id'][idx]\n    labels_indexing_df = split_labels.copy()\n    labels_indexing_df['indexing'] = label_indexing_list\n    return labels_indexing_df, attr2indexing, indexing2attr\n\n\ndef label_indexer_coarse(labels_dataframe):\n    split_labels_dict = dict(attribute_id=[], attr_tier1=[], attr_tier2=[])\n    for i in range(labels_dataframe.shape[0]):\n        tem = [item.strip() for item in labels_dataframe['attribute_name'][i].split('::')]\n        split_labels_dict['attribute_id'].append(labels_dataframe['attribute_id'][i])\n        split_labels_dict['attr_tier1'].append(tem[0])\n        split_labels_dict['attr_tier2'].append(tem[1])\n\n    split_labels = pd.DataFrame(split_labels_dict,\n                                columns=['attribute_id', 'attr_tier1', 'attr_tier2'])\n\n    tier1 = dict()\n    tier2 = dict()\n    attr2indexing = dict()\n    indexing2attr = dict()\n    label_indexing_list = []\n    for i_1, item1 in enumerate(sorted(list(set(list(split_labels['attr_tier1']))))):\n        assert len(list(set(list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1])))) \\\n               == len(list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1]))\n        tier1[item1] = i_1\n        tier2[item1] = dict()\n        list_tem = list(split_labels['attr_tier2'][split_labels['attr_tier1'] == item1])\n        for i_2, item2 in enumerate(list_tem):\n            tier2[item1][item2] = i_2\n    for idx in range(split_labels.shape[0]):\n        tier1_idx = tier1[split_labels['attr_tier1'][idx]]\n        tier2_idx = tier2[split_labels['attr_tier1'][idx]][split_labels['attr_tier2'][idx]]\n        label_indexing_list.append([tier1_idx, tier2_idx])\n        attr2indexing[split_labels['attribute_id'][idx]] = [tier1_idx, tier2_idx]\n        indexing2attr[str([tier1_idx, tier2_idx])] = split_labels['attribute_id'][idx]\n    labels_indexing_df = split_labels.copy()\n    labels_indexing_df['indexing'] = label_indexing_list\n    return labels_indexing_df, attr2indexing, indexing2attr\n\n\ndef imgreader(img_id, ext, path, attr_ids, attr2indexing, length_list, dimension=256,\n              task=('ml', 'ml', 'mc', 'ml', 'ml'), transform='val', grey_scale=False):\n    file_path = f'{path}/{img_id}.{ext}'\n    with open(file_path, 'rb') as f:\n        img_ = Image.open(f)\n        if grey_scale:\n            img = img_.convert('L')\n        else:\n            img = img_.convert('RGB')\n\n    transformer = {\n        'train': Compose([RandomResizedCrop(size=(dimension, dimension)),\n                          ToTensor(),\n                          Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]\n                         ),\n        'val': Compose([Resize(size=(dimension, dimension)),\n                        ToTensor(),\n                        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n    }\n    x = transformer[transform](img)\n    y_list = [attr2indexing[int(attr_id)] for attr_id in attr_ids.split()]\n    y_dict = labels_list2array(y_list, length_list, task)\n    return x, tuple(y_dict.values())\n\n\ndef imgreader_test(file_path, dimension=256, transform='val', grey_scale=False):\n    with open(file_path, 'rb') as f:\n        img_ = Image.open(f)\n        if grey_scale:\n            img = img_.convert('L')\n        else:\n            img = img_.convert('RGB')\n\n    transformer = {\n        'train': Compose([RandomResizedCrop(size=(dimension, dimension)),\n                          ToTensor(),\n                          Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]\n                         ),\n        'val': Compose([Resize(size=(dimension, dimension)),\n                        ToTensor(),\n                        Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])\n    }\n    x = transformer[transform](img)\n\n    return x\n\n\ndef counting_elements(labels_indexing_df):\n    return [len(labels_indexing_df['indexing'][labels_indexing_df['attr_tier1']==catagory])\n            for catagory in sorted(list(set(list(labels_indexing_df['attr_tier1']))))]\n\n\ndef labels_list2array(y_list, length_list, task):\n    y_dict = dict()\n    for i in range(len(task)):\n        if task[i] != 'mc':\n            y_dict[i] = torch.FloatTensor(np.zeros(length_list[i]))\n        else:\n            y_dict[i] = torch.LongTensor([0])\n\n    for idx_list in y_list:\n        if task[idx_list[0]] != 'mc':\n            y_dict[idx_list[0]][idx_list[1]] = 1\n        else:\n            y_dict[idx_list[0]][0] = idx_list[1]+1\n\n    return y_dict\n\n\ndef image_list_scan(data_info, indices):\n    if indices is None:\n        return list(data_info['id']), list(data_info['attribute_ids'])\n    else:\n        return list(data_info['id'][indices]), list(data_info['attribute_ids'][indices])\n\n\ndef f2score(ground_truth, pred, return_mean=True):\n    f_beta = [fbeta_score(ground_truth[i,:], pred[i,:], beta=2) for i in range(ground_truth.shape[0])]\n    if return_mean:\n        return sum(f_beta)/len(f_beta)\n    else:\n        return f_beta\n\n    \ndef regularized_pred(probs, thre, upper_bound=(3, 4, 17, 18), lower_bound=3,\n                     boundary=([0, 100], [100, 781], [786, 2706], [2706, 3474])):\n    thres_array = np.ones((probs.shape[1]), dtype='float')\n    pred = dict()\n    for i in range(len(boundary)):\n        thres_array[boundary[i][0]: boundary[i][1]] = thre[i]\n        probs_tem = probs[:, boundary[i][0]: boundary[i][1]]/thre[i]\n        mask_tem = np.zeros(probs_tem.shape, dtype='float')\n        max_args = probs_tem.argsort(axis=-1)[:,::-1][:, :upper_bound[i]]\n        for i_ in range(mask_tem.shape[0]):\n            mask_tem[i_, :][max_args[i_, :]] = 1\n        probs_tem *= mask_tem\n        probs_tem[probs_tem>=1] = 1\n        probs_tem[probs_tem<1] = 0\n        pred[i] = probs_tem  \n    pred_array = np.concatenate((pred[0], pred[1], \n                                 probs[:, boundary[1][1]: boundary[2][0]], pred[2], pred[3]), axis=-1)\n    no_label = np.where(pred_array.max(axis=-1)==0)[0]\n    if no_label.shape[0] != 0:\n        for idx in no_label:\n            _max_args = (probs[idx, :]/thres_array).argsort(axis=-1)[::-1][:lower_bound]\n            pred_array[idx, :][_max_args] = 1\n    return pred_array","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### datatool","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset\nimport sys\nimport math\nfrom glob import glob\n\n\nclass ImgDataset(Dataset):\n    def __init__(self, x, y, path, attr2indexing, length_list, task,\n                 ext='png', dimension=256, transform='val', grey_scale=False):\n        super().__init__()\n        self.x = x\n        self.y = y\n        self.path = path\n        self.attr2indexing =attr2indexing\n        self.length_list = length_list\n        self.task = task\n        self.ext = ext\n        self.dimension = dimension\n        self.transform = transform\n        self.grey_scale = grey_scale\n\n    def __len__(self):\n        return len(self.x)\n\n    def __getitem__(self, index):\n        img, ys = imgreader(self.x[index], self.ext, self.path, self.y[index],\n                            self.attr2indexing, self.length_list,\n                            self.dimension, self.task, self.transform, self.grey_scale)\n        y0, y1, y2, y3, y4 = ys\n\n        return img, y0, y1, y2, y3, y4\n\n\nclass ImgTestset(Dataset):\n    def __init__(self, x, dimension=256, transform='val', grey_scale=False):\n        super().__init__()\n        self.x = x\n\n        self.dimension = dimension\n        self.transform = transform\n        self.grey_scale = grey_scale\n\n    def __len__(self):\n        return len(self.x)\n\n    def __getitem__(self, index):\n        img = imgreader_test(self.x[index], self.dimension, self.transform, self.grey_scale)\n\n        return img\n\n\nclass TrainValSet:\n    def __init__(self, ext='png', path=None, indices=None, dimension=256, data_info_path=None, labels_info_path=None,\n                 task=('ml', 'ml', 'mc', 'ml', 'ml'), train_transform='train', train_val_split=0.7, seed=0,\n                 test_path=None, test_csv_path=None):\n        super().__init__()\n        self.ext = ext\n        self.dimension = dimension\n        self.indices = indices\n        self.seed = seed\n        self.train_transform = train_transform\n\n        if path is None:\n            self.path = f'{sys.path[0]}/train'\n        else:\n            self.path = path\n\n        self.test_path = test_path\n\n        if data_info_path is None:\n            self.data_info_path = f'{sys.path[0]}/train.csv'\n        else:\n            self.data_info_path = data_info_path\n\n        if labels_info_path is None:\n            self.labels_info_path = f'{sys.path[0]}/labels.csv'\n        else:\n            self.labels_info_path = labels_info_path\n\n        self.labels_info = pd.read_csv(self.labels_info_path)\n        self.data_info = pd.read_csv(self.data_info_path)\n\n        self.labels_indexing_df, self.attr2indexing, self.indexing2attr = label_indexer_coarse(self.labels_info)\n        self.length_list = counting_elements(self.labels_indexing_df)\n        self.X_all, self.Y_all = image_list_scan(self.data_info, indices=self.indices)\n        self.task = task\n\n        if self.test_path is not None:\n            self.test_csv_path = test_csv_path\n            if self.test_csv_path is not None:\n                self.test_csv = pd.read_csv(self.test_csv_path)\n                self.X_test = [f'{self.test_path}/{_filename}.{self.ext}' for _filename in list(self.test_csv['id'])]\n            else:\n                self.X_test = glob(f'{self.test_path}/*.{self.ext}', recursive=True)\n            self.test = ImgTestset(self.X_test, dimension=256, transform='val', grey_scale=False)\n\n        self.train_val_split = train_val_split\n\n        if bool(self.train_val_split):\n            self.all = ImgDataset(self.X_all, self.Y_all, self.path, self.attr2indexing, self.length_list,\n                                  task=self.task, ext=self.ext, dimension=256, transform='val', grey_scale=False)\n            assert(0 < self.train_val_split < 1)\n            num_train = math.ceil(len(self.X_all) * self.train_val_split)\n            np.random.seed(seed=self.seed)\n            indices_array = np.random.permutation(len(self.X_all))\n            self.X_train = [self.X_all[i] for i in indices_array[:num_train]]\n            self.Y_train = [self.Y_all[i] for i in indices_array[:num_train]]\n            self.train = ImgDataset(self.X_train, self.Y_train, self.path, self.attr2indexing, self.length_list,\n                                    task=self.task, ext=self.ext, dimension=256,\n                                    transform=self.train_transform, grey_scale=False)\n            self.X_val = [self.X_all[i] for i in indices_array[num_train:]]\n            self.Y_val = [self.Y_all[i] for i in indices_array[num_train:]]\n            self.val = ImgDataset(self.X_val, self.Y_val, self.path, self.attr2indexing, self.length_list,\n                                  task=self.task, ext=self.ext, dimension=256, transform='val', grey_scale=False)\n        else:\n            if self.test_path is not None:\n                self.all = ImgDataset(self.X_all, self.Y_all, self.path, self.attr2indexing, self.length_list,\n                                      task=self.task, ext=self.ext, dimension=256,\n                                      transform=self.train_transform, grey_scale=False)\n            else:\n                self.all = ImgDataset(self.X_all, self.Y_all, self.path, self.attr2indexing, self.length_list,\n                                      task=self.task, ext=self.ext, dimension=256, transform='val', grey_scale=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### modules","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.models.resnet import ResNet\nimport torch\nfrom torch import nn as nn\nimport collections\nimport torch.nn.functional as F\n\n\nclass ResNet_CNN(ResNet):\n    def __init__(self, block, layers, weight_path, freeze_layers, **kwargs):\n        super().__init__(block, layers, **kwargs)\n        self.weight_path = weight_path\n        if type(freeze_layers) == bool and freeze_layers:\n            self.freeze_layers = 4\n        else:\n            self.freeze_layers = freeze_layers\n        if self.weight_path is not None:\n            self.load_state_dict(torch.load(self.weight_path))\n        del self.fc\n        if bool(self.freeze_layers):\n            self._freeze_layers()\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x) if self.freeze_layers != 1 else self.layer1(x).detach()\n        x = self.layer2(x) if self.freeze_layers != 2 else self.layer2(x).detach()\n        x = self.layer3(x) if self.freeze_layers != 3 else self.layer3(x).detach()\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n\n        if self.freeze_layers != 4:\n            return x\n        else:\n            return x.detach()\n\n    def _freeze_layers(self):\n        _bool = False\n        for name, module in self.named_children():\n            if name == f'layer{self.freeze_layers+1}' or _bool:\n                _bool = True\n            for p in module.parameters():\n                p.requires_grad = _bool\n\n\nclass Classifier(nn.Module):\n    def __init__(self, dim_in, dim_out, dim_hidden, n_layers, task='ml', use_batch_norm=True, dropout_rate=0.01):\n        super().__init__()\n        dims = [dim_in] + [dim_hidden]*(n_layers-1) + [dim_out]\n        self.task = task\n        self.use_batch_norm = use_batch_norm\n        self.dropout_rate = dropout_rate\n        self.classifier = nn.Sequential(collections.OrderedDict(\n            [('Layer {}'.format(i), nn.Sequential(\n                nn.Linear(n_in, n_out),\n                nn.BatchNorm1d(n_out, momentum=.01, eps=0.001) if self.use_batch_norm else None,\n                nn.ReLU() if i < len(dims)-2 else None,\n                nn.Dropout(p=self.dropout_rate) if self.dropout_rate > 0 else None))\n             for i, (n_in, n_out) in enumerate(zip(dims[:-1], dims[1:]))]))\n\n    def get_logits(self, x):\n        for layers in self.classifier:\n            for layer in layers:\n                if layer is not None:\n                    x = layer(x)\n        return x\n\n    def forward(self, x):\n        if self.task == 'mc':\n            return F.softmax(self.get_logits(x), dim=-1)\n        elif self.task == 'ml':\n            return torch.sigmoid(self.get_logits(x))\n        else:\n            raise ValueError(\"The task tag must be either 'ml' (multi-label) or 'mc' (multi-class)!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\nimport torchvision.models.resnet as resnet\nimport collections\nimport torch.nn.functional as F\n\n\nBLOCK = {'18': 'BasicBlock',\n         '34': 'BasicBlock',\n         '50': 'Bottleneck',\n         '101': 'Bottleneck',\n         '152': 'Bottleneck'}\n\n\nLAYERS = {'18': [2, 2, 2, 2],\n          '34': [3, 4, 6, 3],\n          '50': [3, 4, 6, 3],\n          '101': [3, 4, 23, 3],\n          '152': [3, 8, 36, 3]}\n\n\nclass ArtCV(nn.Module):\n    def __init__(self, tag='18', num_labels=(100, 681, 6, 1920, 768),\n                 classifier_layers=(1, 1, 1, 1, 1), classifier_hidden=(2048, 2048, 2048, 2048, 2048),\n                 task=('ml', 'ml', 'mc', 'ml', 'ml'), weights=(1, 1, 1, 1, 1),\n                 use_batch_norm=True, dropout_rate=0.1,\n                 weight_path=None, freeze_cnn=False,\n                 focal_loss=False, focal_loss_mc=False, alpha_t_mc=True,\n                 alpha=(0.25, 0.25, 0.25, 0.25), alpha_mc=(0.25, 0.75, 0.75, 0.75, 0.75, 0.75),\n                 gamma_mc=2, gamma=(2, 2, 2, 2), alpha_t=True, alpha_group=(1, 1), gamma_group=2,\n                 hierarchical=False, label_groups=(0, 1, 1, 1, 1, 1), group_classifier_kwargs=dict(),\n                 weight_group=None):\n        super().__init__()\n        self.tag = tag\n        self.num_labels = num_labels\n        self.classifier_layers = classifier_layers\n        self.classifier_hidden = classifier_hidden\n        self.task = task\n        self.weights = weights\n        self.use_batch_norm = use_batch_norm\n        self.dropout_rate = dropout_rate\n        self.weight_path = weight_path\n        self.freeze_cnn = freeze_cnn\n        self.focal_loss = focal_loss\n        self.focal_loss_mc = focal_loss_mc\n        self.hierarchical = hierarchical\n\n        if self.focal_loss:\n            self.alpha = alpha\n            self.gamma = gamma\n            self.alpha_t = alpha_t\n        if self.focal_loss_mc:\n            self.alpha_mc =alpha_mc\n            self.gamma_mc = gamma_mc\n            self.alpha_t_mc = alpha_t_mc\n            if self.hierarchical:\n                self.alpha_group = alpha_group\n                self.gamma_group = gamma_group\n\n        self.cnn = ResNet_CNN(getattr(resnet, BLOCK[tag]), LAYERS[tag],\n                              weight_path=self.weight_path, freeze_layers=self.freeze_cnn)\n\n        self.classifiers = nn.ModuleDict(\n            collections.OrderedDict(\n                [('classifier{}'.format(i), Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n                                                       dim_out=self.num_labels[i],\n                                                       dim_hidden=self.classifier_hidden[i],\n                                                       n_layers=self.classifier_layers[i],\n                                                       task=self.task[i],\n                                                       use_batch_norm=self.use_batch_norm,\n                                                       dropout_rate=self.dropout_rate))\n                 for i in range(len(self.num_labels))]))\n\n        if self.hierarchical:\n            self.label_groups = np.array(label_groups)\n            self.num_groups = len(np.unique(self.label_groups))\n            self.group_classifier_kwargs = {'dim_hidden': self.classifier_hidden[2],\n                                            'n_layers': self.classifier_layers[2],\n                                            'task': self.task[2],\n                                            'use_batch_norm': self.use_batch_norm,\n                                            'dropout_rate': self.dropout_rate}\n            self.group_classifier_kwargs.update(group_classifier_kwargs)\n            self.group_classifier = Classifier(dim_in=512 * getattr(resnet, BLOCK[tag]).expansion,\n                                               dim_out=self.num_groups, **self.group_classifier_kwargs)\n            self.weight_group = weight_group if weight_group is not None else self.weights[2]\n            self.group_idx_list = torch.nn.ParameterList([torch.nn.Parameter(\n                torch.tensor((self.label_groups == i).astype(np.uint8), dtype=torch.bool), requires_grad=False)\n                for i in range(self.num_groups)])\n\n    def inference(self, x):\n        return self.cnn(x)\n\n    def get_probs_mc(self, x_features):\n        y_pred2 = self.classifiers['classifier2'](x_features)\n\n        if self.hierarchical:\n            y_group2 = self.group_classifier(x_features)\n            y_weighted2 = torch.zeros_like(y_pred2)\n            for i, group_idx in enumerate(self.group_idx_list):\n                y_weighted2[:, group_idx] = y_pred2[:, group_idx] / \\\n                                            (y_pred2[:, group_idx].sum(dim=-1, keepdim=True) + 1e-8) * \\\n                                            y_group2[:, [i]]\n            return y_weighted2.view(-1, self.num_labels[2]), y_group2.view(-1, self.num_groups)\n        else:\n            return y_pred2.view(-1, self.num_labels[2])\n\n    def get_probs(self, x):\n        x_features = self.inference(x)\n        y_pred0 = self.classifiers['classifier0'](x_features)\n        y_pred1 = self.classifiers['classifier1'](x_features)\n        y_pred3 = self.classifiers['classifier3'](x_features)\n        y_pred4 = self.classifiers['classifier4'](x_features)\n        if self.hierarchical:\n            y_weighted2, y_group2 = self.get_probs_mc(x_features)\n            return y_pred0.view(-1, self.num_labels[0]), y_pred1.view(-1, self.num_labels[1]), \\\n                   y_weighted2, \\\n                   y_pred3.view(-1, self.num_labels[3]), y_pred4.view(-1, self.num_labels[4]), \\\n                   y_group2\n        else:\n            return y_pred0.view(-1, self.num_labels[0]), y_pred1.view(-1, self.num_labels[1]), \\\n                   self.classifiers['classifier2'](x_features).view(-1, self.num_labels[2]), \\\n                   y_pred3.view(-1, self.num_labels[3]), y_pred4.view(-1, self.num_labels[4])\n\n    def get_loss_mc(self, x, y2, reduction='sum'):\n        if self.hierarchical:\n            y_pred2, y_group2 = self.get_probs_mc(self.inference(x))\n            if self.focal_loss_mc:\n                loss_group = focal_loss_mc(y_group2,\n                                           torch.tensor(list(map(lambda x: self.label_groups[x],\n                                                                 y2.view(-1)))).view(-1),\n                                           num_classes=self.num_groups, alpha=self.alpha_group,\n                                           gamma=self.gamma_group, alpha_t=self.alpha_t_mc)\n            else:\n                loss_group = F.cross_entropy(y_group2,\n                                             torch.tensor(list(map(lambda x: self.label_groups[x],\n                                                                   y2.view(-1)))).view(-1), reduction='none')\n        else:\n            y_pred2 = self.get_probs_mc(self.inference(x))\n        if self.focal_loss_mc:\n            loss2 = focal_loss_mc(y_pred2, y2.view(-1), num_classes=self.num_labels[2],\n                                  alpha=self.alpha_mc, gamma=self.gamma_mc, alpha_t=self.alpha_t_mc)\n        else:\n            loss2 = F.cross_entropy(y_pred2, y2.view(-1), reduction='none')\n\n        if self.hierarchical:\n            if reduction == 'sum':\n                return loss2 * self.weights[2] + loss_group * self.weight_group\n            elif reduction == 'none':\n                return loss2 * self.weights[2], loss_group * self.weight_group\n        else:\n            return loss2 * self.weights[2]\n\n    def get_loss(self, x, y0, y1, y2, y3, y4):\n        if self.hierarchical:\n            y_pred0, y_pred1, y_pred2, y_pred3, y_pred4, y_group2 = self.get_probs(x)\n            if self.focal_loss_mc:\n                loss_group = focal_loss_mc(y_group2,\n                                           torch.tensor(list(map(lambda x: self.label_groups[x],\n                                                                 y2.view(-1)))).view(-1),\n                                           num_classes=self.num_groups, alpha=self.alpha_group,\n                                           gamma=self.gamma_group, alpha_t=self.alpha_t_mc)\n            else:\n                loss_group = F.cross_entropy(y_group2,\n                                             torch.tensor(list(map(lambda x: self.label_groups[x],\n                                                                   y2.view(-1)))).view(-1), reduction='none')\n        else:\n            y_pred0, y_pred1, y_pred2, y_pred3, y_pred4 = self.get_probs(x)\n        if self.focal_loss:\n            loss0 = torch.mean(focal_loss_ml(y_pred0, y0, alpha=self.alpha[0], gamma=self.gamma[0],\n                                             alpha_t=self.alpha_t), dim=1)\n            loss1 = torch.mean(focal_loss_ml(y_pred1, y1, alpha=self.alpha[1], gamma=self.gamma[1],\n                                             alpha_t=self.alpha_t), dim=1)\n            loss3 = torch.mean(focal_loss_ml(y_pred3, y3, alpha=self.alpha[2], gamma=self.gamma[2],\n                                             alpha_t=self.alpha_t), dim=1)\n            loss4 = torch.mean(focal_loss_ml(y_pred4, y4, alpha=self.alpha[3], gamma=self.gamma[3],\n                                             alpha_t=self.alpha_t), dim=1)\n        else:\n            loss0 = torch.mean(F.binary_cross_entropy(y_pred0, y0, reduction='none'), dim=1)\n            loss1 = torch.mean(F.binary_cross_entropy(y_pred1, y1, reduction='none'), dim=1)\n            loss3 = torch.mean(F.binary_cross_entropy(y_pred3, y3, reduction='none'), dim=1)\n            loss4 = torch.mean(F.binary_cross_entropy(y_pred4, y4, reduction='none'), dim=1)\n        if self.focal_loss_mc:\n            loss2 = focal_loss_mc(y_pred2, y2.view(-1), num_classes=self.num_labels[2],\n                                  alpha=self.alpha_mc, gamma=self.gamma_mc, alpha_t=self.alpha_t_mc)\n        else:\n            loss2 = F.cross_entropy(y_pred2, y2.view(-1), reduction='none')\n        if self.hierarchical:\n            return loss0, loss1, loss2, loss3, loss4, loss_group\n        else:\n            return loss0, loss1, loss2, loss3, loss4\n\n    def forward(self, x, y0, y1, y2, y3, y4, reduction='sum'):\n        if self.hierarchical:\n            loss0, loss1, loss2, loss3, loss4, loss_group = self.get_loss(x, y0, y1, y2, y3, y4)\n            if reduction == 'sum':\n                return loss0 * self.weights[0] + loss1 * self.weights[1] + loss2 * self.weights[2] + \\\n                       loss3 * self.weights[3] + loss4 * self.weights[4] + loss_group * self.weight_group\n            elif reduction == 'none':\n                return loss0 * self.weights[0], loss1 * self.weights[1], loss2 * self.weights[2], \\\n                       loss3 * self.weights[3], loss4 * self.weights[4], loss_group * self.weight_group\n        else:\n            loss0, loss1, loss2, loss3, loss4 = self.get_loss(x, y0, y1, y2, y3, y4)\n            if reduction == 'sum':\n                return loss0*self.weights[0] + loss1*self.weights[1] + loss2*self.weights[2] + \\\n                       loss3*self.weights[3] + loss4*self.weights[4]\n            elif reduction == 'none':\n                return loss0*self.weights[0], loss1*self.weights[1], loss2*self.weights[2], \\\n                       loss3*self.weights[3], loss4*self.weights[4]\n\n    def get_concat_probs(self, x, return_hier_pred=False):\n        if self.hierarchical:\n            if return_hier_pred:\n                y_pred0, y_pred1, y_pred2, y_pred3, y_pred4, y_group2 = self.get_probs(x)\n                return torch.cat((y_pred0, y_pred1,\n                                  F.one_hot(y_pred2.argmax(axis=-1), num_classes=self.num_labels[2]).float()[:, 1:],\n                                  y_pred3, y_pred4), dim=1), \\\n                       F.one_hot(y_group2.argmax(axis=-1), num_classes=self.num_groups)\n            else:\n                y_pred0, y_pred1, y_pred2, y_pred3, y_pred4, _ = self.get_probs(x)\n        else:\n            y_pred0, y_pred1, y_pred2, y_pred3, y_pred4 = self.get_probs(x)\n        return torch.cat((y_pred0, y_pred1,\n                          F.one_hot(y_pred2.argmax(axis=-1), num_classes=self.num_labels[2]).float()[:, 1:],\n                          y_pred3, y_pred4), dim=1)\n\n\ndef focal_loss_ml(inputs, targets, alpha=0.25, gamma=2, alpha_t=False):\n    BCE_loss = F.binary_cross_entropy(inputs, targets, reduction='none')\n    pt = torch.exp(-BCE_loss)\n    if alpha_t:\n        return (-alpha * (targets * 2 -1) + targets) * (1-pt)**gamma * BCE_loss\n    else:\n        return alpha * (1-pt)**gamma * BCE_loss\n\n\ndef focal_loss_mc(inputs, targets,\n                  num_classes, alpha=(0.25, 0.75, 0.75, 0.75, 0.75, 0.75), gamma=2, alpha_t=False):\n    targets_one_hot = F.one_hot(targets, num_classes=num_classes)\n    pt = inputs * targets_one_hot\n    one_sub_pt = 1 - pt\n    log_pt = targets_one_hot * torch.log(inputs + 1e-6)\n    if alpha_t:\n        return torch.sum((-torch.tensor(alpha))*one_sub_pt**gamma*log_pt, dim=-1)\n    else:\n        return torch.sum((-1)*one_sub_pt**gamma*log_pt, dim=-1)\n\n\ndef get_weight_mat(y, ratio=10, base=10):\n    return (torch.ones(y.shape) - y) * (torch.sum(y, dim=-1).view(-1, 1) + base) \\\n           * ratio / (y.shape[1] - torch.sum(y, dim=-1).view(-1, 1)).detach()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### trainer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport sys\nimport copy\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import fbeta_score\n\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import RandomSampler, SequentialSampler\nfrom tqdm import trange\n\n\nclass Trainer:\n    def __init__(self, model, dataset, use_cuda=True,\n                 shuffle=True, epochs=100, extra_epochs_mc=1, train_mc_step=1, \n                 batch_size_train=64, batch_size_val=64, batch_size_all=64, batch_size_test=64,\n                 head_log=True,\n                 monitor_frequency=False, compute_acc=True, printout=False,\n                 dataloader_train_kwargs=dict(), dataloader_val_kwargs=dict(),\n                 dataloader_all_kwargs=dict(),\n                 dataloader_test_kwargs=dict()):\n        self.model = model\n        self.dataset = dataset\n        self.train_val = bool(self.dataset.train_val_split)\n        self.use_cuda = use_cuda\n        if self.use_cuda and torch.cuda.is_available():\n            torch.set_default_tensor_type(torch.cuda.FloatTensor)\n        self.epochs = epochs\n        self.extra_epochs_mc = extra_epochs_mc\n        self.train_mc_step = train_mc_step\n        self.head_log = head_log\n        self.running_loss = []\n        if self.head_log:\n            self.loss_log = dict()\n            self.loss_log['Head 0'] = []\n            self.loss_log['Head 1'] = []\n            self.loss_log['Head 2'] = []\n            self.loss_log['Head 3'] = []\n            self.loss_log['Head 4'] = []\n            if self.model.hierarchical:\n                self.loss_log['Head group classifier'] = []\n\n        if self.train_val:\n            if shuffle:\n                train_sampler = RandomSampler(dataset.train)\n                val_sampler = RandomSampler(dataset.val)\n            else:\n                train_sampler = SequentialSampler(dataset.train)\n                val_sampler = SequentialSampler(dataset.val)\n            self.dataloader_train_kwargs = copy.deepcopy(dataloader_train_kwargs)\n            self.dataloader_train_kwargs.update({'batch_size': batch_size_train, 'sampler': train_sampler})\n            self.dataloader_train = DataLoader(self.dataset.train, **self.dataloader_train_kwargs)\n            self.dataloader_val_kwargs = copy.deepcopy(dataloader_val_kwargs)\n            self.dataloader_val_kwargs.update({'batch_size': batch_size_val, 'sampler': val_sampler})\n            self.dataloader_val = DataLoader(self.dataset.val, **self.dataloader_val_kwargs)\n            self.loss_history_train = []\n            self.loss_history_val = []\n            self.accuracy_history_train = []\n            self.accuracy_history_val = []\n\n        else:\n            sampler = RandomSampler(dataset.all)\n            self.dataloader_train_kwargs = copy.deepcopy(dataloader_train_kwargs)\n            self.dataloader_train_kwargs.update({'batch_size': batch_size_train, 'sampler': sampler})\n            self.dataloader_train = DataLoader(self.dataset.all, **self.dataloader_train_kwargs)\n            self.loss_history_train = []\n            self.accuracy_history_train = []\n\n        all_sampler = SequentialSampler(dataset.all)\n        self.dataloader_all_kwargs = copy.deepcopy(dataloader_all_kwargs)\n        self.dataloader_all_kwargs.update({'batch_size': batch_size_all, 'sampler': all_sampler})\n        self.dataloader_all = DataLoader(self.dataset.all, **self.dataloader_all_kwargs)\n\n        if self.dataset.test_path is not None:\n            test_sampler = SequentialSampler(dataset.test)\n            self.dataloader_test_kwargs = copy.deepcopy(dataloader_test_kwargs)\n            self.dataloader_test_kwargs.update({'batch_size': batch_size_test, 'sampler': test_sampler})\n            self.dataloader_test = DataLoader(self.dataset.test, **self.dataloader_test_kwargs)\n\n        self.use_cuda = use_cuda and torch.cuda.is_available()\n        if self.use_cuda:\n            self.model.cuda()\n        self.monitor_frequency = monitor_frequency\n        self.compute_acc = compute_acc\n        self.printout = printout\n\n    def before_iter(self):\n        pass\n\n    def after_iter(self):\n        pass\n\n    def train_mc(self, lr, parameters=None, betas=(0.9, 0.999), eps=1e-8, weight_decay=0,\n              grad_clip=False, max_norm=1e-5):\n        epochs = self.extra_epochs_mc\n        self.model.classifiers['classifier2'].train()\n        if self.model.hierarchical:\n            self.model.group_classifier.train()\n        params = filter(lambda x: x.requires_grad, self.model.parameters()) \\\n            if parameters is None else parameters\n        optim = torch.optim.Adam(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        for epoch in range(epochs):\n            for x, _, _, y2, _, _ in self.dataloader_train:\n                if self.use_cuda and torch.cuda.is_available():\n                    x = x.cuda()\n                    y2 = y2.cuda()\n                loss = torch.mean(self.model.get_loss_mc(x, y2))\n                optim.zero_grad()\n                loss.backward()\n                if grad_clip:\n                    torch.nn.utils.clip_grad_norm_(params, max_norm=max_norm)\n                optim.step()\n                \n    def train(self, parameters=None, lr=1e-1, mc_lr=1e-1, betas=(0.9, 0.999), eps=1e-8, weight_decay=0,\n              reduce_lr=False, step=5, gamma=0.8,\n              reduce_lr_mc=False, step_mc=5, gamma_mc=0.8,\n              grad_clip=False, max_norm=1e-5,\n              train_mc_kwargs=dict()):\n        epochs = self.epochs\n        self.model.train()\n        params = filter(lambda x: x.requires_grad, self.model.parameters()) \\\n            if parameters is None else parameters\n        optim = torch.optim.Adam(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        if bool(self.extra_epochs_mc):\n            epoch_count_mc = 0\n        with trange(epochs, desc='Training progress: ', file=sys.stdout) as progbar:\n            for epoch_idx in progbar:\n                self.before_iter()\n                progbar.update(1)\n                running_loss = 0\n                if self.head_log:\n                    head_loss0 = 0\n                    head_loss1 = 0\n                    head_loss2 = 0\n                    head_loss3 = 0\n                    head_loss4 = 0\n                    if self.model.hierarchical:\n                        head_loss_group = 0\n                for data_tensors in self.dataloader_train:\n                    data_tensor_tuples = [data_tensors]\n                    if self.head_log:\n                        if self.model.hierarchical:\n                            loss0, loss1, loss2, loss3, loss4, loss_group = self.loss(*data_tensor_tuples,\n                                                                                      head_log=self.head_log)\n                            loss = torch.mean(loss0 + loss1 + loss2 + loss3 + loss4 + loss_group)\n                            head_loss_group += torch.mean(loss_group).item()\n                        else:\n                            loss0, loss1, loss2, loss3, loss4 = self.loss(*data_tensor_tuples, head_log=self.head_log)\n                            loss = torch.mean(loss0 + loss1 + loss2 + loss3 + loss4)\n                        head_loss0 += torch.mean(loss0).item()\n                        head_loss1 += torch.mean(loss1).item()\n                        head_loss2 += torch.mean(loss2).item()\n                        head_loss3 += torch.mean(loss3).item()\n                        head_loss4 += torch.mean(loss4).item()\n                    else:\n                        loss = self.loss(*data_tensor_tuples)\n                    running_loss += loss.item()\n                    optim.zero_grad()\n                    loss.backward()\n                    if grad_clip:\n                        torch.nn.utils.clip_grad_norm_(params, max_norm=max_norm)\n                    optim.step()\n                self.running_loss.append(running_loss/len(self.dataloader_train))\n                if self.head_log:\n                    self.loss_log['Head 0'].append(head_loss0/len(self.dataloader_train))\n                    self.loss_log['Head 1'].append(head_loss1/len(self.dataloader_train))\n                    self.loss_log['Head 2'].append(head_loss2/len(self.dataloader_train))\n                    self.loss_log['Head 3'].append(head_loss3/len(self.dataloader_train))\n                    self.loss_log['Head 4'].append(head_loss4/len(self.dataloader_train))\n                    if self.model.hierarchical:\n                        self.loss_log['Head group classifier'].append(head_loss_group / len(self.dataloader_train))\n\n                if (epoch_idx+1) % step == 0 & reduce_lr:\n                    for p in optim.param_groups:\n                        p['lr'] *= gamma\n\n                if bool(self.monitor_frequency):\n                    if (epoch_idx + 1) % self.monitor_frequency == 0:\n                        current_loss_train = self.compute_loss(tag='train')\n                        self.loss_history_train.append(current_loss_train)\n                        if self.compute_acc:\n                            current_accuracy_train = self.compute_accuracy(tag='train')\n                            self.accuracy_history_train.append(current_accuracy_train)\n                        if self.train_val:\n                            current_loss_val = self.compute_loss(tag='val')\n                            self.loss_history_val.append(current_loss_val)\n                            if self.compute_acc:\n                                current_accuracy_val = self.compute_accuracy(tag='val')\n                                self.accuracy_history_val.append(current_accuracy_val)\n                        if self.printout:\n                            print(\"After %i epochs, loss is %f and prediction accuracy is %f.\"\n                                  % (epoch_idx, current_loss_train, current_accuracy_train))\n                if bool(self.extra_epochs_mc) & (epoch_idx + 1) % self.train_mc_step == 0:\n                    self.train_mc(lr=mc_lr, **train_mc_kwargs)\n                    epoch_count_mc += 1\n                    if epoch_count_mc % step_mc == 0 & reduce_lr_mc:\n                        mc_lr *= gamma_mc\n                                  \n                self.after_iter()\n\n    def loss(self, data_tensors, head_log=False):\n        x, y0, y1, y2, y3, y4 = data_tensors\n        if self.use_cuda and torch.cuda.is_available():\n            x = x.cuda()\n            y0 = y0.cuda()\n            y1 = y1.cuda()\n            y2 = y2.cuda()\n            y3 = y3.cuda()\n            y4 = y4.cuda()\n        if head_log:\n            if self.model.hierarchical:\n                loss0, loss1, loss2, loss3, loss4, loss_group = self.model(x, y0, y1, y2, y3, y4, reduction='none')\n                return loss0, loss1, loss2, loss3, loss4, loss_group\n            else:\n                loss0, loss1, loss2, loss3, loss4 = self.model(x, y0, y1, y2, y3, y4, reduction='none')\n                return loss0, loss1, loss2, loss3, loss4\n        else:\n            loss = torch.mean(self.model(x, y0, y1, y2, y3, y4, reduction='sum'))\n            return loss\n\n    @torch.no_grad()\n    def plot_running_loss(self, epochs_override=None):\n        len_ticks = len(self.running_loss)\n        if epochs_override is None:\n            x_axis = np.linspace(1, self.epochs, len_ticks)\n        else:\n            x_axis = np.linspace(1, epochs_override, len_ticks)\n        plt.figure()\n        plt.plot(x_axis, self.running_loss)\n        plt.xlabel('Number of epochs')\n        plt.ylabel('Estimated loss')\n        plt.show()\n\n    @torch.no_grad()\n    def plot_head_loss(self, epochs_override=None):\n        if not self.head_log:\n            pass\n        else:\n            len_ticks = len(self.running_loss)\n            if epochs_override is None:\n                x_axis = np.linspace(1, self.epochs, len_ticks)\n            else:\n                x_axis = np.linspace(1, epochs_override, len_ticks)\n            plt.figure()\n            for key in self.loss_log.keys():\n                log_array = np.array(self.loss_log[key])\n                plt.plot(x_axis, log_array/log_array[0], label=key)\n            plt.legend(loc='lower left', bbox_to_anchor=(1, 0.25, 0.5, 0.5))\n            plt.xlabel('Number of epochs')\n            plt.ylabel('Estimated loss')\n            plt.show()\n\n    @torch.no_grad()\n    def compute_loss(self, tag):\n        self.model.eval()\n        loss_sum = 0\n        if tag == 'train':\n            _dataloader = self.dataloader_train\n        elif tag == 'val':\n            _dataloader = self.dataloader_val\n        elif tag == 'all':\n            _dataloader = self.dataloader_all\n        else:\n            raise ValueError('Invalid tag!')\n        _dataset = _dataloader.dataset\n        for data_tensors in _dataloader:\n            x, y0, y1, y2, y3, y4 = data_tensors\n            if self.use_cuda and torch.cuda.is_available():\n                x = x.cuda()\n                y0 = y0.cuda()\n                y1 = y1.cuda()\n                y2 = y2.cuda()\n                y3 = y3.cuda()\n                y4 = y4.cuda()\n            loss = self.model(x, y0, y1, y2, y3, y4)\n            loss_sum += torch.sum(loss).item()\n        loss_mean = loss_sum / len(_dataset)\n        self.model.train()\n        return loss_mean\n\n    @torch.no_grad()\n    def loss_history_plot(self, epochs_override=None):\n        len_ticks = len(self.loss_history_train)\n        if epochs_override is None:\n            x_axis = np.linspace(1, self.epochs, len_ticks)\n        else:\n            x_axis = np.linspace(1, epochs_override, len_ticks)\n        plt.figure()\n        if self.train_val:\n            assert (len(self.loss_history_train) == len(self.loss_history_val))\n            plt.plot(x_axis, self.loss_history_train, label='Training set')\n            plt.plot(x_axis, self.loss_history_val, label='Validation set')\n            plt.legend()\n        else:\n            plt.plot(x_axis, self.loss_history_train)\n\n        plt.xlabel('Number of epochs')\n        plt.ylabel('Loss')\n        plt.show()\n\n    @torch.no_grad()\n    def get_probs(self, tag, return_hier_pred=False, return_probs_only=False):\n        self.model.eval()\n        predictions_tem = []\n        if return_hier_pred:\n            hier_pred_tem = []\n        if tag == 'test':\n            _dataloader = self.dataloader_test\n            for data_tensors in _dataloader:\n                x = data_tensors\n                if self.use_cuda and torch.cuda.is_available():\n                    x = x.cuda()\n                y_concat_prob = self.model.get_concat_probs(x)\n                predictions_tem += [y_concat_prob]\n            predictions_array = torch.cat(predictions_tem).detach().cpu().numpy()\n            return predictions_array\n        else:\n            ground_truth = []\n            if tag == 'train':\n                _dataloader = self.dataloader_train\n            elif tag == 'val':\n                _dataloader = self.dataloader_val\n            elif tag == 'all':\n                _dataloader = self.dataloader_all\n            else:\n                raise ValueError('Invalid tag!')\n            for data_tensors in _dataloader:\n                if not return_probs_only:\n                    x, y0, y1, y2, y3, y4 = data_tensors\n                    if self.use_cuda and torch.cuda.is_available():\n                        x = x.cuda()\n                        y0 = y0.cuda()\n                        y1 = y1.cuda()\n                        y2 = y2.cuda()\n                        y3 = y3.cuda()\n                        y4 = y4.cuda()\n                    ground_truth += [torch.cat((y0.long(),\n                                                y1.long(),\n                                                F.one_hot(y2, num_classes=6).squeeze()[:, 1:].long(),\n                                                y3.long(),\n                                                y4.long()), dim=1)]\n                else:\n                    x = data_tensors\n                    if self.use_cuda and torch.cuda.is_available():\n                        x = x.cuda()\n                if return_hier_pred:\n                    y_concat_prob, y_group_pred = self.model.get_concat_probs(x, return_hier_pred=return_hier_pred)\n                    hier_pred_tem += [y_group_pred]\n                else:\n                    y_concat_prob = self.model.get_concat_probs(x, return_hier_pred=return_hier_pred)\n                predictions_tem += [y_concat_prob]\n            predictions_array = torch.cat(predictions_tem).detach().cpu().numpy()\n            if return_hier_pred:\n                hier_pred = torch.cat(hier_pred_tem).detach().cpu().numpy()\n            self.model.train()\n            if not return_probs_only:\n                if return_hier_pred:\n                    return torch.cat(ground_truth).detach().cpu().numpy(), predictions_array, hier_pred\n                else:\n                    return torch.cat(ground_truth).detach().cpu().numpy(), predictions_array\n            else:\n                if return_hier_pred:\n                    return predictions_array, hier_pred\n                else:\n                    return predictions_array\n\n    @torch.no_grad()\n    def make_predictions(self, tag, return_pred_only=False,\n                         thre=(0.08, 0.08, 0.08, 0.08), upper_bound=(3, 4, 17, 18), lower_bound=3,\n                         boundary=([0, 100], [100, 781], [786, 2706], [2706, 3474])):\n        self.model.eval()\n        predictions_tem = []\n        if tag == 'test':\n            _dataloader = self.dataloader_test\n            for data_tensors in _dataloader:\n                x = data_tensors\n                if self.use_cuda and torch.cuda.is_available():\n                    x = x.cuda()\n                y_concat_pred = regularized_pred(self.model.get_concat_probs(x).detach().cpu().numpy(),\n                                                 thre=thre,\n                                                 upper_bound=upper_bound, lower_bound=lower_bound, boundary=boundary)\n                predictions_tem += [y_concat_pred]\n            predictions_array = np.concatenate(predictions_tem)\n            return predictions_array\n        else:\n            ground_truth = []\n            if tag == 'train':\n                _dataloader = self.dataloader_train\n            elif tag == 'val':\n                _dataloader = self.dataloader_val\n            elif tag == 'all':\n                _dataloader = self.dataloader_all\n            else:\n                raise ValueError('Invalid tag!')\n            for data_tensors in _dataloader:\n                if not return_pred_only:\n                    x, y0, y1, y2, y3, y4 = data_tensors\n                    if self.use_cuda and torch.cuda.is_available():\n                        x = x.cuda()\n                        y0 = y0.cuda()\n                        y1 = y1.cuda()\n                        y2 = y2.cuda()\n                        y3 = y3.cuda()\n                        y4 = y4.cuda()\n                    ground_truth += [torch.cat((y0.long(),\n                                                y1.long(),\n                                                F.one_hot(y2, num_classes=6).squeeze()[:, 1:].long(),\n                                                y3.long(),\n                                                y4.long()), dim=1)]\n                else:\n                    x = data_tensors\n                    if self.use_cuda and torch.cuda.is_available():\n                        x = x.cuda()\n                y_concat_pred = regularized_pred(self.model.get_concat_probs(x).detach().cpu().numpy(),\n                                                 thre=thre,\n                                                 upper_bound=upper_bound, lower_bound=lower_bound, boundary=boundary)\n                predictions_tem += [y_concat_pred]\n            predictions_array = np.concatenate(predictions_tem)\n            self.model.train()\n            if not return_pred_only:\n                return torch.cat(ground_truth).detach().cpu().numpy(), predictions_array\n            else:\n                return predictions_array\n\n    @torch.no_grad()\n    def compute_hier_acc(self, tag):\n        ground_truth = []\n        hier_pred_tem = []\n        if tag == 'train':\n            _dataloader = self.dataloader_train\n        elif tag == 'val':\n            _dataloader = self.dataloader_val\n        elif tag == 'all':\n            _dataloader = self.dataloader_all\n        else:\n            raise ValueError('Invalid tag!')\n        for data_tensors in _dataloader:\n            x, _, _, y2, _, _ = data_tensors\n            if self.use_cuda and torch.cuda.is_available():\n                x = x.cuda()\n                y2 = y2.cuda()\n            gt_oh = F.one_hot(y2, num_classes=6).squeeze()[:, 1:].long()\n            ground_truth += [torch.sum(gt_oh, dim=-1)]\n            if self.model.hierarchical:\n                _, y_group_pred = self.model.get_concat_probs(x, return_hier_pred=True)\n                hier_pred_tem += [y_group_pred]\n            else:\n                y_concat_prob = self.model.get_concat_probs(x, return_hier_pred=False)\n                hier_pred_tem += [torch.sum(y_concat_prob[:, 781:786], dim=-1)]\n        if self.model.hierarchical:\n            return np.mean(torch.cat(ground_truth).detach().cpu().numpy() ==\n                           np.where(torch.cat(hier_pred_tem).detach().cpu().numpy() == 1)[1])\n        else:\n            return np.mean(torch.cat(ground_truth).detach().cpu().numpy() ==\n                           torch.cat(hier_pred_tem).detach().cpu().numpy())\n\n    @torch.no_grad()\n    def compute_accuracy(self, tag,\n                         thre=(0.08, 0.08, 0.08, 0.08), upper_bound=(3, 4, 17, 18), lower_bound=3,\n                         boundary=([0, 100], [100, 781], [786, 2706], [2706, 3474])):\n        y_true, y_pred = self.make_predictions(tag=tag, thre=thre,\n                                               upper_bound=upper_bound, lower_bound=lower_bound, boundary=boundary)\n        f_beta = [fbeta_score(y_true[i, :], y_pred[i, :], beta=2) for i in range(y_true.shape[0])]\n        return sum(f_beta) / len(f_beta)\n\n    @torch.no_grad()\n    def accuracy_history_plot(self, epochs_override=None):\n        len_ticks = len(self.accuracy_history_train)\n        if epochs_override is None:\n            x_axis = np.linspace(1, self.epochs, len_ticks)\n        else:\n            x_axis = np.linspace(1, epochs_override, len_ticks)\n        plt.figure()\n        if self.train_val:\n            assert (len(self.accuracy_history_train) == len(self.accuracy_history_val))\n            plt.plot(x_axis, self.accuracy_history_train, label='Training set')\n            plt.plot(x_axis, self.accuracy_history_val, label='Validation set')\n            plt.legend()\n        else:\n            plt.plot(x_axis, self.accuracy_history_train)\n\n        plt.xlabel('Number of epochs')\n        plt.ylabel('Accuracy')\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2) Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_root_path = '/kaggle/input/imet-2020-fgvc7'\npath = f'{_root_path}/train'\ntest_path = f'{_root_path}/test'\ndata_info_path = f'{_root_path}/train.csv'\nlabels_info_path = f'{_root_path}/labels.csv'\ntest_csv_path = f'{_root_path}/sample_submission.csv'\n\ndataset = TrainValSet(train_val_split=False, train_transform='val',\n                      path=path, data_info_path=data_info_path, labels_info_path=labels_info_path,\n                      test_path=test_path, test_csv_path=test_csv_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag='50'\nweight_path = f'/kaggle/input/resnet{tag}/resnet{tag}.pth'\nmodel = ArtCV(tag=tag, weight_path=weight_path, freeze_cnn=True, dropout_rate=0.1, weights=(1, 1, 10, 1, 1),\n              classifier_layers=(2,2,2,2,2), focal_loss=False, focal_loss_mc=False,\n              alpha=(0.25, 0.25, 0.25, 0.25), alpha_mc=(0.25, 0.75, 0.75, 0.75, 0.75, 0.75),\n              gamma_mc=2, gamma=(2, 2, 2, 2), alpha_t=True, alpha_t_mc=True,\n              hierarchical=False, label_groups=(0, 1, 2, 1, 1, 1), alpha_group=(1, 1), gamma_group=2, weight_group=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = Trainer(model, dataset, extra_epochs_mc=1, train_mc_step=2,\n                  batch_size_train=256, batch_size_val=256, batch_size_all=128,\n                  epochs=8, compute_acc=False, head_log=True,\n                  monitor_frequency=False,\n                  dataloader_train_kwargs={'num_workers':2}, dataloader_val_kwargs={'num_workers':2},\n                  dataloader_all_kwargs={'num_workers':2})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train(lr=1e-2, mc_lr=2e-3, reduce_lr=True, reduce_lr_mc=True,\n              step=2, gamma=0.8, step_mc=2, gamma_mc=0.8)\nfile_name = 'frozen_no_crop_resnet50_2layer_8_4_epochs_reduced_lr.model.pkl'\nsave_path = f'{sys.path[0]}/{file_name}'\ntorch.save(trainer.model.state_dict(), save_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.plot_running_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.plot_head_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_array=trainer.make_predictions(tag='test', thre=(0.1, 0.1, 0.1, 0.1), upper_bound=(3, 4, 10, 10), lower_bound=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(f'{_root_path}/sample_submission.csv')\nfor i, one_hot in enumerate(predictions_array):\n    ids = np.where(one_hot)[0]\n    submission.iloc[i].attribute_ids = ' '.join([str(x) for x in ids])\n\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Acknowledge\nIf you find this notebook helpful, please upvote.\n<br>Team member:\n<br>[yunchenyang](https://www.kaggle.com/yunchenyang), email: yunchenyang@hotmail.com;\n<br>[ytisserant](https://www.kaggle.com/ytisserant), email: ytisserant@gmail.com","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}