{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import modules","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datset_count_limit = None # None for all, debug option (whole dataset is 4h per epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os.path\nimport json\nimport codecs\nfrom collections import Counter\nimport random\nfrom datetime import datetime\nimport math\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport warnings\nimport random\nimport numpy as np\n\nfrom tqdm import trange\nfrom tqdm import tqdm as tqdm\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as D\nimport torch.optim as optim\n\nimport torchvision\nimport torchvision.datasets as datasets\nimport torchvision.transforms as transforms\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CUDA = \"cuda:0\"\nCPU = \"cpu\"\nuse_cuda = torch.cuda.is_available()\ndevice = torch.device(CUDA if use_cuda else CPU)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load files","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_PATH = \"../input/herbarium-2020-fgvc7/nybg2020/train/\"\nTRAIN_META_PATH = \"../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json\"\n\nTEST_PATH = \"../input/herbarium-2020-fgvc7/nybg2020/test/\"\nTEST_META_PATH = \"../input/herbarium-2020-fgvc7/nybg2020/test/metadata.json\"\n\nSUBMISSION_PATH = '../input/herbarium-2020-fgvc7/sample_submission.csv'\n\nWEIGHTS_PATH = '../input/fc2ep3/h_2fc_ce_ep3.pth'\n\n\nOUTPUT_PATH =  '/kaggle/working'\n\nwith codecs.open(TRAIN_META_PATH, 'r', encoding='utf-8', errors='ignore') as f:\n    train_meta = json.load(f)\n    \nwith codecs.open(TEST_META_PATH, 'r', encoding='utf-8', errors='ignore') as f:\n    test_meta = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge training data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.DataFrame(train_meta['annotations'])\n\ntrain_cat = pd.DataFrame(train_meta['categories'])\ntrain_cat.columns = ['family', 'genus', 'category_id', 'category_name']\n\ntrain_img = pd.DataFrame(train_meta['images'])\ntrain_img.columns = ['file_name', 'height', 'image_id', 'license', 'width']\n\ntrain_reg = pd.DataFrame(train_meta['regions'])\ntrain_reg.columns = ['region_id', 'region_name']\n\ntrain_df = train_df.merge(train_cat, on='category_id', how='outer')\ntrain_df = train_df.merge(train_img, on='image_id', how='outer')\ntrain_df = train_df.merge(train_reg, on='region_id', how='outer')\n\nsample_sub = pd.read_csv(SUBMISSION_PATH)\n\ndisplay(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"family_index = dict([(v, i) for i, v in enumerate(sorted(list(set(train_df['family']))))])\n# print('Family count: {}'.format(len(family_index)))\n\ncat_id_family_map = dict()\ndistinct_cat_df = train_df.drop_duplicates('category_id')\nfor n in trange(len(distinct_cat_df)):\n    item = distinct_cat_df.iloc[n]\n    cat_id = item['category_id']\n    family_name = item['family']\n    family_id = family_index[family_name]\n    cat_id_family_map[cat_id] = family_id\ncat_family_index = [cat_id_family_map[n] for n in sorted(list(pd.unique(train_df['category_id'])))]\n\n\n\ncat_id_count = [0] * len(pd.unique(train_df['category_id']))\ncat_id_items =[list() for i in range(len(pd.unique(train_df['category_id'])))]\n\nfam_id_count = [0] * len(family_index)\nfor n in trange(len(train_df['category_id'])):\n    cat_id = train_df['category_id'].values[n]\n    cat_id_count[cat_id] = cat_id_count[cat_id] + 1\n    \n    cat_id_items[cat_id].append(n)\n    \n#     family_name = train_df['family'].values[n]\n#     family_id = family_index[family_name]\n#     fam_id_count[family_id] = fam_id_count[family_id] + 1\n\ntrain_ids = []\nthreshold = 20\nfor n in trange(len(cat_id_items)):\n    item_list = cat_id_items[n]\n    random.shuffle(item_list)\n    if len(item_list) > threshold:\n        item_list = item_list[:threshold]\n    train_ids.extend(item_list)\nrandom.shuffle(train_ids)\nprint('train_ids', len(train_ids))\n\n\n# cat_id_weights = [1 / count for count in cat_id_count]\n# fam_id_weights = [1 / count for count in fam_id_count]\n\n# print(len(cat_family_index))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare torch dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\n\ndef make_albs(*args):\n    alb_list = []\n    [alb_list.extend(a) for a in args]\n    transforms = albumentations.Compose(alb_list)\n\n    def f(im):\n        sample = transforms(image=im)\n        return sample['image']\n\n    return f\n\nalb_f = make_albs(\n    [\n                        albumentations.HorizontalFlip(p=0.5),\n                        albumentations.VerticalFlip(p=0.3),\n                        albumentations.RandomBrightnessContrast(brightness_limit=0.05, contrast_limit=0.05),\n                        albumentations.Rotate(limit=25, border_mode=cv2.BORDER_REFLECT),\n    ])\n    \n\nDATASET_SUCCESS_STATUS = -1\nDATASET_ERROR_INDEX = 0\n\nDEFAULT_RESIZE = (480, 320)\nDEFAULT_CROP = (0.1, 0.1)\n\nclass HerbariumDataset(D.Dataset):\n    success = DATASET_SUCCESS_STATUS\n    \n    def __init__(self, data, path, crop=DEFAULT_CROP, resize=DEFAULT_RESIZE, ids=None):\n        self.data = data\n        self.path = path\n        self.crop = crop\n        self.resize = resize\n        self.ids = ids\n        \n        self.error_index = DATASET_ERROR_INDEX\n        \n        self.output_labels = True\n        self.output_fam = True\n\n    def __len__(self):\n        if self.ids is not None:\n            return len(self.ids)\n        return len(self.data)\n    \n    def get_image_path(self, i):\n        fname = str(self.data['file_name'].values[i])\n        fpath = os.path.join(self.path, fname)\n        return fpath\n\n    def __getitem__(self, i):\n        try:\n            if self.ids is not None:\n                i = self.ids[i]\n            # Load image\n            fpath = self.get_image_path(i)\n            image = cv2.imread(fpath)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n            # Crop\n            if self.crop:\n                h, w, _ = image.shape\n                crop_h, crop_w = self.crop\n                image = image[int(crop_h*h):int(-1 * crop_h * h), int(crop_w*w):int(-1 * crop_w * w)]\n\n            # Resize\n            resize_h, resize_w = self.resize\n            image = cv2.resize(image, (resize_w, resize_h))\n            \n            if self.ids is not None:\n                image = alb_f(image)\n\n            # Normalize\n            min_value = 0.0 # np.min(image)\n            max_value = np.max(image)\n            image = (image - min_value) / max_value\n\n            # Convert to Channel First\n            image = np.rollaxis(image, 2, 0)\n\n            # Get label\n            label = np.array([0])\n            if self.output_labels:\n                label = self.data['category_id'].values[i]\n            \n            # Get family\n            fam_id = np.array([0])\n            if self.output_fam:\n                fam_id = cat_id_family_map[label]\n\n            return (torch.tensor(self.success, dtype=torch.long),  # Status\n                    torch.tensor(image.copy(), dtype=torch.float), # Image\n                    torch.tensor(label.copy(), dtype=torch.long),  # Cat\n                    torch.tensor(fam_id, dtype=torch.long)) # Family\n\n        except Exception as e:\n            _, image, label, fam = self.__getitem__(self.error_index)\n            return (torch.tensor(i, dtype=torch.long),\n                    image,\n                    label,\n                    fam)\n\n\nclass HerbariumDictDataset(HerbariumDataset):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.output_labels = False\n        self.output_fam = False\n\n    def get_image_path(self, i):\n        fname = str(self.data[i])\n        fpath = os.path.join(self.path, fname)\n        return fpath\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if datset_count_limit:\n#     train_data, test_data = train_test_split(train_df[:datset_count_limit])\n# else:\n#     train_data, test_data = train_test_split(train_df, test_size=0.125)  # To be close to submission count (139k)\n\n# train_dataset = HerbariumDataset(train_data, TRAIN_PATH)\n# test_dataset = HerbariumDataset(test_data, TRAIN_PATH)  # There should be train path, it is correct\n\ntrain_data = train_df\nif datset_count_limit:\n    train_data = train_df[:datset_count_limit]\ntrain_dataset = HerbariumDataset(train_data, TRAIN_PATH, ids=train_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, img, label, _ = train_dataset[random.randint(0, len(train_dataset))]\nprint(img.shape)\nprint(label)\nplt.imshow(img.transpose(0, 2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# _, img, label, _ = test_dataset[random.randint(0, len(test_dataset))]\n# print(img.shape)\n# print(label)\n# plt.imshow(img.transpose(0, 2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Declare model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# To override exprired sertificate\nimport ssl\nssl._create_default_https_context = ssl._create_unverified_context\n\n\nnum_classes = 32094\nconv_net = net = timm.create_model('resnet34', pretrained=True, num_classes=num_classes)\n\n\nclass TwoStageNet(torch.nn.Module):\n    FAM2CAT = cat_family_index\n    def __init__(self):\n        super(TwoStageNet, self).__init__()\n        self.fam_cat_map = torch.tensor([self.FAM2CAT])\n        \n        self.conv_net = conv_net\n        # Remove classifer - last fc layer (make it transparent)\n        del self.conv_net.fc\n        self.conv_net.fc = lambda x: x\n        \n        self.fc_fam = nn.Linear(in_features=512,\n                             out_features=310,\n                             bias=True)\n        \n        self.fc_cat = nn.Linear(in_features=512,\n                             out_features=32094,\n                             bias=True)\n        \n        \n        \n    def forward(self, x):\n        x = self.conv_net.forward(x)\n\n        fx = self.fc_fam(x)\n        fam_id = fx\n \n        cx = self.fc_cat(x)\n        cat_id = cx\n        \n        gather_index = torch.cat([self.fam_cat_map for i in range(fx.shape[0])]).to(device)\n        fam_id_as_cat_id = torch.gather(F.softmax(fx), dim=1, index=gather_index)\n        result = F.softmax(F.softmax(cat_id) * fam_id_as_cat_id)\n          \n        return cat_id, fam_id, result","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"################################################################################\n# Metrics configuration\n################################################################################\ndef accuracy(x, y):\n    x = x.detach().cpu().numpy()\n    y = y.detach().cpu().numpy()\n    correct = np.sum(x == y)\n    total = y.shape[0]\n    return correct / total\n\n\ndef f1(x, y):\n    x = x.detach().cpu().numpy()\n    y = y.detach().cpu().numpy()\n    return f1_score(y, x, average = 'macro')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"################################################################################\n# Model configuration\n################################################################################\nnum_classes = 32094\n\nnet = TwoStageNet()\nnet.to(device)\nnet.load_state_dict(torch.load(WEIGHTS_PATH))\n################################################################################\n# Training configuration\n################################################################################\ntimestamp = datetime.now().strftime(\"%d_%m_%Y_%H_%M_%S\")\n\nbatch_size = 64\nepoch_count = 1\n\nlearning_rate = 0.00003 #0.001\ndecay = 1e-6\noptimizer = optim.Adam(net.parameters(), lr=learning_rate, weight_decay=decay)\n# scheduler = ReduceLROnPlateau(optimizer,\n#                               'min',\n#                               patience=10,\n#                               factor=0.8,\n#                               min_lr=1e-8)\n\n\n_cat_cross_entropy_loss = nn.CrossEntropyLoss() # weight=torch.tensor(cat_id_weights).to(device))\ncat_criterion = lambda x, y: _cat_cross_entropy_loss(x, y.long())\n\n_fam_cross_entropy_loss = nn.CrossEntropyLoss() # weight=torch.tensor(fam_id_weights).to(device))\nfam_criterion = lambda x, y: _fam_cross_entropy_loss(x, y.long())\n# criterion = torch.nn.BCEWithLogitsLoss()\n\n################################################################################\n# Data configuration\n################################################################################\ntrain_dataset = HerbariumDataset(train_data, TRAIN_PATH, ids=train_ids)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=3,\n                                           shuffle=True)\n################################################################################\n# Run\n################################################################################\nwarnings.filterwarnings(\"ignore\")\n# Set constants\nEPOCH_KEY = 'EPOCH'\nBATCH_KEY = 'BATCH'\n\nLOSS_KEY = 'LOSS'\nACC_KEY = 'ACC'\nF1_KEY = 'F1'\n\nVAL_LOSS_KEY = 'VAL_LOSS'\nVAL_ACC_KEY = 'VAL_ACC'\nVAL_F1_KEY = 'VAL_F1'\n\nLOSS_PRINT_STEP = 1\nbest_loss = float('inf')\nhistory = []\n\ntrain_data_errors = []\n\noptimizer.zero_grad()\n# Run training traversion throught epoches\nfor epoch in range(1, epoch_count+1):\n\n    # Traverse throught batches\n    train_count = len(train_loader)\n    with tqdm(train_loader, position=0) as train_data_iterator:\n        net.train()\n        for batch_n, data in enumerate(train_data_iterator, 1):\n            progress_update = False\n\n            # Get the inputs; data is a list of [inputs, labels]\n            status, inputs, labels, fam_labels = data\n            if not all(status == DATASET_SUCCESS_STATUS):\n                train_data_errors.append((epoch ,batch_n, status))\n            inputs, labels, fam_labels = inputs.to(device), labels.to(device), fam_labels.to(device)\n            \n            # Zero the parameter gradients\n            \n\n            # Forward -> backward -> optimize\n            cat, fam, res = net(inputs)\n            \n            cat_loss = cat_criterion(cat, labels)\n            fam_loss = fam_criterion(fam, fam_labels)\n            \n            loss = cat_loss + fam_loss\n            loss.backward()\n            if (batch_n+1)%4 == 0:\n                optimizer.step()\n                optimizer.zero_grad()\n\n            # Get statistics\n            history_item = {EPOCH_KEY: epoch, BATCH_KEY: batch_n, LOSS_KEY: loss.item()}\n            \n            # Set for stat update if there lable-update batch\n            if batch_n == 1 or batch_n % LOSS_PRINT_STEP == 0:\n                progress_update = True\n                \n                x = torch.argmax(cat, dim=1)\n                history_item['CAT_ACC'] = accuracy(x, labels).item()\n                history_item['CAT_F1'] = f1(x, labels).item()\n                \n                x = torch.argmax(fam, dim=1)\n                history_item['FAM_ACC'] = accuracy(x, fam_labels).item()\n                history_item['FAM_F1'] = f1(x, fam_labels).item()\n                \n                x = torch.argmax(res, dim=1)\n                history_item[ACC_KEY] = accuracy(x, labels).item()\n                history_item[F1_KEY] = f1(x, labels).item()\n            \n            # On epoch end\n            if batch_n == train_count:\n                progress_update = True\n                # Save weights\n                w_path_fname = '/_weights_{}_{}.pth'.format(str(epoch), str(timestamp))\n                w_path = OUTPUT_PATH + w_path_fname\n                torch.save(net.state_dict(), w_path)\n                # Store history to history list\n                history_path = OUTPUT_PATH + '/hisotry' + timestamp + '.json'\n                with open(history_path, 'w') as json_file:\n                    json.dump(history, json_file)\n\n            # Print changes to tqdm bar\n            if progress_update:\n                train_data_iterator.set_postfix(history_item)\n            # Store history to history list\n            history.append(history_item)\n\n\nprint('Train dataset errors: ', str(train_data_errors))\nprint('Training fininshed!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def smooth(x,window_len=11,window='hanning'):\n    if x.ndim != 1:\n        raise ValueError(\"smooth only accepts 1 dimension arrays.\")\n    if x.size < window_len:\n        raise ValueError(\"Input vector needs to be bigger than window size.\")\n    if window_len<3:\n        return x\n    if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:\n        raise ValueError(\"Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'\")\n    s=np.r_[x[window_len-1:0:-1],x,x[-2:-window_len-1:-1]]\n    #print(len(s))\n    if window == 'flat': #moving average\n        w=np.ones(window_len,'d')\n    else:\n        w=eval('np.'+window+'(window_len)')\n    y=np.convolve(w/w.sum(),s,mode='valid')\n    return y\n\nhistory_f1 = [i['F1'] for i in history]\nhistory_loss = [i['LOSS'] for i in history]\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n\nax1.set_title('f1')\nax1.plot(history_f1)\nax1.plot(smooth(np.array(history_f1), window_len=51))\n\nax2.set_title('loss')\nax2.plot(history_loss)\nax2.plot(smooth(np.array(history_loss), window_len=51))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# valid_dataset = HerbariumDataset(test_data, TRAIN_PATH)  # There should be train path, it is correct\n\n# valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, num_workers=2,\n#                                            shuffle=False)\n\n# valid_data_errors = []\n# total_valid_loss = 0.0\n# xs = torch.tensor([]).to(device).long()\n# ys = torch.tensor([]).to(device).long()\n# with torch.no_grad():\n#     net.eval()\n#     with tqdm(valid_loader, position=0) as valid_data_iterator:\n#         net.train()\n#         for valid_batch, (valid_status, valid_indputs, valid_labels, fam_labels) in enumerate(valid_data_iterator, 1):\n#             if not all(valid_status == DATASET_SUCCESS_STATUS):\n#                 valid_data_errors.append((epoch , status))\n#             valid_indputs, valid_labels = valid_indputs.to(device), valid_labels.to(device)\n\n#             # Evaluate batch\n#             _, _, valid_pred = net(valid_indputs)\n\n#             # Summing loss\n#             total_valid_loss += criterion(valid_pred, valid_labels)\n\n#             # Softmax and argmax results to reduce space consumption\n#             x = valid_pred\n#             # x = F.softmax(x, dim=1) # In case of lenet5 softmax is inside NN\n#             x = torch.argmax(x, dim=1)\n#             # Cat batch eval results and gt labels\n#             xs = torch.cat((xs, x.long()), dim=0)\n#             ys = torch.cat((ys, valid_labels.long()), dim=0)\n\n# # Average validation metrics and store to history)\n# avg_valid_loss = total_valid_loss / len(valid_loader)\n# print('Valid loss', float(avg_valid_loss))\n# print('Valid acc.', accuracy(xs, ys).item())\n# print('Valid f1.', f1(xs, ys).item())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(test_meta['images'])\ntest_path_dict = dict([(i, j) for i, j in test_df[['id', 'file_name']].values])\nsubmission_dataset = HerbariumDictDataset(test_path_dict, TEST_PATH)\n# print('Test images count', len(test_path_dict))\n\nwith torch.no_grad():\n    net.eval()\n    for n in trange(len(submission_dataset), desc='Submission'):\n        i = sample_sub.xs(n)['Id']\n        _, x, _, _ = submission_dataset[i]\n        x = torch.unsqueeze(x, 0)\n        x = x.to(device)\n        _, _, y = net(x)\n        y = torch.argmax(y, dim=1)\n        sample_sub.xs(n)['Predicted'] = y.item()\n\nsample_sub.to_csv(OUTPUT_PATH + 'my_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub.to_csv(OUTPUT_PATH + '/my_submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}