{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import library"},{"metadata":{"trusted":true},"cell_type":"code","source":"# unzip the test data\n!tar -xvf ../input/glr-test-set/test.tar","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"accelerator_type = 'CPU'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if(accelerator_type == 'TPU'):    \n    !curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n    !python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev\n    import torch_xla\n    import torch_xla.core.xla_model as xm\n    device = xm.xla_device(n=4, devkind='TPU')\n    import torch\nelif(accelerator_type == 'GPU'):\n    import torch\n    device = torch.device(\"cuda\")\nelse:\n    import torch\n    device = torch.device(\"cpu\")\nprint(\"Device:\",device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport matplotlib.image as mpimg\nimport cv2\nimport seaborn as sn\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch\n#!pip install pytorchcv\n#from pytorchcv.model_provider import get_model as ptcv_get_model\nfrom efficientnet_pytorch import EfficientNet\nimport torchvision\nfrom torchvision import transforms\nimport torchvision.models as models\nfrom torch import Tensor\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.optim as optim\n! pip install torchsummary\nfrom torchsummary import summary\nimport os\n\nimport seaborn as sns\nfrom tqdm import tqdm_notebook\n\n# one-hot vector\nfrom sklearn.preprocessing import LabelEncoder\n\nimport multiprocessing\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"directory = '../input/landmark-recognition-2020/'\ntrain_dir = '../input/landmark-recognition-2020/train/*/*/*/*'\n#test_dir = '../input/landmark-recognition-2020/test/*/*/*/*'\noutput_dir ='../output/kaggle/working/'\nimage_dir_train='../input/landmark-recognition-2020/train/'\nimage_dir_test='../input/landmark-recognition-2020/test/'\nos.listdir(directory)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Visualization "},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(directory,'sample_submission.csv'))\ndf_test['image_']=df_test.id.str[0]+\"/\"+df_test.id.str[1]+\"/\"+df_test.id.str[2]+\"/\"+df_test.id+\".jpg\"\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(directory,'train.csv'))\ndf_train[\"image_\"] = df_train.id.str[0]+\"/\"+df_train.id.str[1]+\"/\"+df_train.id.str[2]+\"/\"+df_train.id+\".jpg\"\ndf_train[\"target_\"] = df_train.landmark_id.astype(str)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total Train Images: {}'.format(len(df_train))) \nprint('Total Test Images: {}'.format(len(df_test)))\nprint('Total Unique Landmarks: {}'.format(df_train.landmark_id.nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"landmarks = df_train.groupby('landmark_id',as_index=False)['id'].count()\\\n    .sort_values('id',ascending=False).reset_index(drop=True)\nlandmarks.rename(columns={'id':'count'},inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_text(ax,fontsize=12):\n    for p in ax.patches:\n        x=p.get_bbox().get_points()[:,0]\n        y=p.get_bbox().get_points()[1,1]\n        ax.annotate('{}'.format(int(y)), (x.mean(), y), ha='center', va='bottom',size=fontsize)\nfig, (ax1,ax2) = plt.subplots(2,1,figsize=(16,8))\nsns.barplot(data=landmarks[:50],x='landmark_id',y='count',ax=ax1,color='#30a2da',\n           order=landmarks[:50]['landmark_id'])\nadd_text(ax1,fontsize=8)\nax1.set_title('Top 50 Landmarks')\nax1.set_ylabel('Number of Images')\nax1.set_xticklabels(ax1.get_xticklabels(), rotation=40, ha=\"right\",size=8)\nsns.barplot(data=landmarks[-50:],x='landmark_id',y='count',ax=ax2,color='#fc4f30')\nax2.set_title('Bottom 50 Landmarks')\nax2.set_ylabel('Number of Images')\nax2.set_xticklabels(ax2.get_xticklabels(), rotation=40, ha=\"right\",size=8)\nplt.tight_layout()\nprint('Total Unique Landmarks: {}'.format(df_train.landmark_id.nunique()))\nprint(f\"Number of Landmarks with less than 3 images are {len(landmarks[landmarks['count']<3])}\")\nprint(f\"Number of Landmarks with less than 5 images are {len(landmarks[landmarks['count']<5])}\")\nprint(f\"Number of Landmarks with less than 10 images are {len(landmarks[landmarks['count']<10])}\")\nprint(f\"Number of Landmarks with less than 20 images are {len(landmarks[landmarks['count']<20])}\")\nprint(f\"Number of Landmarks with less than 30 images are {len(landmarks[landmarks['count']<30])}\")\nprint(f\"Number of Landmarks with less than 50 images are {len(landmarks[landmarks['count']<50])}\")\nprint(f\"Number of Landmarks with less than 100 images are {len(landmarks[landmarks['count']<100])}\")\nplt.show()\nnum_classes = df_train.landmark_id.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16,4))\nax = sns.distplot(df_train['landmark_id'],bins=500)\nax.set_title('Distribution of Landmarks')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image(id):\n    path = os.path.join('/kaggle/input/landmark-recognition-2020/train',\n                        id[0],id[1],id[2],id+'.jpg')\n    img = cv2.imread(path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_data(df,rows,cols):\n    df.reset_index(inplace=True,drop=True)\n    fig = plt.figure(figsize=(24,24))\n    i = 1\n    for r in range(rows):\n        for c in range(cols):\n            id = df.loc[i-1,'id']\n            label = df.loc[i-1,'landmark_id']\n            ax = fig.add_subplot(rows,cols,i)\n            img = get_image(id)\n            ax.set_xticks([])\n            ax.set_yticks([])\n            ax.set_title(label)\n            ax.imshow(img)\n            i+=1\n    return fig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#some random images from the train set\n# some images are not landmarks e.g. 166107 and 90410\ninds = np.random.choice(df_train.index.tolist(),20)\nfig = show_data(df_train.iloc[inds,:],4,5)\nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_images = df_train.drop_duplicates(subset=['landmark_id'])\ndf_images = df_images.sample(n=1000,random_state=23)\ndf_images.reset_index(inplace=True,drop=True)\ndf_images['height'] = 0\ndf_images['width'] = 0\ndf_images['channels'] = 0\nfor i in tqdm_notebook(range(len(df_images))):\n    img = get_image(df_images.loc[i,'id'])\n    df_images.loc[i,'height'] = img.shape[0]\n    df_images.loc[i,'width'] = img.shape[1]\n    df_images.loc[i,'channels'] = img.shape[2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image Size Distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"def img_distribution(df):\n    shape = (np.min(df['width']), np.max(df['width']),\n            np.min(df['height']), np.max(df['height']))\n    fig = px.scatter(df,x='width',y='height')\n    fig.add_shape(\n        x0 = shape[0],\n        x1 = shape[1],\n        y0 = shape[2],\n        y1 = shape[3],\n        fillcolor = 'yellow',\n        opacity=0.3,\n        layer='below'\n    )\n    fig.add_trace(go.Scatter(name='mean',x=[np.mean(df['width'])],y=[np.mean(df['height'])],\n                         marker=dict(color='red',size=10)))\n    #fig.update_traces(marker_line_color='black',marker_line_width=1)\n    fig.update_layout(width=700,height=400,margin=dict(l=0,b=0,r=0,t=40),template='seaborn',\n                 title='Distribution of Image Dimensions', showlegend=False,\n                 xaxis=dict(title='Width', mirror=True, linewidth=2, linecolor='black',showgrid=False),\n                 yaxis=dict(title='Height', mirror=True, linewidth=2, linecolor='black',showgrid=False),\n                 plot_bgcolor='rgb(255,255,255)')\n    return fig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_distribution(df_images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def imshow(img):\n    img = img / 2 + 0.5     # unnormalize\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    plt.show()\n\n    \n'''for i in range(5):\n    image, label = train_set[i]\n    temp = np.array(image)\n    print(image.shape)\n    print(label.item())\n    imshow((torchvision.utils.make_grid(image)))'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''for i in range(5):\n    image = test_set[i]\n    temp = np.array(image)\n    print(image.shape)\n    print(label.item())\n    imshow((torchvision.utils.make_grid(image)))'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Image Dataset and dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"IN_KERNEL = os.environ.get('KAGGLE_WORKING_DIR') is not None\nMIN_SAMPLES_PER_CLASS = 2\nBATCH_SIZE = 64\nNUM_WORKERS = multiprocessing.cpu_count()\nMAX_STEPS_PER_EPOCH = 30000\nNUM_EPOCHS = 1\nNUM_TOP_PREDICTS = 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/landmark-recognition-2020/train.csv')\ntrain_dir = '../input/landmark-recognition-2020/train/'\n#test = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\n#test_dir = '../input/landmark-recognition-2020/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageDataset(torch.utils.data.Dataset):\n    def __init__(self, dataframe: pd.DataFrame, image_dir:str, mode: str):\n        self.df = dataframe\n        self.mode = mode\n        self.image_dir = image_dir\n        \n        transforms_list = []\n        if self.mode == 'train':\n            # Increase image size from (64,64) to higher resolution,\n            # Make sure to change in RandomResizedCrop as well.\n            transforms_list = [\n                transforms.Resize((64,64)),\n                transforms.RandomHorizontalFlip(),\n                transforms.RandomChoice([\n                    transforms.RandomResizedCrop(64),\n                    transforms.ColorJitter(0.2, 0.2, 0.2, 0.2),\n                    transforms.RandomAffine(degrees=15, translate=(0.2, 0.2),\n                                            scale=(0.8, 1.2), shear=15,\n                                            resample=Image.BILINEAR)\n                ]),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ]\n        else:\n            transforms_list.extend([\n                # Keep this resize same as train\n                transforms.Resize((64,64)),\n                transforms.ToTensor(),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                      std=[0.229, 0.224, 0.225]),\n            ])\n        self.transforms = transforms.Compose(transforms_list)\n\n    def __getitem__(self, index: int):\n        image_id = self.df.iloc[index].id\n        image_path = f\"{self.image_dir}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\"\n        image = Image.open(image_path)\n        image = self.transforms(image)\n\n        if self.mode == 'test':\n            return {'image':image,\n                   'target':self.df.iloc[index].target}\n        elif self.mode == 'valid':\n            return {'image':image, \n                    'target':self.df.iloc[index].target}\n        else:\n            ### check the label, if label = 0, change the root address\n            return {'image':image, \n                    'target':self.df.iloc[index].target}\n\n    def __len__(self) -> int:\n        return self.df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(train, valid, test, train_dir, test_dir):\n    counts = train.landmark_id.value_counts()\n    selected_classes = counts[counts >= MIN_SAMPLES_PER_CLASS].index\n    num_classes = selected_classes.shape[0]\n    print('classes with at least N samples:', num_classes)\n\n    train = train.loc[train.landmark_id.isin(selected_classes)]\n    print('train_df', train.shape)\n    print('valid_df', valid.shape)\n    print('test_df', test.shape)\n    \n    # filter non-existing validing images\n    exists = lambda img: os.path.exists(f'{test_dir}/{img[0]}/{img[1]}/{img[2]}/{img}.jpg')\n    valid = valid.loc[valid.id.apply(exists)]\n    print('valid_df after filtering', valid.shape)\n\n    # filter non-existing test images\n    exists = lambda img: os.path.exists(f'{test_dir}/{img[0]}/{img[1]}/{img[2]}/{img}.jpg')\n    test = test.loc[test.id.apply(exists)]\n    print('test_df after filtering', test.shape)\n\n    label_encoder = LabelEncoder()\n    label_encoder.fit(train.landmark_id.values)\n    print('found classes', len(label_encoder.classes_))\n    assert len(label_encoder.classes_) == num_classes\n\n    train.landmark_id = label_encoder.transform(train.landmark_id)\n\n    train_dataset = ImageDataset(train, train_dir, mode='train')\n    \n    valid_dataset = ImageDataset(valid, test_dir, mode='valid')\n    \n    test_dataset = ImageDataset(test, test_dir, mode='test')\n\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,\n                              shuffle=True, num_workers=4, drop_last=False)\n    \n    vaild_loader = DataLoader(valid_dataset, batch_size=BATCH_SIZE,\n                              shuffle=True, num_workers=0, drop_last=False)\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE,\n                             shuffle=False, num_workers=NUM_WORKERS, drop_last=False)\n\n    return train_loader, vaild_loader, test_loader, label_encoder, num_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from sklearn.model_selection import train_test_split\ndirectory = '../input/landmark-recognition-2020/'\ndf_train = pd.read_csv(os.path.join(directory,'train.csv'))\ntest_directory = '../input/comp4471projecttestvalid/'\ndf_valid = pd.read_csv(os.path.join(test_directory,'valid.csv'))\ndf_test = pd.read_csv(os.path.join(test_directory,'test.csv'))\n\n### train filter (fil the classes appear in test set from training set)\n#\nlandmark_ids = np.sort(df_train.landmark_id.unique())\nlandmark_id2class = {lid:i for i,lid in enumerate(landmark_ids)}\ndf_train['target'] = df_train['landmark_id'].apply(lambda x: landmark_id2class[x])\n\n### train\nnum_classes = 81313\nval_counts = df_train.target.value_counts().sort_index().values\nclass_weights = 1/np.log1p(val_counts)\nclass_weights = (class_weights / class_weights.sum()) * num_classes\nclass_weights = torch.tensor(class_weights, dtype=torch.float32)\n### valid\nlandmark_id2class_val = landmark_id2class.copy()\nlandmark_id2class_val[-1] = num_classes\ndf_valid['target'] = df_valid['landmark_id'].apply(lambda x: landmark_id2class_val.get(x,-1))\ndf_test['target'] = df_test['landmark_id'].apply(lambda x: landmark_id2class_val.get(x,-1))\n# add 1 to target to shirft it\ndf_train['target'] = df_train['target'] + 1\ndf_valid['target'] = df_valid['target'] + 1\ndf_test['target'] = df_test['target'] + 1\nallowed_classes = np.sort(df_valid[df_valid.target!=num_classes].target.unique())\n\ntest_dir = 'test/'\n\ntrain_loader, valid_loader, test_loader, label_encoder, num_classes = load_data(df_train, df_valid, df_test, train_dir, test_dir)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    ''' Computes and stores the average and current value '''\n    def __init__(self) -> None:\n        self.reset()\n\n    def reset(self) -> None:\n        self.val = 0.0\n        self.avg = 0.0\n        self.sum = 0.0\n        self.count = 0\n\n    def update(self, val: float, n: int = 1) -> None:\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### GPA Evaluations\n\nGlobal Precision Average Evaluation\n\nSubmissions are evaluated using Global Average Precision (GAP) at \\\\(k\\\\), where \\\\(k=1\\\\). This metric is also known as micro Average Precision (\\\\(\\mu\\\\)AP), as per [1,2]. It works as follows:\n\nFor each test image, you will predict one landmark label and a corresponding confidence score. The evaluation treats each prediction as an individual data point in a long list of predictions (sorted in descending order by confidence scores), and computes the Average Precision based on this list.\n\nIf a submission has \\\\(N\\\\) predictions (label/confidence pairs) sorted in descending order by their confidence scores, then the Global Average Precision is computed as:\n\n$$GAP = \\frac{1}{M}\\sum_{i=1}^N P(i) rel(i)$$\n\nwhere:\n\n* \\\\(N\\\\) is the total number of predictions returned by the system, across all queries\n* \\\\(M\\\\) is the total number of queries with at least one landmark from the training set visible in it (note that some queries may not depict landmarks)\n* \\\\(P(i)\\\\) is the precision at rank \\\\(i\\\\)\n* \\\\(rel(i)\\\\) denotes the relevance of prediciton \\\\(i\\\\): it’s 1 if the \\\\(i\\\\)-th prediction is correct, and 0 otherwise"},{"metadata":{"trusted":true},"cell_type":"code","source":"def global_average_precision_score(y_true, y_pred, ignore_non_landmarks=False):\n    indexes = np.argsort(y_pred[1].cpu().numpy())[::-1]\n    queries_with_target = (y_true < num_classes).sum()\n    correct_predictions = 0\n    total_score = 0.\n    i = 1\n    for k in indexes:\n        if ignore_non_landmarks and y_true[k] == num_classes:\n            continue\n        if y_pred[0][k] == num_classes:\n            continue\n        relevance_of_prediction_i = 0\n        if y_true[k] == y_pred[0][k]:\n            correct_predictions += 1\n            relevance_of_prediction_i = 1\n        precision_at_rank_i = correct_predictions / i\n        total_score += precision_at_rank_i * relevance_of_prediction_i\n        i += 1\n    return 1 / queries_with_target * total_score\n\n\ndef comp_metric(y_true, logits, ignore_non_landmarks=False):\n    score = global_average_precision_score(y_true, logits, ignore_non_landmarks=ignore_non_landmarks)\n    return score\n\ndef cos_similarity_matrix(a, b, eps=1e-8):\n    a_n, b_n = a.norm(dim=1)[:, None], b.norm(dim=1)[:, None]\n    a_norm = a / torch.max(a_n, eps * torch.ones_like(a_n))\n    b_norm = b / torch.max(b_n, eps * torch.ones_like(b_n))\n    sim_mt = torch.mm(a_norm, b_norm.transpose(0, 1))\n    return sim_mt\n\ndef get_topk_cossim(test_emb, tr_emb, batchsize = 64, k=10, device='cuda:0',verbose=True):\n    tr_emb = torch.tensor(tr_emb, dtype = torch.float32, device=torch.device(device))\n    test_emb = torch.tensor(test_emb, dtype = torch.float32, device=torch.device(device))\n    vals = []\n    inds = []\n    for test_batch in test_emb.split(batchsize):\n        sim_mat = cos_similarity_matrix(test_batch, tr_emb)\n        vals_batch, inds_batch = torch.topk(sim_mat, k=k, dim=1)\n        vals += [vals_batch.detach().cpu()]\n        inds += [inds_batch.detach().cpu()]\n    vals = torch.cat(vals)\n    inds = torch.cat(inds)\n    return vals, inds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train function"},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy \nfrom collections import OrderedDict\nfrom pytorch_lightning import Trainer\nfrom sklearn.preprocessing import StandardScaler\ndef train_step(train_loader, \n          valid_loader,\n          model,\n          num_classes,\n          metric_crit,\n          metric_crit_val,\n          optimizer,\n          num_epochs, \n          lr_scheduler, MAX_STEPS_PER_EPOCH, allowed_classes, name):\n    \n    for epoch in range(num_epochs):\n        print(f'epoch {epoch}')\n        batch_time = AverageMeter()\n        losses = AverageMeter()\n        valid_avg_score=AverageMeter()\n        valid_avg_score_landmarks = AverageMeter()\n        valid_avg_score_pp = AverageMeter()\n        valid_avg_score_landmarks_pp = AverageMeter()\n        valid_avg_loss=AverageMeter()\n        \n        train_loss = []\n        lr_list = []\n        val_loss_list = []\n        val_score_list = []\n        val_score_landmarks_list = []\n        val_score_pp_list = []\n        val_score_landmarks_pp_list = []\n\n        model.train()\n        num_steps = min(len(train_loader), MAX_STEPS_PER_EPOCH)\n\n        print(f'total batches: {num_steps}')\n\n        end = time.time()\n        lr = None\n\n        for iter, data in enumerate(tqdm_notebook(train_loader)):\n            input_ = data['image']\n            target = data['target']\n            batch_size, _, _, _ = input_.shape\n            \n            train_output = model(input_.to(device))\n            #loss = criterion(train_output, target.to(device))\n            \n            loss = loss_fn(metric_crit, data, train_output, num_classes)\n            s = metric_crit.s\n            tb_dict = {'train_loss': loss, 'arcface_s':s}\n            for i,param_group in enumerate(optimizer.param_groups):\n                tb_dict[f'lr/lr{i}'] = param_group['lr']\n                \n            output = OrderedDict({\n                'loss': loss,\n                'log': tb_dict,\n            })\n            #print(tb_dict['train_loss'].cpu().item())\n            \n            losses.update(tb_dict['train_loss'].cpu().item(), input_.size(0))\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            lr_scheduler.step()\n            lr = optimizer.param_groups[0]['lr']\n            batch_time.update(time.time() - end)\n            end = time.time()\n            if iter % (BATCH_SIZE-1) == 0:\n                train_loss.append(losses.avg)\n                lr_list.append(lr)\n                print(f'epoch {epoch} [{iter}/{num_steps}]\\t'\n                        f'time {batch_time.avg:.3f}\\t'\n                        f'loss {losses.avg:.4f}\\t'\n                        + f'lr {lr:.4f}')\n                \n        # validation\n        with torch.no_grad():\n            model.eval()\n            for i, data in enumerate(tqdm(valid_loader)):\n                valid_input_ = data['image']\n                valid_target = data['target']\n                valid_batch_size, _, _, _ = valid_input_.shape\n                valid_output = model(valid_input_.to(device), get_embeddings=True)\n                loss = loss_fn(metric_crit_val, data, valid_output, num_classes, val=True)\n                logits = valid_output['logits']\n                embeddings = valid_output['embeddings']\n                preds_conf, preds = torch.max(logits.softmax(1),1)\n                temp_allowed_classes = torch.Tensor(allowed_classes).long().to(logits.device)\n                temp = temp_allowed_classes.repeat(logits.size(0),1)\n                ### replace -1 to 0\n                temp[temp==-1] = 0\n                \n                preds_conf_pp, preds_pp = torch.max(logits.gather(1, temp).softmax(1),1)\n                preds_pp = temp_allowed_classes[preds_pp]\n                val_score = comp_metric(valid_target.to(device), [preds, preds_conf])   \n                val_score_landmarks = comp_metric(valid_target.to(device), [preds, preds_conf], ignore_non_landmarks=True)\n                val_score_pp = comp_metric(valid_target.to(device), [preds_pp, preds_conf_pp])\n                val_score_landmarks_pp = comp_metric(valid_target.to(device), [preds_pp, preds_conf_pp], ignore_non_landmarks=True)\n                #print(np.sum(loss.view(1).item()))\n                valid_avg_score.update(val_score)\n                valid_avg_score_landmarks.update(val_score_landmarks)\n                valid_avg_score_pp.update(val_score_pp)\n                valid_avg_score_landmarks_pp.update(val_score_landmarks_pp)\n                valid_avg_loss.update(np.sum(loss.view(1).item()))\n            val_score_list.append(valid_avg_score.avg)\n            val_score_landmarks_list.append(valid_avg_score_landmarks.avg)\n            val_score_pp_list.append(valid_avg_score_pp.avg)\n            val_score_landmarks_pp_list.append(valid_avg_score_landmarks_pp.avg)\n            val_loss_list.append(valid_avg_loss.avg)\n            print(f'valid {epoch}\\t'\n                        f'loss {valid_avg_loss.avg:.4f}\\t'\n                        f'score {valid_avg_score.avg:.4f}\\t'\n                        f'score_lms {valid_avg_score_landmarks.avg:.4f}\\t'\n                        f'score_pp {valid_avg_score_pp.avg:.4f}\\t'\n                        f'score_lms_pp {valid_avg_score_landmarks_pp.avg:.4f}\\t'\n                 )\n        # save data to csv\n        # train data\n        train_data=np.array([train_loss, lr_list]).T\n        df_tr_data = pd.DataFrame(train_data, columns=['loss', 'lr'])\n        df_tr_data.to_csv('train_data_epoch' + str(epoch) + '.csv', index=False)\n        #valid data\n        valid_data=np.array([val_loss_list, val_score_list, val_score_landmarks_list, val_score_pp_list,val_score_landmarks_pp_list]).T\n        df_tr_data = pd.DataFrame(valid_data, columns=['loss', 'score', 'score_landmark', 'score_pp', 'score_landmark_pp'])\n        df_tr_data.to_csv('valid_data_epoch' + str(epoch) + '.csv', index=False)\n\n    print('Finish training')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install transformers\nfrom transformers import get_cosine_schedule_with_warmup\ndef lr_sched(optimizer, train_loader, lr_type=None):\n    if(lr_type == \"Cosine\"):\n        gradient_accumulation_steps = 4\n        max_epochs = 10\n        warmup_epochs = 1\n        num_train_steps = math.ceil(len(train_loader) / (len('0,1'.split(','))*gradient_accumulation_steps) )\n        return get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=num_train_steps * warmup_epochs, \n                                                        num_training_steps=int(num_train_steps * (max_epochs)))\n        #return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader), eta_min=1e-6)\n    elif(lr_type == \"StepLR\"):\n        return torch.optim.lr_scheduler.StepLR(optimizer, step_size=64, gamma=0.999)\n    else:\n        ### exponentLR\n        return torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99)\n    \ndef get_optimizer(model, metric_crit, optimizer_type=None):\n    if(optimizer_type=='sgd'):\n        return torch.optim.SGD([{'params': model.parameters()}, {'params': metric_crit.parameters()}], lr=1e-3, momentum=0.9, nesterov=True, weight_decay=1e-4)\n    else:\n        return torch.optim.Adam([{'params': model.parameters()}, {'params': metric_crit.parameters()}], lr=1e-3, betas=(0.9,0.999), weight_decay=1e-4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ArcMarginProduct"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.nn import functional as F\nfrom torch.nn.parameter import Parameter\nclass ArcMarginProduct(nn.Module):\n    def __init__(self, in_features, out_features):\n        super().__init__()\n        self.weight = nn.Parameter(torch.Tensor(out_features, in_features))\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        nn.init.xavier_uniform_(self.weight)\n\n    def forward(self, features):\n        cosine = F.linear(F.normalize(features), F.normalize(self.weight))\n        return cosine","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### EfficientNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"### EfficientNEt\nclass EfficientNetEncoderHead(nn.Module):\n    def __init__(self, depth, num_classes):\n        super(EfficientNetEncoderHead, self).__init__()\n        self.depth = depth\n        self.base = EfficientNet.from_pretrained(f'efficientnet-b{self.depth}')\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.output_filter = self.base._fc.in_features\n        self.classifier = nn.Linear(self.output_filter, num_classes)\n        ### follows the 1st in 2019\n        #self.final = ArcMarginProduct(in_features=num_classes, out_features=num_classes, s=30, m=0.50, easy_margin=False, ls_eps=0.0)\n        \n    def forward(self, x):\n        x = self.base.extract_features(x)\n        x = self.avg_pool(x).squeeze(-1).squeeze(-1)\n        x = self.classifier(x)\n        ### follows the 1st in 2019\n        #x = self.final(x, label)\n        return x\n### Resnet\nclass ResNetXT101(nn.Module):\n    def __init__(self, num_classes, in_channels=3):\n        super(ResNetXT101, self).__init__()\n\n        # Load a pretrained resnet model from torchvision.models in Pytorch\n        self.model = models.resnext101_32x8d(pretrained=True)\n\n        # Change the input layer to take Grayscale image, instead of RGB images. \n        # Hence in_channels is set as 1 or 3 respectively\n        # original definition of the first layer on the ResNet class\n        # self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        self.model.conv1 = nn.Conv2d(in_channels, 64, kernel_size=7, stride=2, padding=3, bias=False)\n\n        # Change the output layer to output 10 classes instead of 1000 classes\n        num_ftrs = self.model.fc.in_features\n        self.model.fc = nn.Linear(num_ftrs, num_classes)\n\n    def forward(self, x):\n        return self.model(x)\n    \nclass Net(nn.Module):\n    def __init__(self, num_classes, model_name=None, embedding_size=512):\n        super(Net, self).__init__()\n        if(model_name=='resnet'):\n            self.model = ResNetXT101(numclasses)\n        else:\n            self.model = EfficientNetEncoderHead(depth=7, num_classes=num_classes)\n        self.neck = nn.Sequential(\n                nn.Linear(num_classes, embedding_size, bias=True),\n                nn.BatchNorm1d(embedding_size),\n                torch.nn.PReLU()\n            )\n        self.head = ArcMarginProduct(embedding_size, num_classes)\n    def forward(self, x, get_embeddings=False, get_attentions=False):\n        x = self.model(x)\n        x = self.neck(x)\n        logits = self.head(x)\n        \n        if get_embeddings:\n            return {'logits': logits, 'embeddings': x}\n        else:\n            return {'logits': logits}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(device)\nmodel = Net(num_classes, model_name='effnet')\nsummary(model.cuda(), (3, 64, 64))\n#efficientnet = EfficientNetEncoderHead(depth=7, num_classes=num_classes)\n#efficientnet = efficientnet.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nclass ArcFaceLoss(nn.modules.Module):\n    def __init__(self, s=45.0, m=0.1, crit=\"bce\", weight=None, reduction=\"mean\", class_weights_norm=None):\n        super().__init__()\n        \n        self.weight = weight\n        self.reduction = reduction\n        \n        if crit == \"focal\":\n            self.crit = FocalLoss(gamma=args.focal_loss_gamma)\n        elif crit == \"bce\":\n            self.crit = nn.CrossEntropyLoss(reduction=\"none\")   \n\n        if s is None:\n            self.s = torch.nn.Parameter(torch.tensor([45.], requires_grad=True, device='cuda'))\n        else:\n            self.s = s\n\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n        \n        self.class_weights_norm = class_weights_norm\n        \n    def forward(self, logits, labels):\n\n        logits = logits.float()\n        cosine = logits\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        \n        labels2 = torch.zeros_like(cosine)\n        labels2.scatter_(1, labels.view(-1, 1).long(), 1)\n        output = (labels2 * phi) + ((1.0 - labels2) * cosine)\n\n        s = self.s\n\n        output = output * s\n        loss = self.crit(output, labels)\n\n        if self.weight is not None:\n            w = self.weight[labels].to(logits.device)\n\n            loss = loss * w\n            if self.class_weights_norm == \"batch\":\n                loss = loss.sum() / w.sum()\n            if self.class_weights_norm == \"global\":\n                loss = loss.mean()\n            else:\n                loss = loss.mean()\n            \n            return loss\n        if self.reduction == \"mean\":\n            loss = loss.mean()\n        elif self.reduction == \"sum\":\n            loss = loss.sum()\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_fn(metric_crit, target_dict, output_dict, num_classes, val=False):\n    \n    y_true = target_dict['target'].to(device)\n    y_pred = output_dict['logits']\n    #ignore invalid classes for val loss\n    mask = y_true < num_classes\n    if mask.sum() == 0:\n        return torch.zeros(1,  device = y_pred.device)\n    loss = metric_crit(y_pred[mask], y_true[mask])\n\n    return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer, loss func and scheduler"},{"metadata":{"trusted":true},"cell_type":"code","source":"landmark_ids = np.sort(df_train.landmark_id.unique())\nlandmark_id2class = {lid:i for i,lid in enumerate(landmark_ids)}\ntrain['target'] = train['landmark_id'].apply(lambda x: landmark_id2class[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#criterion = nn.CrossEntropyLoss()\nmetric_crit = ArcFaceLoss(s=45, m=0.4, crit=\"bce\", weight=class_weights, class_weights_norm='batch')\nmetric_crit_val = ArcFaceLoss(s=45, m=0.4, crit=\"bce\", weight=None, reduction=\"sum\", class_weights_norm=None)\noptimizer = get_optimizer(model, metric_crit, optimizer_type='Adam')\n### decrease the learning rate\nlr_scheduler = lr_sched(optimizer, train_loader, \"StepLR\")\nprint(lr_scheduler)\n#MAX_STEPS_PER_EPOCH = 30000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### True, if there is saved model\nif True:\n    checkpoint = torch.load('../input/comp4471efficmodel/EffNet_ADAM_checkpoint.pth')\n    model = checkpoint['model']\n    #optimizer.load_state_dict(checkpoint['optimizer'])\n    metric_crit = checkpoint['metric_crit']\n    lr_scheduler = checkpoint['scheduler']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### EfficientNet Training Process"},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch=1\ntrain_step(train_loader, valid_loader, model.to(device), num_classes, metric_crit, metric_crit_val, optimizer, epoch, lr_scheduler, MAX_STEPS_PER_EPOCH, allowed_classes, \"effnetb7\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save EfficientNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"### save model, optimizer and scheduler\ncheckpoint = { \n    'model': model,\n    'optimizer': optimizer.state_dict(),\n    'scheduler': lr_scheduler.state_dict(),\n    'metric_crit': metric_crit}\ntorch.save(checkpoint, 'EfficientNet_adam_step_checkpoint.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### download model\nimport os\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink(r'EfficientNet_adam_step_checkpoint.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loss and GAP"},{"metadata":{"trusted":true},"cell_type":"code","source":"df0 = pd.read_csv('../input/comp4471efficmodel/EffNet_SGD_train_data_epoch0.csv')\ndf1 = pd.read_csv('../input/comp4471efficmodel/EffNet_SGD_train_data_epoch1.csv')\ndf2 = pd.read_csv('../input/comp4471efficmodel/ResNet_SGD_train_epoch0.csv')\ndf3 = pd.read_csv('../input/comp4471efficmodel/ResNet_SGD_train_epoch1.csv')\n#df4 = \n#df5 = \n#df6 = \n#df7 = \ndata = pd.concat([df0, df1])\nsteps = range(len(data))\ndata2 = pd.concat([df0, df2])\nfig, ax1 = plt.subplots()\n###\nplt.title('Loss (EfficientNetb7)')\nax1.plot(steps, data['loss'], label='train loss', color='blue')\nax1.tick_params(axis='y', labelcolor='blue')\n#ax1.legend()\nax1.set_xlabel('steps')\nax1.set_ylabel('loss', color='blue')\nax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\nax2.set_ylabel('lr', color='red')  # we already handled the x-label with ax1\nax2.plot(steps, data['lr'], label='learning rate', color='red')\nax2.tick_params(axis='y', labelcolor='red')\nfig.legend(loc=\"upper right\")\nplt.axvline(x=391, ls='--', color='green')\nplt.show()\n###\nfig, ax1 = plt.subplots()\nplt.title('Loss (ResNet)')\nax1.plot(steps, data2['loss'], label='train loss', color='blue')\nax1.tick_params(axis='y', labelcolor='blue')\nax1.set_xlabel('steps')\nax1.set_ylabel('loss', color='blue')\nax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis\nax2.set_ylabel('lr', color='red')  # we already handled the x-label with ax1\nax2.plot(steps, data2['lr'], label='learning rate', color='red')\nax2.tick_params(axis='y', labelcolor='red')\nfig.legend(loc=\"upper right\")\nplt.axvline(x=391, ls='--', color='green')\n###\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Evaluation (Model ebsemble)"},{"metadata":{"trusted":true},"cell_type":"code","source":"effnet_checkpoint = torch.load('../input/comp4471efficmodel/EfficientNet_checkpoint.pth')\nmodel1 = effnet_checkpoint['model']\nresnet_checkpoint = torch.load('../input/comp4471efficmodel/ResNet_SGD_checkpoint.pth')\nmodel2 = resnet_checkpoint['model']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import QuantileTransformer\nwith torch.no_grad():\n    model1 = model1\n    model2 = model2\n    model1.eval() # efficientNet7b\n    model2.eval() # resnetxt101\n    test_avg_score = AverageMeter()\n    test_avg_score_landmarks = AverageMeter()\n    test_avg_score_pp = AverageMeter()\n    test_avg_score_landmarks_pp = AverageMeter()\n    for i, data in enumerate(tqdm(test_loader)):\n        test_input_ = data['image']\n        test_target = data['target']\n        test_batch_size, _, _, _ = test_input_.shape\n        test_output1 = model1(test_input_.to(device), get_embeddings=True) # model1 out\n        test_output2 = model1(test_input_.to(device), get_embeddings=True) # model2 out\n        logits1 = test_output1['logits']\n        logits2 = test_output2['logits']\n        logits = (logits1 + logits2)/2\n        preds_conf, preds = torch.max(logits.softmax(1),1)\n        temp_allowed_classes = torch.Tensor(allowed_classes).long().to(logits.device)\n        temp = temp_allowed_classes.repeat(logits.size(0),1)\n        preds_conf_pp, preds_pp = torch.max(logits.gather(1, temp).softmax(1),1)\n        preds_pp = temp_allowed_classes[preds_pp]    \n        test_score = comp_metric(test_target.to(device), [preds, preds_conf])\n        test_score_landmarks = comp_metric(test_target.to(device), [preds, preds_conf], ignore_non_landmarks=True)\n        test_score_pp = comp_metric(test_target.to(device), [preds_pp, preds_conf_pp])\n        test_score_landmarks_pp = comp_metric(test_target.to(device), [preds_pp, preds_conf_pp], ignore_non_landmarks=True)\n        test_avg_score.update(test_score)\n        test_avg_score_landmarks.update(test_score_landmarks)\n        test_avg_score_pp.update(test_score_pp)\n        test_avg_score_landmarks_pp.update(test_score_landmarks_pp)\n  \n    print(f'score {test_avg_score.avg:.4f}\\t'\n          f'score_lms {test_avg_score_landmarks.avg:.4f}\\t'\n          f'score_pp {test_avg_score_pp.avg:.4f}\\t'\n          f'score_lms_pp {test_avg_score_landmarks_pp.avg:.4f}\\t')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}