{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n       # print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install faiss\n!pip install faiss-gpu\nimport faiss\n#import faiss-gpu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!git clone https://github.com/amarmrf/google-retrieval-challenge-2019-fastai-starter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://github.com/filipradenovic/cnnimageretrieval-pytorch/blob/master/cirtorch/utils/download.py\n#from third_party import download\n\nimport sys\nsys.path.insert(1, 'google-retrieval-challenge-2019-fastai-starter/third_party')\nimport download","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR='data/'\n!mkdir $DATA_DIR","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"download.download_test(DATA_DIR)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nplt.imshow(Image.open('data/test/oxford5k/jpg/oxford_001195.jpg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"download.download_train(DATA_DIR)\n#takes ~40 min","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from third_party.datahelpers import cid2filename\nfrom datahelpers import cid2filename\nimport pickle\n\nwith open('data/train/retrieval-SfM-120k/retrieval-SfM-120k.pkl', 'rb') as f:\n    db = pickle.load(f)['train']\n    \nimages = [cid2filename(db['cids'][i], 'data/train/retrieval-SfM-120k/ims/') for i in range(len(db['cids']))]\nclusters = db['cluster']\nqpool = db['qidxs']\nppool = db['pidxs']\nqpidxs = list(set([(qpool[i], ppool[i]) for i in range(len(ppool))]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Gather all \"query+positive pool\" into classes\ninvert_list = {}\nsubclusters = {}\nsubidx = 0\nfor q,p in qpidxs:\n    #print (q,p)\n    if q not in invert_list and p not in invert_list:\n        subclusters[subidx] = []\n        subclusters[subidx].append(p)\n        subclusters[subidx].append(q)\n        invert_list[p] = subidx\n        invert_list[q] = subidx\n        subidx+=1\n    else:\n        if p in invert_list:\n            exist_sub_idx = invert_list[p]\n            invert_list[q] = exist_sub_idx\n        else:\n            exist_sub_idx = invert_list[q]\n            invert_list[p] = exist_sub_idx\n        subclusters[exist_sub_idx].append(p)\n        subclusters[exist_sub_idx].append(q)\nprint (len(subclusters))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Here we create full csv and small (for quick experimenting)\nimport os\ncount_no_class= 0\nprefix = os.path.join(DATA_DIR, 'train/retrieval-SfM-120k/ims/')\n\nwith open('train_sfm120k_1000_classes.csv', 'w') as f:\n    f.write('Image,cluster,Id\\n')\n    for i in range(len(images)):\n        img = images[i].replace(prefix,'')\n        if i in invert_list:\n            if invert_list[i] > 1000:\n                continue\n            f.write(img+','+str(clusters[i])+','+str(invert_list[i])+'\\n')\nprint (count_no_class, len(images))\nwith open('train_sfm120k_full.csv', 'w') as f:\n    f.write('Image,cluster,Id\\n')\n    for i in range(len(images)):\n        img = images[i].replace(prefix,'')\n        if i in invert_list:\n            f.write(img+','+str(clusters[i])+','+str(invert_list[i])+'\\n')\n        else:\n            f.write(img+','+str(clusters[i])+',-1\\n')\n            count_no_class+=1\nprint (count_no_class, len(images))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Val no train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#import sys\n#sys.path.insert(1, 'google-retrieval-challenge-2019-fastai-starter/third_party')\nimport matplotlib.pyplot as plt\nfrom testdataset import configdataset\nfrom evaluate import compute_map_and_print\n\nfrom fastai import *\nfrom fastai.vision import *\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset ='oxford5k'\nDATA_DIR='data/'\ncfg = configdataset(dataset, os.path.join(DATA_DIR, 'test'))\nimages = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]\nqimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(images, columns=['Image'])\nqdf  = pd.DataFrame(qimages, columns=['qimages'])\nbbxs = {qimages[i]:tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BS=1\nNUM_WORKERS=8\ntfms = get_transforms(do_flip=False)\ntfms = (tfms[1],tfms[1]) #no transforms\ndata = (ImageList.from_df(df,'', cols=['Image'])\n        .split_none()\n        .label_const()\n        .transform(tfms, resize_method=ResizeMethod.NO)\n        .databunch(bs=BS, num_workers=NUM_WORKERS)\n        .normalize(imagenet_stats)\n       ) \ndata.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(data.train_ds)\ndata.train_dl.dl.batch_sampler.drop_last = False\n#No shuffle\ndata.show_batch()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(1, 'google-retrieval-challenge-2019-fastai-starter')\nfrom arch import GeM, L2Norm\nclass GeMNet(nn.Module):\n    def __init__(self, new_model):\n        super().__init__()\n        self.cnn =  new_model.features\n        self.head = nn.Sequential(nn.ReLU(),\n                                  GeM(3.0),\n                                  Flatten(),\n                                  L2Norm())\n    def forward(self, x):\n        x = self.cnn(x)\n        out = self.head(x)\n        return out\nInferenceNet =  GeMNet(models.densenet121(pretrained=True))\nOUT_DIM = 1024","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We will extract database descriptors, then query descriptors and then calculate mean average precision (mAP)\nfrom fastprogress import master_bar, progress_bar\ndef extract_vectors(data,model):\n    model.cuda()\n    model.eval()\n    with torch.no_grad():\n        vectors = torch.zeros(len(data.train_dl), OUT_DIM).cpu()\n        idx=0\n        for img_label in progress_bar(data.train_dl):\n            img,label = img_label\n            vectors[idx,:] = model(img).cpu()\n            idx+=1\n    return vectors\ndb_vectors = extract_vectors(data,InferenceNet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def open_image_cropped(fn:PathOrStr, div:bool=True, convert_mode:str='RGB', cls:type=Image)->Image:\n    \"Return `Image` object created from image in file `fn`.\"\n    #fn = getattr(fn, 'path', fn)\n    x = PIL.Image.open(fn).convert(convert_mode).crop(bbxs[str(fn).replace('./','')])\n    x = pil2tensor(x,np.float32)\n    if div: x.div_(255)\n    return cls(x)\n\nclass ImageItemCrop(ImageList):\n    def open(self, fn:PathOrStr)->Image:\n        return open_image_cropped(fn)\nBS=1\nNUM_WORKERS=8\ntfms = get_transforms(do_flip=False)\ntfms = (tfms[1],tfms[1]) #no transforms\n\nquery_data = (ImageItemCrop.from_df(qdf,'', cols=['qimages'])\n        .split_none()\n        .label_const()\n        .transform(tfms, resize_method=ResizeMethod.NO)\n        .databunch(bs=BS, num_workers=NUM_WORKERS)\n        .normalize(imagenet_stats)\n       ) \nquery_data.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(query_data.train_ds)\nquery_data.train_dl.dl.batch_sampler.drop_last = False\n#No shuffle\nquery_data.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"query_vectors = extract_vectors(query_data,InferenceNet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('>> {}: Evaluating...'.format(dataset))\n# convert to numpy\nvecs = db_vectors.numpy()\nqvecs = query_vectors.numpy()\n# search, rank, and print\nscores = np.dot(vecs, qvecs.T)\nranks = np.argsort(-scores, axis=0)\ncompute_map_and_print(dataset, ranks, cfg['gnd'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this might be bad, smlyaka do 89 something in oxford set and got 33 in retrieval 2019","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Train val good","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.environ['CUDA_VISIBLE_DEVICES']=\"1\"\n!pip install pretrainedmodels\n\nfrom fastprogress import master_bar, progress_bar\nfrom fastai.vision import *\nfrom fastai.metrics import accuracy\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\nimport torch.nn as nn\nimport pretrainedmodels\nimport math\nimport sys\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lets create validations set\ndf = pd.read_csv('train_sfm120k_full.csv')\n\ndf.Id=df.Id.apply(str)\ndf.cluster=df.cluster.apply(str)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im_count = df[df.cluster != \"-1\"].cluster.value_counts()\nim_count.name = 'sighting_count'\ndf = df.join(im_count, on='cluster')\ndf.head()\npath_prefix = 'data/train/retrieval-SfM-120k/ims'\nif True:\n    val_fns = set(df.sample(frac=1)[(df.cluster != \"-1\") & (df.sighting_count > 2)].groupby('cluster').first().Image)\n    val_fns = list(val_fns)\n    val_fns2= []\n    for k in val_fns:\n        val_fns2.append(path_prefix+'/'+k)\n    pd.to_pickle(val_fns2, 'data/val_fns_from_train_full.pkl')\n\nval_fns = pd.read_pickle('data/val_fns_from_train_full.pkl')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Bagian bawah ini keluar error kalo di kaggle, kalo di colab bahkan gak nyampe sini udah ke cancel runtime nya","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SZ = 256\nBS = 48\nNUM_WORKERS = 10\nSEED=0\n\n\nDATA_DIR='data/'\n\nname = f'DenseNet121-GeMConst-{SZ}-Ring-CELU'\nfn2label = {path_prefix+'/'+row[1].Image: row[1].cluster for row in df.iterrows()}\ndata = (\n    ImageList\n        .from_df(df[df.cluster != \"-1\"], path=path_prefix, cols=['Image'])\n        .split_by_valid_func(lambda path: path in val_fns)\n        .label_from_func(lambda path: fn2label[path])\n        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='')\n        .normalize(imagenet_stats)\n)\ndata.show_batch(rows=3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_CLASSES= len(data.classes)\nprint (NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from arch import GeM, L2Norm\n\n# Based on my solution to whale-identification-competition\n# https://medium.com/@ducha.aiki/thanks-radek-7th-place-solution-to-hwi-2019-competition-738624e4c885\nclass RingHead(nn.Module):\n    def __init__(self, num_classes, feat_dim, in_feat = 1024, r_init =1.5):\n        super(RingHead,self).__init__()\n        self.eps = 1e-10\n        self.num_classes = num_classes\n        self.feat_dim = feat_dim\n        self.feature_extractor = nn.Sequential(\n                        nn.ReLU(),\n                        GeM(3.74), Flatten(),\n                        nn.BatchNorm1d(in_feat, eps=1e-05, momentum=0.1,\n                                                       affine=True, track_running_stats=True),\n                        nn.Dropout(p=0.3),\n                        nn.Linear(in_features=in_feat, out_features=feat_dim, bias=True),\n                        nn.CELU(inplace=True),\n                        nn.BatchNorm1d(feat_dim,eps=1e-05, momentum=0.1,\n                                                       affine=True, track_running_stats=True))\n        \n        self.ring =  nn.Parameter(torch.ones(1).cuda()*r_init)\n        self.clf = nn.Sequential(nn.Dropout(p=0.5),\n                        nn.Linear(in_features=feat_dim, out_features=num_classes, bias=False))\n    def forward(self, x):\n        feats = self.feature_extractor(x)\n        preds = self.clf(feats)\n        return preds,feats\n\nclass RingGeMNet(nn.Module):\n    def __init__(self, new_model, n_classes, in_feats=1024, out_feats=1024):\n        super().__init__()\n        self.cnn =  new_model.features\n        self.head = RingHead(NUM_CLASSES, out_feats, in_feats)\n    def forward(self, x):\n        x = self.cnn(x)\n        preds,feats = self.head(x)\n        return preds,feats\n@dataclass\nclass RingLoss(Callback):\n    learn:Learner\n    alpha:float=0.01\n    def on_loss_begin(self, last_output:Tuple[tensor,tensor], **kwargs):\n        \"Save the extra outputs for later and only returns the true output.\"\n        self.feature_out = last_output[1]\n        return {'last_output':last_output[0]}\n\n    def on_backward_begin(self,\n                          last_loss:Rank0Tensor,\n                          **kwargs):\n        \n        x = self.feature_out\n        R = self.learn.model.head.ring\n        loss = None\n        cc=0\n        x_norm = x.pow(2).sum(dim=1).pow(0.5)\n        diff = torch.mean(torch.abs(x_norm - R.expand_as(x_norm))**2)\n        if loss is None:\n            loss = diff.mean()\n        else:\n            loss = loss + diff.mean()\n        if self.alpha != 0.:  last_loss += (self.alpha * loss).sum()\n        return {'last_loss':last_loss}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = Learner(data, RingGeMNet(models.densenet121(pretrained=True), NUM_CLASSES),\n                   metrics=[accuracy],\n                   loss_func=nn.CrossEntropyLoss(),\n                   callback_fns = [RingLoss])\nlearn.split([learn.model.cnn, learn.model.head])\nlearn.freeze()\nlearn.clip_grad();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(20, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()\nlearn.recorder.plot_metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"name = f'DenseNet121-GeM-{SZ}-Ring-CELU'\nlearn.save(name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()\nmax_lr = 1e-3\nlrs = [max_lr/5., max_lr]\nlearn.fit_one_cycle(32, lrs)\nlearn.recorder.plot_losses()\nlearn.recorder.plot_metrics()\nlearn.save(name+\"_unfreeze\")\n#Accuracy gain seems minor, but key factor here is to finetune backbone network","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nimport torch\nfrom fastprogress import master_bar, progress_bar\nimport PIL\nimport matplotlib.pyplot as plt\nfrom third_party.testdataset import configdataset\nfrom third_party.evaluate import compute_map_and_print\n\nimport pandas as pd\n\n\ndef open_image_cropped(fn:PathOrStr, div:bool=True, convert_mode:str='RGB', cls:type=Image)->Image:\n    \"Return `Image` object created from image in file `fn`.\"\n    #fn = getattr(fn, 'path', fn)\n    x = PIL.Image.open(fn).convert(convert_mode).crop(bbxs[str(fn).replace('./','')])\n    x = pil2tensor(x,np.float32)\n    if div: x.div_(255)\n    return cls(x)\n\nclass ImageItemCrop(ImageList):\n    def open(self, fn:PathOrStr)->Image:\n        return open_image_cropped(fn)\n\ndef extract_vectors_batched(data,model,bs):\n    model.cuda()\n    model.eval()\n    num_img = len(data.train_ds)\n    vectors = None\n    with torch.no_grad():\n        idx=0\n        for img_label in progress_bar(data.train_dl):\n            st=idx*bs\n            fin=min((idx+1)*bs, num_img)\n            img,label = img_label\n            out = model(img).cpu()\n            if vectors is None:\n                vectors = torch.zeros(num_img, out.size(1))\n            vectors[st:fin,:] = out\n            idx+=1\n    return vectors\n\ndef validate_on_dataset(model, dataset_name='oxford5k', DATA_DIR='data/'):\n    cfg = configdataset(dataset_name, os.path.join(DATA_DIR, 'test'))\n    images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]\n    qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]\n    df = pd.DataFrame(images, columns=['Image'])\n    qdf  = pd.DataFrame(qimages, columns=['qimages'])\n    global bbxs\n    bbxs = {qimages[i]:tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])}\n    BS=1\n    NUM_WORKERS=8\n    tfms = get_transforms(do_flip=False)\n    tfms = (tfms[1],tfms[1]) #no transforms\n    query_data = (ImageItemCrop.from_df(qdf,'', cols=['qimages'])\n        .split_none()\n        .label_const()\n        .transform(tfms, resize_method=ResizeMethod.NO)\n        .databunch(bs=BS, num_workers=NUM_WORKERS)\n        .normalize(imagenet_stats)\n       )\n    query_data.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(query_data.train_ds)\n    query_data.train_dl.dl.batch_sampler.drop_last = False\n    print ('Extracting query features...')\n    query_vectors = extract_vectors_batched(query_data,model, 1)\n    data = (ImageList.from_df(df,'', cols=['Image'])\n            .split_none()\n            .label_const()\n            .transform(tfms, resize_method=ResizeMethod.NO)\n            .databunch(bs=BS, num_workers=NUM_WORKERS)\n            .normalize(imagenet_stats)\n           )\n    data.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(data.train_ds)\n    data.train_dl.dl.batch_sampler.drop_last = False\n    print ('Extracting index features...')\n    db_vectors = extract_vectors_batched(data,model,1)\n    print('>> {}: Evaluating...'.format(dataset_name))\n    # convert to numpy\n    vecs = db_vectors.numpy()\n    qvecs = query_vectors.numpy()\n    # search, rank, and print\n    scores = np.dot(vecs, qvecs.T)\n    ranks = np.argsort(-scores, axis=0)\n    compute_map_and_print(dataset_name, ranks, cfg['gnd'])\n    return vecs, qvecs #If you want to check some kind of query expansion","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Neccessary functions from validation notebook are already in utils.py. So lets just import them\n#from utils import validate_on_dataset\n\n#Lets create inference network without clf output and with normalization:\nclass RingGeMNetInfer(nn.Module):\n    def __init__(self, new_model, n_classes, in_feats=1024, out_feats=1024):\n        super().__init__()\n        self.cnn =  new_model.features\n        self.head = RingHead(NUM_CLASSES, out_feats, in_feats)\n    def forward(self, x):\n        x = self.cnn(x)\n        preds,feats = self.head(x)\n        return L2Norm()(feats)\n\nInferenceNet =  RingGeMNetInfer(models.densenet121(pretrained=True),\n                               NUM_CLASSES)\nInferenceNet.load_state_dict(learn.model.state_dict())\n\n\nval_index_feats, val_query_feats = validate_on_dataset(InferenceNet,'oxford5k')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#What about original barebone network?\nclass GeMNet(nn.Module):\n    def __init__(self, new_model):\n        super().__init__()\n        self.cnn =  new_model.features\n        self.head = nn.Sequential(nn.ReLU(),\n                                  GeM(3.0),\n                                  Flatten(),\n                                  L2Norm())\n    def forward(self, x):\n        x = self.cnn(x)\n        out = self.head(x)\n        return out\n\nInferenceNet =  GeMNet(models.densenet121(pretrained=True))\nval_index_feats, val_query_feats = validate_on_dataset( InferenceNet,'oxford5k')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('now finetuned network')\nInferenceNet.cnn.load_state_dict(learn.model.cnn.state_dict())\nval_index_feats, val_query_feats = validate_on_dataset(InferenceNet, 'oxford5k')\n\n#Nice! We have improved from 41 mAP on Oxford5k to 50 in just 45 minutes.\n# It also seems that learned \"head\" is too specific to our training dataset, so better to discard it","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now let`s check and idea from https://arxiv.org/pdf/1902.05509.pdf\n# train GeM with p=3 with small images (256px), \n# and test on big (1024px), but with increased p=5\n\nclass GeMNet(nn.Module):\n    def __init__(self, new_model):\n        super().__init__()\n        self.cnn =  new_model.features\n        self.head = nn.Sequential(nn.ReLU(),\n                                  GeM(5.0),\n                                  Flatten(),\n                                  L2Norm())\n    def forward(self, x):\n        x = self.cnn(x)\n        out = self.head(x)\n        return out\n    \nInferenceNet =  GeMNet(models.densenet121(pretrained=True))\nInferenceNet.cnn.load_state_dict(learn.model.cnn.state_dict())\nvalidate_on_dataset( InferenceNet, 'oxford5k')\n\n#Seems to work!\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom third_party.testdataset import configdataset\nfrom third_party.evaluate import compute_map_and_print\n\nfrom fastai import *\nfrom fastai.vision import *\nimport pandas as pd\nfrom arch import RingGeMNet, GeMNet\nfrom losses import RingLoss\nfrom utils import extract_vectors_batched","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#I suppose, that competition dataset is downloaded to COMP_DATA_DIR\n#and the list of images are in index_image_list_fullpath.txt\nCOMP_DATA_DIR = '/mnt/fry2/users/datasets/landmarkscvprw18/retrieval'\ndf = pd.read_csv(os.path.join(COMP_DATA_DIR, 'index_image_list_fullpath.txt'),\n                  usecols=[0], names=['Image'])\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass ImageListAbsPath(ImageList):\n    def open(self, fn:PathOrStr)->Image:\n        return open_image(fn.replace('./',''))\ntfms = get_transforms(do_flip=False)\ntfms = (tfms[1],tfms[1]) \nDO_FULL_SIZE = False \n# Extracting features from full-size images would take 15 hours. You probably want to do this \n#for real submission, not for sample. So lets do it on 256x256 images\nif DO_FULL_SIZE:\n    BS=1\n    NUM_WORKERS=8\n    data = (ImageListAbsPath.from_df(df,path='', cols=['Image'])\n            .split_none()\n            .label_const()\n            .transform(tfms, resize_method=ResizeMethod.NO)\n            .databunch(bs=BS, num_workers=NUM_WORKERS)\n            .normalize(imagenet_stats)\n           ) \n    data.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(data.train_ds)\n    data.train_dl.dl.batch_sampler.drop_last = False\nif not DO_FULL_SIZE:\n    BS=64\n    NUM_WORKERS=8\n    data = (ImageListAbsPath.from_df(df,path='', cols=['Image'])\n            .split_none()\n            .label_const()\n            .transform(tfms, resize_method=ResizeMethod.SQUISH, size=192)\n            .databunch(bs=BS, num_workers=NUM_WORKERS)\n            .normalize(imagenet_stats)\n           ) \n    data.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(data.train_ds)\n    data.train_dl.dl.batch_sampler.drop_last = False\n    #index_features = extract_vectors_batched(data,InferenceNet, BS)\n#torch.save(index_features, 'densenet121_pretrained_256px_index_feats.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now we will load training network and transfer its weights to the inference net\nTRAIN_CLASSES=780 #that is hardcoded from number of classes in training dataset\nlearn = Learner(data, RingGeMNet(models.densenet121(pretrained=True), TRAIN_CLASSES),\n                   metrics=[accuracy],\n                   loss_func=nn.CrossEntropyLoss())\n\nInferenceNet =  GeMNet(models.densenet121())\nInferenceNet.cnn.load_state_dict(learn.model.cnn.state_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This will take 30 min on Titan X\nif not DO_FULL_SIZE:\n    qdf = pd.read_csv(os.path.join(COMP_DATA_DIR, 'test_image_list_fullpath.txt'),\n                  usecols=[0], names=['Image'])\n    qdf.head()\n    BS=64\n    NUM_WORKERS=6\n    qdata = (ImageListAbsPath.from_df(qdf,path='', cols=['Image'])\n            .split_none()\n            .label_const()\n            .transform(tfms, resize_method=ResizeMethod.SQUISH, size=256)\n            .databunch(bs=BS, num_workers=NUM_WORKERS)\n            .normalize(imagenet_stats)\n           ) \n    qdata.train_dl.dl.batch_sampler.sampler = torch.utils.data.SequentialSampler(qdata.train_ds)\n    qdata.train_dl.dl.batch_sampler.drop_last = False\n    #query_features = extract_vectors_batched(qdata,InferenceNet, BS)\n#torch.save(query_features, 'densenet121_pretrained_256px_query_feats.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"query_features = torch.load('densenet121_pretrained_256px_query_feats.pth').numpy()\nindex_features = np.zeros((len(data.train_ds),1024)).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import faiss\n\nquery_fnames = [x.split('/')[-1].replace('.jpg','') for x in qdf.Image.tolist()]\nindex_fnames = [x.split('/')[-1].replace('.jpg','') for x in df.Image.tolist()]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastprogress import master_bar, progress_bar\n\ndef get_idxs_and_dists(query_features, index_features, BS = 32):\n    flat_config = faiss.GpuIndexFlatConfig()\n    flat_config.device = 0\n    res = faiss.StandardGpuResources()\n    co = faiss.GpuClonerOptions()\n    FEAT_DIM = index_features.shape[1]\n    cpu_index = faiss.IndexFlatL2(FEAT_DIM)\n    cpu_index.add(index_features)\n    index = faiss.index_cpu_to_gpu(res, 0, cpu_index, co)\n    out_dists = np.zeros((len(query_features), 100), dtype=np.float32)\n    out_idxs = np.zeros((len(query_features), 100), dtype=np.int32)\n    NUM_QUERY = len (query_features)\n    for ind in progress_bar(range(0, len(query_features), BS)):\n        fin = ind+BS\n        if fin > NUM_QUERY:\n            fin = NUM_QUERY\n        q_descs = query_features[ind:fin]\n        D, I = index.search(q_descs, 100)\n        out_dists[ind:fin] = D\n        out_idxs[ind:fin] = I\n    return out_idxs, out_dists\n\ndef create_submission_from_features(query_features,\n                                    index_features,\n                                    fname,\n                                    query_fnames,\n                                    index_fnames):\n    out_idxs, out_dists = get_idxs_and_dists(query_features, index_features, BS = 32)\n    print (f'Writing {fname}')\n    with open(fname, 'w') as f:\n        f.write('id,images\\n')\n        for i in progress_bar(range(len(query_fnames))):\n            ids = [index_fnames[x] for x in out_idxs[i]]\n            f.write(query_fnames[i] + ',' + ' '.join(ids)+'\\n')\n    print('Done!')\n    return\n#create_submission_from_features(query_features, index_features, 'test_submission.csv',query_fnames, index_fnames)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}