{"cells":[{"metadata":{"_uuid":"50b116a79a91d4d314e4a7d074cc2893d665b551"},"cell_type":"markdown","source":"This notebook is using latest Fastai version to implement ArcfFace loss using Custom Head for whale species identification using Humpback . ArcFace loss has been used by many top rankers following paper\nhttps://arxiv.org/pdf/1801.07698.pdf\nIm able to implement only a part of it following the solutions of Top3 rankers but not able to implement Feature centre part for which I need help from the suggestions . Please give your input how i can implement the Feature centralization part which will major boost to this solution. "},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"id":"4OmBUFoZFVsw","colab_type":"code","outputId":"454c9b1c-d5eb-43c7-a227-61349db1f39b","colab":{"base_uri":"https://localhost:8080/","height":136},"trusted":true,"_uuid":"3ef295090483a7d108293f96bfe6f11428108b84"},"cell_type":"code","source":"%%time\n! nvidia-smi\n#! rm -rf resnet_324.pth\n#!echo c.ExecutePreprocessor.timeout","execution_count":1,"outputs":[{"output_type":"stream","text":"Sun Mar 31 14:54:51 2019 \n+-----------------------------------------------------------------------------+\n| NVIDIA-SMI 410.104 Driver Version: 410.104 CUDA Version: 10.0 |\n|-------------------------------+----------------------+----------------------+\n| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Tesla P100-PCIE... On | 00000000:00:04.0 Off | 0 |\n| N/A 45C P0 28W / 250W | 0MiB / 16280MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n \n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Type Process name Usage |\n|=============================================================================|\n| No running processes found |\n+-----------------------------------------------------------------------------+\nCPU times: user 4 ms, sys: 12 ms, total: 16 ms\nWall time: 533 ms\n","name":"stdout"}]},{"metadata":{"id":"neZYPlb-GFqK","colab_type":"code","colab":{},"trusted":true,"_uuid":"7acaa39532858ad2b384f98fe5ec43b182b385d9"},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nfrom fastai.callbacks import *\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6216aec8b017600f8b03e9f53de4b1fd0dfdef1e"},"cell_type":"code","source":"! ls -l ../input/","execution_count":3,"outputs":[{"output_type":"stream","text":"total 20\r\ndrwxr-xr-x 4 root root 4096 Mar 31 10:10 arcface-humpback-customhead-fastai-score919\r\ndrwxr-xr-x 2 root root 4096 Mar 14 05:57 cropped-img\r\ndrwxr-xr-x 2 root root 4096 Mar 14 06:35 dense284\r\ndrwxr-xr-x 2 root root 4096 Mar 20 17:50 dense324\r\ndrwxr-xr-x 4 root root 4096 Nov 30 20:48 humpback-whale-identification\r\n","name":"stdout"}]},{"metadata":{"id":"m_IqjU3gGGv7","colab_type":"code","outputId":"70000517-f3c1-4b3f-f059-f81e867896d5","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"6723b28a8b8495f092c5726c7e9d3220f3341318"},"cell_type":"code","source":"#path = Path('./data/')\npath_t=Path('../input/humpback-whale-identification/')\npath_b=Path('../input/')\npath1='.'\ndf = pd.read_csv(path_t/'train.csv'); \n\n#!pip install fastai=='1.0.44'\n\nimport fastai\nfastai.__version__","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"'1.0.50.post1'"},"metadata":{}}]},{"metadata":{"id":"VN_T8304GBh7","colab_type":"code","colab":{},"trusted":true,"_uuid":"ee319d3e119a498827a2179faa713002b4cbe8c7"},"cell_type":"code","source":"#df = pd.read_csv(LABELS).set_index('Image')\nexclude_list=['0b1e39ff.jpg',\n'0c11fa0c.jpg',\n'1b089ea6.jpg',\n'2a2ecd4b.jpg',\n'2c824757.jpg',\n'3e550c8a.jpg',\n'56893b19.jpg',\n'613539b4.jpg',\n'6530809b.jpg',\n'6b753246.jpg',\n'6b9f5632.jpg',\n'75c94986.jpg',\n'7f048f21.jpg',\n'7f7702dc.jpg',\n'806cf583.jpg',\n'95226283.jpg',\n'a3e9070d.jpg',\n'ade8176b.jpg',\n'b1cfda8a.jpg',\n'b24c8170.jpg',\n'b7ea8be4.jpg',\n'b9315c19.jpg',\n'b985ae1e.jpg',\n'baf56258.jpg',\n'c4ad67d8.jpg',\n'c5da34e7.jpg',\n'c5e3df74.jpg',\n'ced4a25c.jpg',\n'd14f0126.jpg',\n'e0b00a14.jpg',\n'e6ce415f.jpg',\n'e9bd2e9c.jpg',\n'f4063698.jpg',\n'f9ba7040.jpg']\nnew_whale_df = df[df.Id == \"new_whale\"] # only new_whale dataset\ntrain_df = df[~(df.Id == \"new_whale\")] # no new_whale dataset, used for training\nunique_labels = np.unique(train_df.Id.values)\ntrn_imgs=train_df.copy().reset_index(drop=True)\ncnter = Counter(trn_imgs.Id.values)\ntrn_imgs['cnt']=trn_imgs['Id'].apply(lambda x: cnter[x])\n#trn_imgs['target'] = 1\ntrn_imgs['target'] = 0 # 0 for same images\ntrn_imgs1 = trn_imgs.copy()\n#trn_imgs1['target'] = 0\ntrn_imgs1['target'] = 1 # 1 for dissimilar images\n#trn_imgs = trn_imgs.append(trn_imgs1)\ntarget_col = 3\ntrn_imgs.head(1)\ntrn_imgs=trn_imgs[~trn_imgs.Image.isin(exclude_list)]","execution_count":5,"outputs":[]},{"metadata":{"id":"Rgaekz_HRV8e","colab_type":"code","colab":{},"trusted":true,"_uuid":"9d7546402c68cf503e906fc99affddaef028a39d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"id":"CwZqSQprGOaS","colab_type":"code","colab":{},"trusted":true,"_uuid":"02883061ec423029c924db162c699dae54e955a8"},"cell_type":"code","source":"def read_img(fname,box_df,img,sz=224):\n \n \n x0,y0,x1,y1 = tuple(box_df.loc[fname,['x0','y0','x1','y1']].tolist())\n #print(img.shape)\n l1,l0 = img.shape[1],img.shape[2]\n b0,b1 = x1-x0, y1-y0\n #padding\n x0n,x1n = max(int(x0 - b0*0.05),0), min(int(x1 + b0*0.05),l0-1)\n y0n,y1n = max(int(y0 - b1*0.05),0), min(int(y1 + b1*0.05),l1-1)\n img=to_np(img)\n \n #print(img.shape,x0,y0,x1,y1)\n if not (x0 >= x1 or y0 >= y1):\n None\n \n #img = img[:,y0n:y1n, x0n:x1n]\n #print(img.shape,'img')\n #if self.tfms_g != None: img = self.tfms_g.augment_image(img)\n img = img[:,y0n:y1n, x0n:x1n]\n #print(img.T.shape)\n #img = cv2.resize(img.T, (sz,sz))\n return Image(pil2tensor(img.astype(np.float)/255, np.float32).float())","execution_count":6,"outputs":[]},{"metadata":{"id":"UEO9IcO64B3L","colab_type":"code","colab":{},"trusted":true,"_uuid":"d745049d00dc92a33f3055c67641eb13f0909167"},"cell_type":"code","source":"def crop_loose_bbox(img,area, val=0.2):\n #img=np.asarray(img)\n #print(img.shape)\n l1, l0,_ = img.shape\n #print(img.shape)\n b0 = area[2] - area[0]\n b1 = area[3] - area[1]\n x0n,x1n = max(int(area[0] - b0*0.05),0), min(int(area[2] + b0*0.05),l0-1)\n y0n,y1n = max(int(area[1] - b1*0.05),0), min(int(area[3] + b1*0.05),l1-1)\n \n #print(img[y0n:y1n,x0n:x1n,:].shape )\n #img = cv2.resize(img[y0n:y1n,x0n:x1n,:], (224,224))\n \"\"\"\n area2 = (max(0, int(area[0] - 0.5*val*w)),\n max(0, int(area[1] - 0.5*val*h)),\n min(img_w, int(area[2] + 0.5*val*w)),\n min(img_h, int(area[3] + 0.5*val*h)))\n \"\"\"\n return img[y0n:y1n,x0n:x1n,:] #img.crop(area2)\n","execution_count":7,"outputs":[]},{"metadata":{"id":"_uIqL-r2anv2","colab_type":"code","colab":{},"trusted":true,"_uuid":"d1e9124f553c9cb5ffd0c9d049fda61215c72351"},"cell_type":"code","source":" \"\"\"\n def __call__(self, fname):\n fname = os.path.basename(fname)\n #x0,y0,x1,y1 = tuple(self.boxes.loc[fname,['x0','y0','x1','y1']].tolist())\n img = open_image(os.path.join(self.path,fname))\n l1,l0,_ = img.shape\n b0,b1 = x1-x0, y1-y0\n #padding\n x0n,x1n = max(int(x0 - b0*0.05),0), min(int(x1 + b0*0.05),l0-1)\n y0n,y1n = max(int(y0 - b1*0.05),0), min(int(y1 + b1*0.05),l1-1)\n \n if self.tfms_g != None: img = self.tfms_g.augment_image(img)\n img = cv2.resize(img[y0n:y1n,x0n:x1n,:], (sz,sz))\n if self.tfms_px != None: img = self.tfms_px.augment_image(img)\n return img.astype(np.float)/255\n \"\"\"","execution_count":8,"outputs":[{"output_type":"execute_result","execution_count":8,"data":{"text/plain":"\"\\n def __call__(self, fname):\\n fname = os.path.basename(fname)\\n #x0,y0,x1,y1 = tuple(self.boxes.loc[fname,['x0','y0','x1','y1']].tolist())\\n img = open_image(os.path.join(self.path,fname))\\n l1,l0,_ = img.shape\\n b0,b1 = x1-x0, y1-y0\\n #padding\\n x0n,x1n = max(int(x0 - b0*0.05),0), min(int(x1 + b0*0.05),l0-1)\\n y0n,y1n = max(int(y0 - b1*0.05),0), min(int(y1 + b1*0.05),l1-1)\\n \\n if self.tfms_g != None: img = self.tfms_g.augment_image(img)\\n img = cv2.resize(img[y0n:y1n,x0n:x1n,:], (sz,sz))\\n if self.tfms_px != None: img = self.tfms_px.augment_image(img)\\n return img.astype(np.float)/255\\n \""},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"890d7a1f4869bb4082157fd97ba360f6f2d90169"},"cell_type":"code","source":"#bbox_df = pd.read_csv(path_b/'cropped-img'/'bounding_boxes.csv').set_index('Image')","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"51d0b3344c50a70140e9097e033afeec556aee1f"},"cell_type":"markdown","source":"Custom open program to read the bounding boxes and crop the image to remove the unwanted background. Bounding boxes provided in public kernels are exteremely important in getting a high score. "},{"metadata":{"id":"KxP_V_au30HT","colab_type":"code","colab":{},"trusted":true,"_uuid":"274a0baff89e71e2f59744b1013e63b979e126ef"},"cell_type":"code","source":"def open_4_channel2(fname):\n fname = str(fname)\n bbox_df = pd.read_csv(path_b/'cropped-img'/'bounding_boxes.csv').set_index('Image')\n #print(fname)\n # strip extension before adding color\n x0,y0,x1,y1=bbox_df.loc[fname[fname.rfind('/')+1:]]\n area=(x0,y0,x1,y1) \n #print(fname)\n img = cv2.imread(fname)\n #PIL.Image.open(fname)\n #print(img.size)\n \n img=crop_loose_bbox(img,area)\n \n #img=np.asarray(img)\n #print(img.shape)\n #print(img.shape)\n \n #import time\n #a=time.time()\n \n \n return Image(pil2tensor(img/255, np.float32).float())","execution_count":10,"outputs":[]},{"metadata":{"_uuid":"ba845f8f79f83ec86de56e3f24fcb26af6f1da22"},"cell_type":"markdown","source":"Doing the oversampling of those species which are having only a single image and 2 images"},{"metadata":{"id":"snxus_ZzOr3o","colab_type":"code","colab":{},"trusted":true,"_uuid":"7b6e0be20778da7bfd65ce8393f59a987368ab3d"},"cell_type":"code","source":"trn_imgs=trn_imgs.append(trn_imgs.loc[trn_imgs.cnt==2],ignore_index=True) \ntrn_imgs=trn_imgs.append(trn_imgs.loc[trn_imgs.cnt==1],ignore_index=True) ","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"21bb87ad3fdbe7a607705d120d1e09a3d107f07a"},"cell_type":"markdown","source":"BUilding a balanced validation set to ensure we are having all type of species for validation based on the count of the image for species"},{"metadata":{"id":"yKZuWa_f-G8C","colab_type":"code","outputId":"e449c8be-f86a-43b3-dd05-e79fd4125158","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"c0cc039d04e4b019ef710faab3646d7ce1e213b3"},"cell_type":"code","source":"\nval_idx=[]\nimport random\nfor i in trn_imgs[trn_imgs.cnt>5].Id.unique():\n tmp=list(trn_imgs.loc[trn_imgs.Id==i].index.values)\n #print(tmp)\n val_idx=val_idx+(random.sample(tmp,1))\nlen(val_idx)\n#since images less than 5 are less in number we dont select much from them \nfor i in trn_imgs[(trn_imgs.cnt<5) &( trn_imgs.cnt>2)].Id.unique():\n \n tmp=list(trn_imgs.loc[trn_imgs.Id==i].index.values)\n #print(type(tmp))\n \n if len(val_idx) < 1300 :\n \n val_idx=val_idx+(random.sample(tmp,1))\nlen(val_idx)\n","execution_count":12,"outputs":[{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"1300"},"metadata":{}}]},{"metadata":{"id":"rLpTmP7tNY8P","colab_type":"code","colab":{},"trusted":true,"_uuid":"bb67ed6a3134c65518089b1550a275ff4da3dcb7"},"cell_type":"code","source":"#val_idx[0:5]\n#train_idx=\n#trn_imgs[trn_imgs.Id=='w_f48451c']\n","execution_count":13,"outputs":[]},{"metadata":{"id":"Xmx4bu_84d-7","colab_type":"code","outputId":"d744b65d-4117-4e99-f782-469133a9e9f7","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"6d54c0dc90c450cab8c53da16bfbe02f36d3f615"},"cell_type":"code","source":"#bbox_df = pd.read_csv(path/'bounding_boxes.csv').set_index('Image')\n#x0,y0,x1,y1=bbox_df.loc['72c3ce75c.jpg']\n#crop_loo\n\n#open_4_channel(path/'train'/'0001f9222.jpg')\n #crop_loose_bbox(img,area, val=0.2)\nlen(trn_imgs.Id.unique())\n\n#trn_imgs[trn_imgs.cnt<3].Id.unique().shape","execution_count":14,"outputs":[{"output_type":"execute_result","execution_count":14,"data":{"text/plain":"5004"},"metadata":{}}]},{"metadata":{"id":"tpkfDOeKQBz5","colab_type":"code","colab":{},"trusted":true,"_uuid":"74e2396ed82d8f57b436b9e8ba6dcb7f7876f899"},"cell_type":"code","source":"val_idx=list(trn_imgs.iloc[val_idx].index.values)\ntrn_idx=set(list(trn_imgs.index.values))-set(val_idx) # generating only trn idx to run\ndf_i=trn_imgs.iloc[list(trn_idx)].reset_index(drop=True) # this will be used latter on to run CV loop\n#fn2label = {row[1].Image: row[1].Id for row in df.iterrows()}\n#path2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)","execution_count":15,"outputs":[]},{"metadata":{"id":"zp6Khn5NHiCP","colab_type":"code","colab":{},"trusted":true,"_uuid":"5d6f044bb113c6748c415e8c5fbcaeb080c056f1"},"cell_type":"code","source":"#df_i.index.size 0000e88ab.jpg w_f48451c\n#df_i.Id.nunique()\n#trn_imgs.to_csv() \n#trn_imgs[trn_imgs.Image=='0001f9222.jpg']#w_c3d896a\t\n#df_i.head(2)","execution_count":16,"outputs":[]},{"metadata":{"_uuid":"8b24e14ab2b7b7d676e41b9975abcce7f097d0c9"},"cell_type":"markdown","source":"Generate the dataset using FAI datablock "},{"metadata":{"id":"RwKKpmX76_Gz","colab_type":"code","colab":{},"trusted":true,"_uuid":"63adfd8103d93104bb77e9f1376b09e03006e142"},"cell_type":"code","source":"src1= (ImageList.from_df(trn_imgs[['Image','Id']],path_t, folder='train') #ImageList\n .split_by_idx(val_idx)\n .label_from_df( cols=1))","execution_count":17,"outputs":[]},{"metadata":{"id":"g_iF5iotmhrM","colab_type":"code","colab":{},"trusted":true,"_uuid":"5a8769cbcdb31d10f55198bb183dbb5e84c3adf6"},"cell_type":"code","source":"#trn_imgs.head(2)\n","execution_count":18,"outputs":[]},{"metadata":{"id":"G9td57J-4q2_","colab_type":"code","outputId":"1c7e85f9-572e-4d51-fc43-81031aed5b89","colab":{"base_uri":"https://localhost:8080/","height":54},"trusted":true,"_uuid":"61d7060169746eb1315a5b0ca55f95dff6d58269"},"cell_type":"code","source":"\"\"\"\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.preprocessing import OneHotEncoder\nmlb = OneHotEncoder([i for i in range(5004)],sparse=False)\n#MultiLabelBinarizer([i for i in range(5004)],sparse_output=False)\n#y=mlb.fit_transform(np.array(list(1) ).reshape(-1,1))\n#trn_imgs['hot']=trn_imgs.Image.apply(lambda i : y[trn_imgs[trn_imgs.Image==i].index.values])\n#y[0]\n#trn_imgs.head(1)\n\n#np.array([1,2])\na=[one_hot(i,5004 )for i in range(5)]\n\nnp.array(a).reshape(5,-1).shape\n\"\"\"\n","execution_count":19,"outputs":[{"output_type":"execute_result","execution_count":19,"data":{"text/plain":"\"\\nfrom sklearn.preprocessing import MultiLabelBinarizer\\nfrom sklearn.preprocessing import OneHotEncoder\\nmlb = OneHotEncoder([i for i in range(5004)],sparse=False)\\n#MultiLabelBinarizer([i for i in range(5004)],sparse_output=False)\\n#y=mlb.fit_transform(np.array(list(1) ).reshape(-1,1))\\n#trn_imgs['hot']=trn_imgs.Image.apply(lambda i : y[trn_imgs[trn_imgs.Image==i].index.values])\\n#y[0]\\n#trn_imgs.head(1)\\n\\n#np.array([1,2])\\na=[one_hot(i,5004 )for i in range(5)]\\n\\nnp.array(a).reshape(5,-1).shape\\n\""},"metadata":{}}]},{"metadata":{"id":"mv2MkdIE6ypc","colab_type":"code","outputId":"05ac7919-52fb-49ad-fabf-fa29d3a3d60d","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":true,"_uuid":"fe8a5b4fada7ecbadf8a356ff95e465c562d3b3c"},"cell_type":"code","source":"test_ids = list(sorted({fname for fname in os.listdir(path_t/'test')}))\n\n\n#protein_stats = ([0.16258, 0.13877, 0.10067, 0.16358], [0.21966, 0.18559, 0.25573,0.22066])\ntest_fnames = [path_t/'test'/test_id for test_id in test_ids]\n\ntest_fnames[:3]","execution_count":20,"outputs":[{"output_type":"execute_result","execution_count":20,"data":{"text/plain":"[PosixPath('../input/humpback-whale-identification/test/00028a005.jpg'),\n PosixPath('../input/humpback-whale-identification/test/000dcf7d8.jpg'),\n PosixPath('../input/humpback-whale-identification/test/000e7c7df.jpg')]"},"metadata":{}}]},{"metadata":{"id":"JF-KqqjalT80","colab_type":"code","colab":{},"trusted":true,"_uuid":"eeb467be95cbcc877fdbf0bb6a2b07d72c78f4af"},"cell_type":"code","source":"#np.where(list(trn_imgs.hot.values)[1]==[1])[1]\n\n","execution_count":21,"outputs":[]},{"metadata":{"id":"bJdqCJACShzm","colab_type":"code","colab":{},"trusted":true,"_uuid":"e1e442c0afd8f701282f205cc99359993b607e56"},"cell_type":"code","source":"import cv2\nsrc1.train.x.create_func = open_4_channel2\nsrc1.train.x.open = open_4_channel2\n\nsrc1.valid.x.create_func = open_4_channel2\nsrc1.valid.x.open = open_4_channel2\nsrc1.add_test(test_fnames);\nsrc1.test.x.create_func = open_4_channel2\nsrc1.test.x.open = open_4_channel2\n# combine dataset/transform/dataloader into one dataobject called databunch in fastai\ntrn_tfms,_ = get_transforms(do_flip=False, flip_vert=True, max_rotate=5., max_zoom=1.08,\n max_lighting=0.15, max_warp=0. )\n\ndata1 = (src1.transform((trn_tfms,trn_tfms), size=224,resize_method=ResizeMethod.SQUISH)\n .databunch(bs=64,num_workers=0).normalize(imagenet_stats))\n\ndata2 = (src1.transform((trn_tfms,trn_tfms), size=484,resize_method=ResizeMethod.SQUISH)\n .databunch(bs=36,num_workers=0).normalize(imagenet_stats))","execution_count":22,"outputs":[]},{"metadata":{"id":"lGWk4WCvRzvF","colab_type":"code","colab":{},"trusted":true,"_uuid":"03eafc2af8c6df1c90ecbd345816017d572d7e40"},"cell_type":"code","source":"\n#a=[one_hot(i.unsqueeze(-1),5004 ) for i in tensor(data1.train_ds.y.items[0:5])]\n#listify(x)\n#np.where(a[0]==[1])\n#tensor(data1.train_ds.y.items[0:5])\n#type(a)\n#torch.from_numpy(np.array(a)).size()\n\n#data1.show_batch(2)\n#import pylot as plt\n#i=PIL.Image('data/train/3ece2140f.jpg')\n#print(i.shape)\n#plt.imshow(i)","execution_count":23,"outputs":[]},{"metadata":{"id":"xQvP8Tbif-ox","colab_type":"code","colab":{},"trusted":true,"_uuid":"7d939b7cb98d4cc1ec6ac417a507fced78d014e8"},"cell_type":"code","source":"#data1.c\n#data1.show_batch(2)\n#!cp *.csv ./data/","execution_count":24,"outputs":[]},{"metadata":{"id":"B-v_G_29VEXN","colab_type":"code","colab":{},"trusted":true,"_uuid":"2a07b520fe2d971023a9eae32ddf00408dd8cfa5"},"cell_type":"code","source":"from fastai.metrics import accuracy","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"f8d4f4c9a87ebd7153011bbe52e5ac5f2a899aa5"},"cell_type":"markdown","source":"Build dense 121 network . We can also use similarly Resnet50"},{"metadata":{"id":"EOdXa2geSu_u","colab_type":"code","colab":{},"trusted":true,"_uuid":"48294653b57fb593b6d0b3da1d56d28c0f365087"},"cell_type":"code","source":"from torchvision import models as m\ndef dense(pre):\n \n #model=nn.Sequential(body, head)\n model = m.densenet121(pretrained=pre)\n \n model.classifier = (nn.Linear(1024, 5004))\n\n \n return model\ndef _densenet_split(m): return (m[0][0][6],m[1]) ","execution_count":26,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dense(True)","execution_count":27,"outputs":[]},{"metadata":{"id":"ac8BzGZeRGKB","colab_type":"code","colab":{},"trusted":true,"_uuid":"85fdf13b1781057b6d522c9455a81c61bff6bb48"},"cell_type":"code","source":"def acc (input:Tensor, targs:Tensor)->Rank0Tensor:\n \n \"Compute accuracy with `targs` when `input` is bs * n_classes.\"\n n = targs.shape[0]\n input = input.argmax(dim=-1).view(n,-1)\n targs = targs.view(n,-1)\n return (input==targs).float().mean()","execution_count":28,"outputs":[]},{"metadata":{"id":"SVJlKQweWZc-","colab_type":"code","outputId":"c25417b8-29a6-41ca-a2f2-05282cf1db97","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":true,"_uuid":"b3bc1025222c6f746270264046bccd0261acb065"},"cell_type":"code","source":"i=torch.rand(3,2)\n#j=torch.ones(48,1)\nprint(i)\n#acc(i,j)\n#F.softmax(i,1) \n#torch.empty(5004, 1024)\n#nn.init.kaiming_normal_(torch.FloatTensor(5004, 1024))\n#torch.randint(4, (3,), dtype=torch.int64)","execution_count":29,"outputs":[{"output_type":"stream","text":"tensor([[0.1489, 0.2840],\n [0.3263, 0.6973],\n [0.5395, 0.3902]])\n","name":"stdout"}]},{"metadata":{"_uuid":"935a6f4bf390f68d833a95b8742d5173a29da179"},"cell_type":"markdown","source":"Custom Head building . THis will be responsibly for generating the normalized features/weights which are needed as per ArcFace paper. "},{"metadata":{"id":"t1SFlBth0cxy","colab_type":"code","colab":{},"trusted":true,"_uuid":"26b37d600a4503c980511dde91a8453deec4a6c2"},"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n r\"\"\"Implement of large margin arc distance: :\n Args:\n in_features: size of each input sample\n out_features: size of each output sample\n s: norm of input feature\n m: margin\n cos(theta + m)\n \"\"\"\n def __init__(self, in_features, out_features=5004):\n \n \n super(ArcMarginProduct, self).__init__()\n self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n self.reset_parameters()\n #nn.init.kaiming_uniform_(self.weight)\n\n def reset_parameters(self):\n stdv = 1. / math.sqrt(self.weight.size(1)) # eq to input . This is more or less like Kaiming normal .\n #we use this to ensure values remain between 0 and 1 . Since std deviation reduces to almost half every layer\n # we can try some trick multiplying it by 2 \n self.weight.data.uniform_(-stdv, stdv)\n \n def forward(self, features):\n #x=self.head(features)\n #print(self.weight.shape)\n #self.fc1.weight=nn.Parameter(F.normalize(self.fc1.weight)).cuda()\n cosine = F.linear(F.normalize(features), F.normalize(self.weight.cuda()))\n #cosine = cosine.clamp(-1, 1)\n #self.fc1(F.normalize(x))\n #F.linear(F.normalize(x), F.normalize(self.weight.cuda()))\n return cosine \n\nclass Customhead(nn.Module):\n r\"\"\"Implement of large margin arc distance: :\n Args:\n in_features: size of each input sample\n out_features: size of each output sample\n s: norm of input feature\n m: margin\n cos(theta + m)\n \"\"\"\n def __init__(self, in_features, out_features=5004):\n super(Customhead, self).__init__()\n #self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n #self.register_parameter('normweights',self.weight)\n # nn.init.xavier_uniform_(self.weight)\n #body = create_body(m.densenet121, True, -1)\n body = create_body(m.resnet50, True, -2)\n#body = create_head(ArcMarginProduct, pretrained, 0)\n nf = num_features_model(nn.Sequential(*body.children())) * 2\n #head = \n self.head=create_head(nf, 1024,[2048], ps=0.5, bn_final=False) # 1024 no of classes\n self.arc_margin=ArcMarginProduct(in_features,out_features)\n #self.fc1=nn.Linear(1024,5004,bias=False)\n #self.custom=nn.Sequential(self.head,self.fc1)\n #self.reset_parameters()\n\n # def reset_parameters(self):\n #stdv = 1. / math.sqrt(self.weight.size(1))\n #self.weight.data.uniform_(-stdv, stdv)\n \n\n def forward(self, features):\n x=self.head(features)\n #w=self.fc1.weight\n #self.fc1.weight=nn.Parameter(F.normalize(self.fc1.weight)).cuda()\n cosine = self.arc_margin(x)\n #F.linear(F.normalize(x), F.normalize(w))\n #self.arc_margin(x)\n #F.linear(F.normalize(x), F.normalize(self.weight.cuda()))\n cosine = cosine.clamp(-1, 1)\n #self.fc1(F.normalize(x))\n #F.linear(F.normalize(x), F.normalize(self.weight.cuda()))\n return cosine","execution_count":30,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e9b5f17612c378eb1f396caadefee60b1df8367"},"cell_type":"code","source":"#for i in Customhead(1024,5004).parameters():\n #print( i.size())\n#Customhead(1024,5004)\n\nclass CustomheadRes(nn.Module):\n r\"\"\"Implement of large margin arc distance: :\n Args:\n in_features: size of each input sample\n out_features: size of each output sample\n s: norm of input feature\n m: margin\n cos(theta + m)\n \"\"\"\n def __init__(self, in_features, out_features=5004):\n super(CustomheadRes, self).__init__()\n #self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n #self.register_parameter('normweights',self.weight)\n # nn.init.xavier_uniform_(self.weight)\n body = create_body(m.densenet121, True, -1)\n #body = create_body(m.resnet50, True, -2)\n#body = create_head(ArcMarginProduct, pretrained, 0)\n nf = num_features_model(nn.Sequential(*body.children())) * 2\n #head = \n self.head=create_head(nf, 1024, ps=0.5, bn_final=False) # 1024 no of classes\n self.arc_margin=ArcMarginProduct(in_features,out_features)\n #self.fc1=nn.Linear(1024,5004,bias=False)\n #self.custom=nn.Sequential(self.head,self.fc1)\n #self.reset_parameters()\n\n # def reset_parameters(self):\n #stdv = 1. / math.sqrt(self.weight.size(1))\n #self.weight.data.uniform_(-stdv, stdv)\n \n\n def forward(self, features):\n x=self.head(features)\n #w=self.fc1.weight\n #self.fc1.weight=nn.Parameter(F.normalize(self.fc1.weight)).cuda()\n cosine = self.arc_margin(x)\n #F.linear(F.normalize(x), F.normalize(w))\n #self.arc_margin(x)\n #F.linear(F.normalize(x), F.normalize(self.weight.cuda()))\n cosine = cosine.clamp(-1, 1)\n #self.fc1(F.normalize(x))\n #F.linear(F.normalize(x), F.normalize(self.weight.cuda()))\n return cosine","execution_count":31,"outputs":[]},{"metadata":{"_uuid":"3bbb2e61f2649b9755f810636abc48f6bf25f793"},"cell_type":"markdown","source":"Arface loss getting normalized output from Fc1"},{"metadata":{"id":"O9QmQL-5QME-","colab_type":"code","colab":{},"trusted":true,"_uuid":"fb211e754f4a8323daec6fae534f4ded456c9957"},"cell_type":"code","source":"#data1.show_batch(2)\n#src1.xtra.Id\nclass ArcFaceLoss(nn.modules.Module):\n def __init__(self,s=30.0,m=0.5):\n super(ArcFaceLoss, self).__init__()\n self.classify_loss = nn.CrossEntropyLoss()\n self.s = s\n self.easy_margin = False\n self.cos_m = math.cos(m) \n self.sin_m = math.sin(m) \n self.th = math.cos(math.pi - m)\n self.mm = math.sin(math.pi - m) * m\n\n def forward(self, inputs, labels, epoch=0,reduction=None):\n cosine = inputs\n sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n phi = cosine * self.cos_m - sine * self.sin_m\n if self.easy_margin:\n phi = torch.where(cosine > 0, phi, cosine)\n else:\n phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n one_hot = torch.zeros(cosine.size(), device='cuda')\n one_hot.scatter_(1, labels.view(-1, 1).long(), 1)\n # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n output *= self.s\n loss1 = self.classify_loss(output, labels) # this is as per paper what is missing here is centralized features\n loss2 = self.classify_loss(cosine, labels)\n gamma=1\n loss=(loss1+gamma*loss2)/(1+gamma)\n return loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"id":"MuIAOdLcWAmA","colab_type":"code","colab":{},"trusted":true,"_uuid":"bd7720b0193c2977c177bbcbe23c6c10bba7ae11"},"cell_type":"code","source":"\ndef resnet501(pre):\n \n model = m.resnet50(pretrained=pre)\n #w=model.features[0].weight\n #model.features[0]=nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3, bias=False)\n #model.fc = (nn.Linear(1024, 5004))\n\n #print(w.shape)\n # model.features[0].weight=torch.nn.Parameter(torch.cat((w, w[:,:1,:,:]),dim=1))\n #print(model.features[0].weight.shape)\n return model\ndef _resnet_split(m): return (m[0][6],m[1])","execution_count":33,"outputs":[]},{"metadata":{"id":"LNWZlQeY2TWM","colab_type":"code","colab":{},"trusted":true,"_uuid":"54170c503a5d9304eba3472d2a63da9e48f8371b"},"cell_type":"code","source":"ar=ArcFaceLoss().cuda() # this may not be needed just try it out","execution_count":34,"outputs":[]},{"metadata":{"id":"7_PSZzslMjzd","colab_type":"code","colab":{},"trusted":true,"_uuid":"f3a1bafae2bc43ebd83b4d911e284877893bfbd4"},"cell_type":"code","source":"\n#m1=nn.Sequential(body, head)\n#m1[:-1]\n\n#custom_head","execution_count":35,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training Process**\n1) Resnet 50 \n* 1) Starting with 224 Number of Epochs 10-12 . We should not train for more epoch on a single image size to avoid overfitting .This should get a score 70+\n* 2) Change the image to 284 and run the stratified CV loop with 9 folds\n* 3) Run the complete Training set with number of epoch around 10 - At the end of epoch we should toch the score of 80 or 80 plus . Threshold at ths time .52 to .535\n* 4) Change the image size to 324 ,run the stratified loop \n* 5) Train with compelte Training set with image size 324 for 8 to 9 epochs.By this time score should be hovering around 84 to 85 \n* 6) Change the image size to 384 and repeat the process . As model gets better threshold will have to be increased \n\n2) Dense 121 \n Repeat the same way as above\n \nCurrent score with Image size 424 stands out to be .92942 and ensemble of Resnet 50 and dense121.\n\nI am expecting to hit 93 to 94 as i gradually increase the image size to 512. One more thing I want to try at the end is Averaging the Intraclass predictions using training set predictions to produce matrix of 5004 * 5004 and then use this embedding to find cosine similarity between Test prediction and matrix to generate the classes\nThanks @lafoss for helping me understand this."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"id":"bv7kdn-LTWod","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":105},"outputId":"daefc0f8-2a70-4888-cb26-e99860307e74","trusted":true,"_uuid":"4ca0547cd1095c226047236815b9e31e7129096c"},"cell_type":"code","source":"f1_score = partial(fbeta, thresh=0.4, beta=1)\nacc_02 = partial(accuracy_thresh, thresh=0.2)\n\ncustom_head=Customhead(1024,5004)\ncustom_headres=CustomheadRes(1024,5004)\nar=ArcFaceLoss().cuda()\nfrom fastai.torch_core import *\nfrom fastai.callbacks import *\nfrom fastai.basic_train import *\nfrom torch.autograd import Variable\n\n#callbacks=[partial(GradientClipping, clip=1),partial(SaveModelCallback,monitor='trn_loss',mode='min')\n# ,ReduceLROnPlateauCallback(learn, min_delta=1e-5, patience=3)]\nimport gc\ngc.collect()\nlearn1 = create_cnn(\n data2,\n #dense,\n resnet501,\n #dense,\n #cut=-1,\n cut=-2,\n split_on=_resnet_split,\n #_densenet_split,\n lin_ftrs=[1024],\n custom_head=custom_head,\n \n #lambda m: (m[0][11], m[1]),\n loss_func=ar,\n #torch.nn.MultiLabelSoftMarginLoss(),\n #F.binary_cross_entropy_with_logits,\n #FocalLoss(),\n #F.binary_cross_entropy_with_logits,\n path=path1, \n metrics=[accuracy], callback_fns= partial(GradientClipping, clip=1))\n \n#learn1 = Learner(data1, dense(), loss_func=torch.nn.MultiLabelSoftMarginLoss(),path=path,\n #metrics=[acc_02,f1_scorestd], callback_fns= partial(GradientClipping, clip=1))\nlearn1.callback_fns.append(partial(SaveModelCallback,monitor='val_loss',mode='min'))\nlearn1.callback_fns.append(partial(ReduceLROnPlateauCallback, min_delta=1e-5, patience=3))\n\n# dense net used for building CNN.\nlearn2 = create_cnn(\n data2,\n dense,\n #resnet501,\n #dense,\n cut=-1,\n #cut=-2,\n #split_on=_resnet_split,\n split_on=_densenet_split,\n lin_ftrs=[512],\n custom_head=custom_headres,\n #lambda m: (m[0][11], m[1]),\n loss_func=ar,\n #torch.nn.MultiLabelSoftMarginLoss(),\n #F.binary_cross_entropy_with_logits,\n #FocalLoss(),\n #F.binary_cross_entropy_with_logits,\n path=path1, \n metrics=[accuracy], callback_fns= partial(GradientClipping, clip=1))\n \n#learn1 = Learner(data1, dense(), loss_func=torch.nn.MultiLabelSoftMarginLoss(),path=path,\n #metrics=[acc_02,f1_scorestd], callback_fns= partial(GradientClipping, clip=1))\n#learn2.callback_fns.append(partial(SaveModelCallback,monitor='val_loss',mode='min'))\nlearn2.callback_fns.append(partial(ReduceLROnPlateauCallback, min_delta=1e-5, patience=3))\n#learn2=learn2.to_fp16()","execution_count":null,"outputs":[]},{"metadata":{"id":"wBBdfLUbML9G","colab_type":"code","colab":{},"trusted":true,"_uuid":"64b64b04c5da4f530d6bd88ca39e8d784e3b4d08"},"cell_type":"code","source":"#learn1.model[1]\n#data1.c\n#!cp dens* ./data/models/\n#learn2.model#[6].weight.shape\n#learn2.save('save')\n#learn2.model\n#torch.FloatTensor(2,3)\n! mkdir /kaggle/working/models\n#! ls -l /kaggle/input/\n#!cp /kaggle/input/dense324/*.pth /kaggle/working/models/\n#!cp /kaggle/input/arcface-humpback-customhead-fastai-score919/models/resnet_ar_c384.pth /kaggle/working/models/\n#!cp /kaggle/input/arcface-humpback-customhead-fastai/models/dense_ar_c1284.pth /kaggle/working/models/\n#!cp /kaggle/input/arcface-humpback-customhead-fastai-score919/models/resnet_ar_c484.pth /kaggle/working/models/\n!cp /kaggle/input/arcface-humpback-customhead-fastai-score919/models/dense_ar_c424_1.pth /kaggle/working/models/\n\n#!cp /kaggle/input/arcface-humpback-customhead-fastai-score919/models/resnet_ar_c424_2.pth /kaggle/working/models/\n!cp /kaggle/input/arcface-humpback-customhead-fastai-score919/models/resnet_ar_c424_1.pth /kaggle/working/models/\n#!mv /kaggle/working/models/dense_ar_c324 /kaggle/working/models/dense_ar_c324.pth\n! ls -l /kaggle/working/models","execution_count":37,"outputs":[{"output_type":"stream","text":"mkdir: cannot create directory ‘/kaggle/working/models’: File exists\ntotal 1539080\n-rw-r--r-- 1 root root 164522717 Mar 31 14:55 dense_ar_c424_1.pth\n-rw-r--r-- 1 root root 469973463 Mar 31 14:55 resnet_ar_c424_1.pth\n-rw-r--r-- 1 root root 469973468 Mar 31 14:55 resnet_ar_c424_2.pth\n-rw-r--r-- 1 root root 469973463 Mar 31 14:55 resnet_ar_c484.pth\n","name":"stdout"}]},{"metadata":{"_uuid":"eff5b5c5400da61c7938a42432a408448493b24b"},"cell_type":"markdown","source":"Get LR using best lf find . 1e-2 to 3e-2 is the suitable LR."},{"metadata":{"id":"RK4ZDzm1S7Ft","colab_type":"code","outputId":"0b75c854-90a0-414a-90b6-4ced6fc313b9","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"d1ea27b6e391056e1f66c7728fd7099b76de6701"},"cell_type":"code","source":"import gc\ngc.collect()\n#learn2.lr_find()\n#learn2.recorder.plot()\n#!rm -rf ./data/models/\n#len(data1.train_dl)\n#learn1.loss_func\n#data1.show_batch(2)\n#learn1.save('dense_224')\n#for i in learn2.model[1].parameters():\n #print(i.size())","execution_count":38,"outputs":[{"output_type":"execute_result","execution_count":38,"data":{"text/plain":"740"},"metadata":{}}]},{"metadata":{"id":"WgtZ_0pJiNLA","colab_type":"code","outputId":"067cabe0-8b81-4bb6-af3e-8596c2723ee6","colab":{"base_uri":"https://localhost:8080/","height":361},"trusted":true,"_uuid":"8cd27ffdc6b03db8bf1a56a1c4f445ca29303f87"},"cell_type":"code","source":"#learn2.recorder.plot()\n#push\n#learn2.model[1]","execution_count":39,"outputs":[]},{"metadata":{"id":"k1iIsxkr1QQO","colab_type":"code","outputId":"a3861634-8da7-431f-a1f7-3fa96eec968d","colab":{"base_uri":"https://localhost:8080/","height":361},"trusted":true,"_uuid":"b859aaa4c9a4d5edcb0dac51bdf76557762c710a"},"cell_type":"code","source":"#x,y=next(iter(learn1.data.train_dl))\n#! rm -rf ./models\n#learn1.recorder.plot()\n#for i in trainable_params(learn2.model[1]):\n #print(i.size())\n#push","execution_count":40,"outputs":[]},{"metadata":{"id":"OXHDaZgTWaQv","colab_type":"code","outputId":"099ddfd3-4c9c-4c86-e23f-6307e0fc962c","colab":{"base_uri":"https://localhost:8080/","height":168},"trusted":true,"_uuid":"d76ec074b599cf0666390d8fe48e2791566dfc4e","scrolled":false},"cell_type":"code","source":"#learn2.unfreeze()\n#learn2.load('dense_arc1')\n#learn2.fit_one_cycle(2,3e-2)","execution_count":41,"outputs":[]},{"metadata":{"id":"dQG2GjofJs5g","colab_type":"code","colab":{},"trusted":true,"_uuid":"699c123bd01ecab869d5a49fe153f493d962f9d3","scrolled":true},"cell_type":"code","source":"#!ls -l\n#!cp ./data/models/dense_arc1.pth ./\n\n#learn1.save('dense_arc1')\nfor i in learn1.model[1].parameters():\n print(i.shape)\n#learn2.model[1]\nlearn2.model[1]","execution_count":42,"outputs":[{"output_type":"stream","text":"torch.Size([4096])\ntorch.Size([4096])\ntorch.Size([2048, 4096])\ntorch.Size([2048])\ntorch.Size([2048])\ntorch.Size([2048])\ntorch.Size([1024, 2048])\ntorch.Size([1024])\ntorch.Size([5004, 1024])\n","name":"stdout"},{"output_type":"execute_result","execution_count":42,"data":{"text/plain":"CustomheadRes(\n (head): Sequential(\n (0): AdaptiveConcatPool2d(\n (ap): AdaptiveAvgPool2d(output_size=1)\n (mp): AdaptiveMaxPool2d(output_size=1)\n )\n (1): Flatten()\n (2): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (3): Dropout(p=0.25)\n (4): Linear(in_features=2048, out_features=512, bias=True)\n (5): ReLU(inplace)\n (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (7): Dropout(p=0.5)\n (8): Linear(in_features=512, out_features=1024, bias=True)\n )\n (arc_margin): ArcMarginProduct()\n)"},"metadata":{}}]},{"metadata":{"id":"PuremDLbJ88A","colab_type":"code","outputId":"8ad2a214-0554-4c25-b3d4-d29825e129ce","colab":{"base_uri":"https://localhost:8080/","height":405},"trusted":true,"_uuid":"bea1ae288660fe2f6eb3c45f14fee59a50b14c06"},"cell_type":"code","source":"\"\"\"\nlr=1e-2 # ran stratified 224,284*2,now ffull\nlearn2.unfreeze()\n#learn2.load('save')\n#learn2.load('dense_ar_c324') #0.026030\t0.656774\t0.892308\nlearn2.fit_one_cycle(14,slice(2e-4,lr/2))\nlearn2.save('dense_ar_c324')\n\"\"\"","execution_count":43,"outputs":[{"output_type":"execute_result","execution_count":43,"data":{"text/plain":"\"\\nlr=1e-2 # ran stratified 224,284*2,now ffull\\nlearn2.unfreeze()\\n#learn2.load('save')\\n#learn2.load('dense_ar_c324') #0.026030\\t0.656774\\t0.892308\\nlearn2.fit_one_cycle(14,slice(2e-4,lr/2))\\nlearn2.save('dense_ar_c324')\\n\""},"metadata":{}}]},{"metadata":{"id":"cVBWoEXqkEVe","colab_type":"code","outputId":"62cba07b-35c8-4739-f6a6-e733bf6f7909","colab":{"base_uri":"https://localhost:8080/","height":332},"trusted":true,"_uuid":"f5d03c76d829486a0f9acd29d6b382d189724fa3"},"cell_type":"code","source":"#learn2.fit_one_cycle(8,slice(2e-4,lr/2)) # run for 30 epochs \nlr=2e-2 # ran stratified 224,284*2,now ffull\n#learn2.unfreeze()\n#learn2.load('save')\n#learn2.load('dense_ar_c56') #0.026030\t0.656774\t0.892308\n#learn1.fit_one_cycle(2,slice(2e-4,lr/2))\n#learn1.save('resnet_ar_c224')","execution_count":44,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nlr=2e-2 # ran stratified 224,284*2,now ffulls\n\nlearn1.unfreeze()\n#learn2.load('save')\nlearn1.load('resnet_ar_c424') #0.026030\t0.656774\t0.892308\nlearn1.fit_one_cycle(7,slice(2e-4,lr/2))\nlearn1.save('resnet_ar_c424_1')\n\"\"\"\nlr=3e-2\nlearn1.unfreeze()\n#learn2.load('save')\nlearn1.load('resnet_ar_c424_1') #0.026030\t0.656774\t0.892308\nlearn1.fit_one_cycle(9,slice(2e-5,lr/2))\nlearn1.save('resnet_ar_c424_2')\n\n ","execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"","text/html":"\n
\n \n \n 85.71% [6/7 3:07:23<31:13]\n
\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
epochtrain_lossvalid_lossaccuracytime
04.1266223.8937650.99923131:59
14.3936583.9330780.99923131:21
24.4479713.9546710.99923131:08
34.3161063.9515150.99923131:03
44.0954063.9314600.99846231:01
53.9885813.9111320.99846230:48

\n\n

\n \n \n 18.37% [97/528 05:19<23:40 3.9634]\n
\n "},"metadata":{}},{"output_type":"stream","text":"Better model found at epoch 0 with val_loss value: 3.893765449523926.\nEpoch 4: reducing lr to 0.0007150975805486059\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" \nlr=1e-2 # ran stratified 224,284*2,now ffulls\nlearn2.unfreeze()\n#learn2.load('save')\nlearn2.load('dense_ar_c384') #0.026030\t0.656774\t0.892308\nlearn2.fit_one_cycle(11,slice(2e-5,lr/2))\nlearn2.save('dense_ar_c424_1')\n \n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8702d5374ae1ccbf478df1c6a8291d9bd4e552ac"},"cell_type":"code","source":"\nprint('Train_loss',learn1.recorder.losses[-1])\nprint('Val loss',learn1.recorder.val_losses)\n\nprint('Accuracy',learn1.recorder.metrics)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train_loss',learn2.recorder.losses[-1])\nprint('Val loss',learn2.recorder.val_losses)\n\nprint('Accuracy',learn2.recorder.metrics)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4a5700070a9592079e38ec34831e7fe465b19bc"},"cell_type":"markdown","source":"Prediction Part using TTA . TTA helps scoring better . In the base line model where in i used pure classification . I was able to get a score 82 in Private leader board.Using ArcFace loss on Pure classification able to reach to 87 on Private LB image sz-324"},{"metadata":{"trusted":true,"_uuid":"226fca94d64d8f167635ea5a403e67e2a4f0021e"},"cell_type":"code","source":"learn1.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#learn2.recorder.plot_losses()\n#w=tensor([0,1,0])\n#torch.nonzero(w == 0)\n#y1t[487]\n\n#len(set(data2.train_ds.y.items))","execution_count":null,"outputs":[]},{"metadata":{"id":"aB4iP3cp8ddq","colab_type":"code","outputId":"e93b53d0-c004-465a-a887-999c4c912c39","colab":{"base_uri":"https://localhost:8080/","height":40},"trusted":true,"_uuid":"f2d5580c681ba0d0866b90130269edfb0f31e7a4"},"cell_type":"code","source":"\"\"\" \nlearn2.model.eval()\n#!cp unfreeze_284_1.pth ./data/models/\nlearn2.load('dense_ar_c424_1')\npreds2,y = learn2.TTA(ds_type=DatasetType.Test,beta=0.30,with_loss=False,scale=1.08)\n\"\"\"\nlearn1.load('resnet_ar_c424_2')\npreds1,y = learn1.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"id":"8ttH7xbX8mFC","colab_type":"code","outputId":"d5829649-9333-465d-9523-a06c3eabc94b","colab":{"base_uri":"https://localhost:8080/","height":40},"trusted":true,"_uuid":"902e86d07588347de2238343934a2e3438e5ae71"},"cell_type":"code","source":"\n \n\nlearn1.model.eval()\nlearn1.load('resnet_ar_c424_2')\npreds1,y1 = learn1.TTA(ds_type=DatasetType.Test,beta=0.30,with_loss=False,scale=1.1)\n\n#preds1,y = learn1.get_preds(ds_type=DatasetType.Test)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" \nlearn1.model.eval()\nlearn1.load('resnet_ar_c424_1')\npreds1t,y1t = learn1.TTA(ds_type=DatasetType.Train,beta=0.30,with_loss=False,scale=1.1)\n\"\"\" ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" \n# intra class mean\nl=[]\nfor c in list(set(data2.train_ds.y.items)):\n l.append(torch.mean(preds1t[torch.nonzero(y1t == c)],dim=0))\n\ntrn_centre=torch.cat(l)\ntrn_centre.size()\n \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1.size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n#learn1.load('resnet_ar_c424_1')\nlearn1.model.eval()\n#%%time\nsims = []\nwith torch.no_grad():\n \n for feat in preds1:\n dists = F.cosine_similarity(trn_centre, feat.unsqueeze(0).repeat(5004, 1))\n predicted_similarity = dists.cuda()#learn.model.head(dists.cuda())\n sims.append(predicted_similarity.squeeze().detach().cpu())\n \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sims[20][sims[20].argsort(descending=True)[:5]]","execution_count":null,"outputs":[]},{"metadata":{"id":"WRxUb_f7R4mK","colab_type":"code","outputId":"d0cb31b8-763f-46c0-f615-94515bca12e7","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"e91f609d458b470ac172f2e26b39e0a09ab1dd02"},"cell_type":"code","source":"#! cp /kaggle/working/models/resnet_ar_c356.pth /kaggle/working/\n#!ls -l ./models/\n#! cd models\n#torch.max(preds1,preds2).shape\n\n#torch.mean(preds1,preds2)\n#FileLink('resnet_ar_c356.pth')\n\"\"\"\nlearn1.model.eval()\nlearn1.load('resnet_ar_c356_1')\npredsv,y_v = learn1.TTA(ds_type=DatasetType.Valid,beta=0.30,with_loss=False,scale=1.08)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apk(actual, predicted, k=10):\n if len(predicted)>k:\n predicted = predicted[:k]\n\n score = 0.0\n num_hits = 0.0\n\n for i,p in enumerate(predicted):\n if p in actual and p not in predicted[:i]:\n num_hits += 1.0\n score += num_hits / (i+1.0)\n\n if not actual:\n return 0.0\n\n return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=10):\n return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])\ndef sigmoid_np(x):\n return 1.0/(1.0 + np.exp(-x))\n#preds_t = np.stack(preds1[0], axis=-1)\n#print(preds_t.shape)\npreds_tv = sigmoid_np(predsv )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#np.linspace(0.5, 1, 10)\ni=preds_tv[99,:].argsort(descending=True)\nprobs = preds_tv[0,i]\nprobs[:5] \nnp.stack(top_5s_v).shape\nlabels_list[2]\n#y_v[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nclasses = df.Id.unique()\nnew_whale_idx = np.where(classes == 'new_whale')[0][0]\n#top_5s = []\nfor thresh in np.linspace(0.5, 1, 20):\n top_5s_v = []\n for sim in preds_tv:\n idxs = sim.argsort(descending=True)\n probs = sim[idxs]\n top_5 = []\n for i, p in zip(idxs, probs):\n #if 'new_whale' not in top_5 and p =0:\n \n \n \n trn_tfms,_ = get_transforms(do_flip=False, flip_vert=True, max_rotate=5., max_zoom=1.08,\n max_lighting=0.15, max_warp=0. )\n #protein_stats = ([0.16258, 0.13877, 0.10067 ], [0.21966, 0.18559, 0.25573 ])\n \n\n data = (src.transform((trn_tfms,trn_tfms), size=484,resize_method=ResizeMethod.SQUISH)\n .databunch(bs=32,num_workers=0).normalize(imagenet_stats)) #40\n #data.c=5004\n learn = create_cnn(\n data,\n resnet501,\n split_on=_resnet_split,\n cut=-2,\n # dense,\n \n #cut=-1,\n # split_on=_densenet_split,\n \n lin_ftrs=[1024],\n \n #lambda m: (m[0][11], m[1]),\n loss_func=ar,\n #custom_head=custom_headres,\n custom_head=custom_head,\n #custom_head,\n #torch.nn.MultiLabelSoftMarginLoss(),\n #F.binary_cross_entropy_with_logits,\n #FocalLoss(),\n #F.binary_cross_entropy_with_logits,\n path=path1, \n metrics=[accuracy], callback_fns= partial(GradientClipping, clip=1))\n #learn.callback_fns.append(partial(SaveModelCallback,monitor='val_loss',mode='min'))\n learn.callback_fns.append(partial(ReduceLROnPlateauCallback, min_delta=1e-5, patience=3))\n #learn.to_fp16()\n #learn.load('stage-1-rn50')\n #print('load')\n \n lr=1e-2\n #4e-4 # every 3-4 epocs reduce by 1 2e-3,1e-3.slice (lr/10,lr )\n #learn.load('stage-1-rn50-u7datablocks')\n #learn.load('stage-1-rn50-u11_512')\n if i==0:\n \n learn.load('resnet_ar_c424_1')\n print('x')\n else :\n\n learn.load('resnet_ar_c484') \n print(i) \n learn.unfreeze()\n learn.fit_one_cycle(1, slice(2e-5,lr/2))\n learn.save('resnet_ar_c484')\n i=i+1\n#re\n \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33e5708a19ed1ad043b2e48971ba10d5ec92bd23"},"cell_type":"code","source":"#learn.recorder.losses\n\"\"\" \nlr=2e-2 # ran stratified 224,284*2,now ffulls\n\nlearn2.unfreeze()\n#learn2.load('save')\nlearn2.load('dense_ar_c424') #0.026030\t0.656774\t0.892308\nlearn2.fit_one_cycle(7,slice(2e-4,lr/2))\nlearn2.save('dense_ar_c424')\n\"\"\"\n\"\"\" \nlr=1e-2 # ran stratified 224,284*2,now ffulls\n\nlearn1.unfreeze()\n#learn2.load('save')\nlearn1.load('resnet_ar_c484') #0.026030\t0.656774\t0.892308\nlearn1.fit_one_cycle(7,slice(2e-5,lr/2))\nlearn1.save('resnet_ar_c484_2')\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print('Train_loss',learn2.recorder.losses[-1])\n#print('Val loss',learn2.recorder.val_losses)\n\n#print('Accuracy',learn2.recorder.metrics)","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"hump_multilable_classification","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"accelerator":"GPU","language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}