{"cells":[{"metadata":{"_uuid":"c209bf0fe1740b1ad55fe814cd375e41830f6cf8"},"cell_type":"markdown","source":"First implementation of kernel was taken from https://www.kaggle.com/matthewa313/resnet50 .\nModel Architecture is changed in this kernel\nThis kernel uses Stochastic Gradient Descent with restarts to train model, it performed better than resnet101 trained without restarts for same number of epochs"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install fastai==0.7.0 --no-deps\n!pip install torch==0.4.1 torchvision==0.2.1\n!pip install torchtext==0.2.3","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import cv2\nfrom fastai.conv_learner import *\nfrom fastai.dataset import *\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport os\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nimport matplotlib.pyplot as plt\nimport math\n\n\nMODEL_NAME = 'Densenet201'\nTRAIN = '../input/train/'\nTEST = '../input/test/'\nLABELS = '../input/train.csv'\nSAMPLE_SUB = '../input/sample_submission.csv'\n\n# Backbone architecture\narch = resnet50 #dn201 #resnet50\n# Number of workers for data preprocessing\nnum_workers = 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa04184e7949d211f82469d57b721109e6dc3250"},"cell_type":"code","source":"df = pd.read_csv(LABELS).set_index('Image')\nnew_whale_df = df[df.Id == \"new_whale\"] # only new_whale dataset\ntrain_df = df[~(df.Id == \"new_whale\")] # no new_whale dataset, used for training\nunique_labels = np.unique(train_df.Id.values)\n\nlabels_dict = dict()\nlabels_list = []\nfor i in range(len(unique_labels)):\n    labels_dict[unique_labels[i]] = i\n    labels_list.append(unique_labels[i])\nprint(\"Number of classes: {}\".format(len(unique_labels)))\ntrain_df.Id = train_df.Id.apply(lambda x: labels_dict[x])\ntrain_labels = np.asarray(train_df.Id.values)\ntest_names = [f for f in os.listdir(TEST)]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f4ee95748815fa0ae0d021d3098223ce9a9eea2"},"cell_type":"code","source":"labels_count = train_df.Id.value_counts()\n\nplt.figure(figsize=(18, 4))\nplt.subplot(121)\n_, _,_ = plt.hist(labels_count.values)\nplt.ylabel(\"frequency\")\nplt.xlabel(\"class size\")\n\nplt.title('class distribution; log scale')\nlabels_count.head()\n\nprint(\"Count for class new_whale: {}\".format(labels_count[0]))\nplt.subplot(122)\n_ = plt.plot(labels_count[1:].values)\nplt.title('w/o class new_whale; log scale')\nplt.xlabel(\"class\")\nplt.ylabel(\"log(size)\")\nplt.gca().set_yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29ba82ed93529abd58689820de5fb2d475458d78"},"cell_type":"code","source":"train_df['image_name'] = train_df.index\n\nrs = np.random.RandomState(42) # set random seed to be equal to the sense of life\nperm = rs.permutation(len(train_df))\n\ntr_n = train_df['image_name'].values\n# Yes, we will validate on the subset of training data\nval_n = train_df['image_name'].values[perm][:1000]\n\nprint('Train/val:', len(tr_n), len(val_n))\nprint('Train classes', len(train_df.loc[tr_n].Id.unique()))\nprint('Val classes', len(train_df.loc[val_n].Id.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffb18ab30ad867dd4c57d4c3fbd0a9d2bc676978"},"cell_type":"code","source":"class HWIDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.train_df = train_df\n        super().__init__(fnames, transform, path)\n\n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        img = cv2.resize(img, (self.sz, self.sz))\n        return img\n\n    def get_y(self, i):\n        if (self.path == TEST): return 0\n        return self.train_df.loc[self.fnames[i]]['Id']\n\n    def get_c(self):\n        return len(unique_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"386e2adbd9063b28e7a07130c216beff6b91cff4"},"cell_type":"code","source":"class RandomLighting(Transform):\n    def __init__(self, b, c, tfm_y=TfmType.NO):\n        super().__init__(tfm_y)\n        self.b, self.c = b, c\n\n    def set_state(self):\n        self.store.b_rand = rand0(self.b)\n        self.store.c_rand = rand0(self.c)\n\n    def do_transform(self, x, is_y):\n        if is_y and self.tfm_y != TfmType.PIXEL: return x  # add this line to fix the bug\n        b = self.store.b_rand\n        c = self.store.c_rand\n        c = -1 / (c - 1) if c < 0 else c + 1\n        x = lighting(x, b, c)\n        return x\n    \ndef get_data(sz, batch_size):\n    \"\"\"\n    Read data and do augmentations\n    \"\"\"\n    aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),\n                RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),\n                RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),\n                RandomFlip(tfm_y=TfmType.NO)]\n    tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,\n                           aug_tfms=aug_tfms)\n    ds = ImageData.get_ds(HWIDataset, (tr_n[:-(len(tr_n) % batch_size)], TRAIN),\n                          (val_n, TRAIN), tfms, test=(test_names, TEST))\n    md = ImageData(\"./\", ds, batch_size, num_workers=num_workers, classes=None)\n    return md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e1836188afb8d11d47cf9506fc3d05468d36eca"},"cell_type":"code","source":"image_size = 224\nbatch_size = 48\nmd = get_data(image_size, batch_size)\nextra_fc_layers_size = []\nlearn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size) \nlearn.opt_fn = optim.Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1977d9edd73904c382b0ab3b0458ce5a2744ad01"},"cell_type":"code","source":"print('Number of layer groups:', len(learn.get_layer_groups()), '\\t(first 2 groups is pretrained backbone)')\nprint('This is our extra thin on top of the backbone Resnet50 architecture:')\nlearn.get_layer_groups()[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"998c40209f58b8ccf515d57cef536b464dd5c60d"},"cell_type":"code","source":"# learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ca6bf4a3b2a4620554d7dae94fc27ddb1eb4176"},"cell_type":"code","source":"# learn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc88e322f9ae552d0f46f6ba69f6c7a4b9368ac6"},"cell_type":"code","source":"base_lr = 5e-4 # lr for the backbone\nfc_lr = 5e-3 # lr for the classifer\n\nlrs = [base_lr, base_lr, fc_lr]\n# Freeze backbone and train the classifier for 2 epochs\nlearn.fit(lrs=lrs, n_cycle=2, cycle_len=None)\n\n# Unfreeze backbone and continue training for 9 epochs\nlearn.unfreeze()\nlearn.fit(lrs, n_cycle=3, cycle_len=1, cycle_mult=2)\nlearn.save('weights')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7f6048cbe3d055bec2ce262cb2c176c57bb11fb"},"cell_type":"code","source":"image_size = 448\nbatch_size = 24\nmd = get_data(image_size, batch_size)\nlearn.set_data(md)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbd207c8a0f0c28d92b1c8c648af415b6083b03f"},"cell_type":"code","source":"base_lr = 1e-5 # lr for the backbone\nfc_lr = 1e-3 # lr for the classifer\nlrs = [base_lr, base_lr, fc_lr]\nlearn.fit(lrs, n_cycle=6, cycle_len=1)\nlearn.save('weights_v2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b33a1feee5166231674d96b6893db94519b0fdb"},"cell_type":"code","source":"best_th = 0.38\npreds_t,y_t = learn.TTA(is_test=True,n_aug=8)\npreds_t = np.stack(preds_t, axis=-1)\npreds_t = np.exp(preds_t)\npreds_t = preds_t.mean(axis=-1)\npreds_t = np.concatenate([np.zeros((preds_t.shape[0],1))+best_th, preds_t],axis=1)\nnp.save('preds_dn201.npy', preds_t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b44e8886b0d0781bca7d1817ad2900f175dab91"},"cell_type":"code","source":"sample_df = pd.read_csv(SAMPLE_SUB)\nsample_list = list(sample_df.Image)\nlabels_list = [\"new_whale\"]+labels_list\npred_list = [[labels_list[i] for i in p.argsort()[-5:][::-1]] for p in preds_t]\npred_dic = dict((key, value) for (key, value) in zip(learn.data.test_ds.fnames,pred_list))\npred_list_cor = [' '.join(pred_dic[id]) for id in sample_list]\ndf = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})\ndf.to_csv('sub_{}.csv'.format(MODEL_NAME), header=True, index=False)\ndf.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}