{"cells":[{"metadata":{"_uuid":"a707b88775cf9e88004afb837f38cbd907ef030d"},"cell_type":"markdown","source":"First implementation of kernel was taken from https://www.kaggle.com/matthewa313/resnet50\nThis kernel uses Stochastic Gradient Descent with restarts to train model, it performed better than resnet101 trained without restarts for same number of epochs"},{"metadata":{"trusted":true,"_uuid":"0771db954b7e1a7dc9cc8af35681af45b9e9ce85"},"cell_type":"code","source":"!pip install fastai==0.7.0 --no-deps\n!pip install torch==0.4.1 torchvision==0.2.1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.dataset import *\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport os\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nimport matplotlib.pyplot as plt\nimport math\n\n\nMODEL_NAME = 'Resnet101'\nTRAIN = '../input/train/'\nTEST = '../input/test/'\nLABELS = '../input/train.csv'\nSAMPLE_SUB = '../input/sample_submission.csv'\n\n# Backbone architecture\narch = resnet101\n# Number of workers for data preprocessing\nnum_workers = 4","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv(LABELS).set_index('Image')\nnew_whale_df = df[df.Id == \"new_whale\"] # only new_whale dataset\ntrain_df = df[~(df.Id == \"new_whale\")] # no new_whale dataset, used for training\nunique_labels = np.unique(train_df.Id.values)\n\nlabels_dict = dict()\nlabels_list = []\nfor i in range(len(unique_labels)):\n    labels_dict[unique_labels[i]] = i\n    labels_list.append(unique_labels[i])\nprint(\"Number of classes: {}\".format(len(unique_labels)))\ntrain_df.Id = train_df.Id.apply(lambda x: labels_dict[x])\ntrain_labels = np.asarray(train_df.Id.values)\ntest_names = [f for f in os.listdir(TEST)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3446e72607c71f781d0b0359d929b2eaabc3a665"},"cell_type":"code","source":"labels_count = train_df.Id.value_counts()\n\nplt.figure(figsize=(18, 4))\nplt.subplot(121)\n_, _,_ = plt.hist(labels_count.values)\nplt.ylabel(\"frequency\")\nplt.xlabel(\"class size\")\n\nplt.title('class distribution; log scale')\nlabels_count.head()\n\nprint(\"Count for class new_whale: {}\".format(labels_count[0]))\nplt.subplot(122)\n_ = plt.plot(labels_count[1:].values)\nplt.title('w/o class new_whale; log scale')\nplt.xlabel(\"class\")\nplt.ylabel(\"log(size)\")\nplt.gca().set_yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63145b7ea0499f3496b7aceb0a44b9a3eadf51b6"},"cell_type":"code","source":"train_df['image_name'] = train_df.index\n\nrs = np.random.RandomState(42) # set random seed to be equal to the sense of life\nperm = rs.permutation(len(train_df))\n\ntr_n = train_df['image_name'].values\n# Yes, we will validate on the subset of training data\nval_n = train_df['image_name'].values[perm][:1000]\n\nprint('Train/val:', len(tr_n), len(val_n))\nprint('Train classes', len(train_df.loc[tr_n].Id.unique()))\nprint('Val classes', len(train_df.loc[val_n].Id.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1f0ff43bdb86d4609f4f8fc9e7fc20b8a650369"},"cell_type":"code","source":"class HWIDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.train_df = train_df\n        super().__init__(fnames, transform, path)\n\n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        img = cv2.resize(img, (self.sz, self.sz))\n        return img\n\n    def get_y(self, i):\n        if (self.path == TEST): return 0\n        return self.train_df.loc[self.fnames[i]]['Id']\n\n    def get_c(self):\n        return len(unique_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70f9972ba636d3bcdb5a03e83ef0773e5610b893"},"cell_type":"code","source":"class RandomLighting(Transform):\n    def __init__(self, b, c, tfm_y=TfmType.NO):\n        super().__init__(tfm_y)\n        self.b, self.c = b, c\n\n    def set_state(self):\n        self.store.b_rand = rand0(self.b)\n        self.store.c_rand = rand0(self.c)\n\n    def do_transform(self, x, is_y):\n        if is_y and self.tfm_y != TfmType.PIXEL: return x  # add this line to fix the bug\n        b = self.store.b_rand\n        c = self.store.c_rand\n        c = -1 / (c - 1) if c < 0 else c + 1\n        x = lighting(x, b, c)\n        return x\n    \ndef get_data(sz, batch_size):\n    \"\"\"\n    Read data and do augmentations\n    \"\"\"\n    aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),\n                RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),\n                RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),\n                RandomFlip(tfm_y=TfmType.NO)]\n    tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,\n                           aug_tfms=aug_tfms)\n    ds = ImageData.get_ds(HWIDataset, (tr_n[:-(len(tr_n) % batch_size)], TRAIN),\n                          (val_n, TRAIN), tfms, test=(test_names, TEST))\n    md = ImageData(\"./\", ds, batch_size, num_workers=num_workers, classes=None)\n    return md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0ccb8d363341f41cc54f1c194f7077da213f732"},"cell_type":"code","source":"image_size = 299\nbatch_size = 32\nmd = get_data(image_size, batch_size)\nextra_fc_layers_size = []\nlearn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size) \nlearn.opt_fn = optim.Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a924c658ffcbc5e6b6ef4e7bfadbe32823c9be22"},"cell_type":"code","source":"print('Number of layer groups:', len(learn.get_layer_groups()), '\\t(first 2 groups is pretrained backbone)')\nprint('This is our extra thin on top of the backbone Resnet50 architecture:')\nlearn.get_layer_groups()[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28b75c3889c82b51c220d9056ed6bcf4c1273e38"},"cell_type":"code","source":"# learn.lr_find()\n# learn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b15b73459fcaae2a6ac1edabf747017c33648457"},"cell_type":"code","source":"base_lr = 5e-4 # lr for the backbone\nfc_lr = 1e-3 # lr for the classifer\n\nlrs = [base_lr, base_lr, fc_lr]\n# Freeze backbone and train the classifier for 2 epochs\nlearn.fit(lrs=lrs, n_cycle=2, cycle_len=None)\n\n# Unfreeze backbone and continue training for 9 epochs\nlearn.unfreeze()\nlearn_bn_unfreeze()\nlearn.fit(lrs, n_cycle=4, cycle_len=1, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce0b07248966f7b3bcbe3c6e54259c6583feddb0"},"cell_type":"code","source":"base_lr = 1e-4 # lr for the backbone\nfc_lr = 1e-3 # lr for the classifer\nlrs = [base_lr, base_lr, fc_lr]\nlearn.fit(lrs, n_cycle=6, cycle_len=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9489adc51153c2dc01dee74b3fbd6d0a0c152a53"},"cell_type":"code","source":"best_th = 0.38\npreds_t,y_t = learn.TTA(is_test=True,n_aug=8)\npreds_t = np.stack(preds_t, axis=-1)\npreds_t = np.exp(preds_t)\npreds_t = preds_t.mean(axis=-1)\npreds_t = np.concatenate([np.zeros((preds_t.shape[0],1))+best_th, preds_t],axis=1)\nnp.save('preds.npy', preds_t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a2311f2174a423c30a003277dbee915127b651a"},"cell_type":"code","source":"sample_df = pd.read_csv(SAMPLE_SUB)\nsample_list = list(sample_df.Image)\nlabels_list = [\"new_whale\"]+labels_list\npred_list = [[labels_list[i] for i in p.argsort()[-5:][::-1]] for p in preds_t]\npred_dic = dict((key, value) for (key, value) in zip(learn.data.test_ds.fnames,pred_list))\npred_list_cor = [' '.join(pred_dic[id]) for id in sample_list]\ndf = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})\ndf.to_csv('submission_{}.csv'.format(MODEL_NAME), header=True, index=False)\ndf.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}