{"cells":[{"metadata":{"_uuid":"ebc69f6b291192996581866767c35bba3cb440dd"},"cell_type":"markdown","source":"Train the Resnet50 from this kernel: https://www.kaggle.com/suicaokhoailang/resnet50-bounding-boxes-0-628-lb\n\nTraining another Resnet101, just change the architecture.\n\nPlug your outputs to this kernel and you're good to go.\n\nThis kernel is based on the approach from https://www.kaggle.com/suicaokhoailang/ensembling-with-averaged-probabilities-0-701-lb"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.dataset import *\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport os\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nimport matplotlib.pyplot as plt\nimport math\n\narch = resnet50\nnum_workers = 4\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nLABELS = '../input/humpback-whale-identification/train.csv'\nSAMPLE_SUB = '../input/humpback-whale-identification/sample_submission.csv'\nBBOX = '../input/generating-whale-bounding-boxes/bounding_boxes.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56a165755a0d2918d6c5721c7bd360192ce9a897"},"cell_type":"code","source":"df = pd.read_csv(LABELS).set_index('Image')\nnew_whale_df = df[df.Id == \"new_whale\"] # only new_whale dataset\ntrain_df = df[~(df.Id == \"new_whale\")] # no new_whale dataset, used for training\nunique_labels = np.unique(train_df.Id.values)\n\nlabels_dict = dict()\nlabels_list = []\nfor i in range(len(unique_labels)):\n    labels_dict[unique_labels[i]] = i\n    labels_list.append(unique_labels[i])\ntrain_df.Id = train_df.Id.apply(lambda x: labels_dict[x])\ntrain_labels = np.asarray(train_df.Id.values)\ntest_names = [f for f in os.listdir(TEST)]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df['image_name'] = train_df.index\ntr_n = train_df['image_name'].values\n# Yes, we will validate on the subset of training data\nval_n = train_df['image_name']\nprint('Train/val:', len(tr_n), len(val_n))\nprint('Train classes', len(train_df.loc[tr_n].Id.unique()))\nprint('Val classes', len(train_df.loc[val_n].Id.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ae7d05736f4e9da200eb2e6d7d3a4aec6e048a6"},"cell_type":"code","source":"class HWIDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.train_df = train_df\n        super().__init__(fnames, transform, path)\n\n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        bbox = bbox_df.loc[self.fnames[i]]\n        x0, y0, x1, y1 = bbox['x0'], bbox['y0'], bbox['x1'],  bbox['y1']\n        if not (x0 >= x1 or y0 >= y1):\n            img = img[y0:y1, x0:x1,:]\n        img = cv2.resize(img, (self.sz, self.sz))\n        return img\n\n    def get_y(self, i):\n        if (self.path == TEST): return 0\n        return self.train_df.loc[self.fnames[i]]['Id']\n\n    def get_c(self):\n        return len(unique_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f3140b745078c19caefb4fa9a49004a01e73767"},"cell_type":"code","source":"class HWIDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.train_df = train_df\n        super().__init__(fnames, transform, path)\n\n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        bbox = bbox_df.loc[self.fnames[i]]\n        x0, y0, x1, y1 = bbox['x0'], bbox['y0'], bbox['x1'],  bbox['y1']\n        if not (x0 >= x1 or y0 >= y1):\n            img = img[y0:y1, x0:x1,:]\n        img = cv2.resize(img, (self.sz, self.sz))\n        return img\n\n    def get_y(self, i):\n        if (self.path == TEST): return 0\n        return self.train_df.loc[self.fnames[i]]['Id']\n\n    def get_c(self):\n        return len(unique_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9235bc25b4dc81c11e4fc45ab30ca5cf602c0f5a"},"cell_type":"code","source":"def get_data(sz, batch_size):\n    \"\"\"\n    Read data and do augmentations\n    \"\"\"\n    aug_tfms = []\n    tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,\n                           aug_tfms=aug_tfms)\n    ds = ImageData.get_ds(HWIDataset, (tr_n[:-(len(tr_n) % batch_size)], TRAIN),\n                          (val_n, TRAIN), tfms, test=(test_names, TEST))\n    md = ImageData(\"./\", ds, batch_size, num_workers=num_workers, classes=None)\n    return md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a88144dba6d62bcd029d0779860659111dae8f29"},"cell_type":"code","source":"image_size = 384\nbatch_size = 32\nmd = get_data(image_size, batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0333ca7f8dfd10bc26e717364ccae5d868d0c26"},"cell_type":"code","source":"best_th = 0.38","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a54245cb37186218a7f2f18bd804a25786396e4"},"cell_type":"code","source":"preds_t = np.load(\"../input/humpbackdata/resnet101.npy\") + np.load(\"../input/humpbackdata/resnet50.npy\") \npreds_t /= 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f927dd6b9382c179e72ca9b45aa241d715bcfd3"},"cell_type":"code","source":"sample_df = pd.read_csv(SAMPLE_SUB)\nsample_list = list(sample_df.Image)\nlabels_list = [\"new_whale\"]+labels_list\npred_list = [[labels_list[i] for i in p.argsort()[-5:][::-1]] for p in preds_t]\npred_dic = dict((key, value) for (key, value) in zip(md.test_ds.fnames,pred_list))\npred_list_cor = [' '.join(pred_dic[id]) for id in sample_list]\ndf = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})\ndf.to_csv('submission.csv', header=True, index=False)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43312dac8fdcbda62faac6cbdf74f1089a4a3002"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}