{"cells":[{"metadata":{"trusted":true,"_uuid":"4fe61ce9653332d4790733dfdf1915aaeb433dd4"},"cell_type":"code","source":"!pip install fastai==0.7.0 --no-deps\n!pip install torch==0.4.1 torchvision==0.2.1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.dataset import *\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport os\nfrom sklearn.model_selection import train_test_split, StratifiedShuffleSplit\nimport matplotlib.pyplot as plt\nimport math\nimport cv2\n\narch = resnext50\nnum_workers = 4\nTRAIN = '../input/humpback-whale-identification/train/'\nTEST = '../input/humpback-whale-identification/test/'\nLABELS = '../input/humpback-whale-identification/train.csv'\nSAMPLE_SUB = '../input/humpback-whale-identification/sample_submission.csv'\nBBOX = '../input/generating-whale-bounding-boxes/bounding_boxes.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9adfc15b56c7f80f291c66dc6d6f38d4d55e6a2"},"cell_type":"code","source":"df = pd.read_csv(LABELS).set_index('Image')\nnew_whale_df = df[df.Id == \"new_whale\"] # only new_whale dataset\ntrain_df = df[~(df.Id == \"new_whale\")] # no new_whale dataset, used for training\nunique_labels = np.unique(train_df.Id.values)\n\nlabels_dict = dict()\nlabels_list = []\nfor i in range(len(unique_labels)):\n    labels_dict[unique_labels[i]] = i\n    labels_list.append(unique_labels[i])\ntrain_df.Id = train_df.Id.apply(lambda x: labels_dict[x])\ntrain_labels = np.asarray(train_df.Id.values)\ntest_names = [f for f in os.listdir(TEST)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a494334eac4b3e32e4756395dd2167ee94368f86"},"cell_type":"code","source":"train_df['image_name'] = train_df.index\ntr_n = train_df['image_name'].values\n# Yes, we will validate on the subset of training data\nval_n = train_df['image_name']\nprint('Train/val:', len(tr_n), len(val_n))\nprint('Train classes', len(train_df.loc[tr_n].Id.unique()))\nprint('Val classes', len(train_df.loc[val_n].Id.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94af91d70819db979d39a4d77b2e30493498978b"},"cell_type":"code","source":"class HWIDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.train_df = train_df\n        super().__init__(fnames, transform, path)\n\n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        bbox = bbox_df.loc[self.fnames[i]]\n        x0, y0, x1, y1 = bbox['x0'], bbox['y0'], bbox['x1'],  bbox['y1']\n        if not (x0 >= x1 or y0 >= y1):\n            img = img[y0:y1, x0:x1,:]\n        img = cv2.resize(img, (self.sz, self.sz))\n        return img\n\n    def get_y(self, i):\n        if (self.path == TEST): return 0\n        return self.train_df.loc[self.fnames[i]]['Id']\n\n    def get_c(self):\n        return len(unique_labels)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"140dfae2b41cbe4f770f8d80fcaba0ebc772e983"},"cell_type":"code","source":"def get_data(sz, batch_size):\n    \"\"\"\n    Read data and do augmentations\n    \"\"\"\n    aug_tfms = []\n    tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,\n                           aug_tfms=aug_tfms)\n    ds = ImageData.get_ds(HWIDataset, (tr_n[:-(len(tr_n) % batch_size)], TRAIN),\n                          (val_n, TRAIN), tfms, test=(test_names, TEST))\n    md = ImageData(\"./\", ds, batch_size, num_workers=num_workers, classes=None)\n    return md\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8258255beb8fb608abb8a292b07c7161580007e"},"cell_type":"code","source":"image_size = 448\nbatch_size = 8\nmd = get_data(image_size, batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ed2aa4395dd4f573aeaad5ac25a5f4f5078354f"},"cell_type":"code","source":"best_th = 0.38","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cbfaedbad6bac01b06d87eaf3723dd260b7a51e"},"cell_type":"code","source":"\npreds_t = np.load(\"../input/resnext50-sz224-b32-591/rx50_preds.npy\") + np.load(\"../input/fork-of-two-resnext50-sz448-674/rx50_448_preds.npy\")\npreds_t /= 2\nnp.save(\"rx50_448_preds.npy\",preds_t)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82b4e79d2a05267790200de4860fd25a0a669f7f"},"cell_type":"markdown","source":"Finally, our submission."},{"metadata":{"trusted":true,"_uuid":"f5fbd91e970d375debc3270ebd5b08bb41eeb66e"},"cell_type":"code","source":"sample_df = pd.read_csv(SAMPLE_SUB)\nsample_list = list(sample_df.Image)\nlabels_list = [\"new_whale\"]+labels_list\npred_list = [[labels_list[i] for i in p.argsort()[-5:][::-1]] for p in preds_t]\npred_dic = dict((key, value) for (key, value) in zip(md.test_ds.fnames,pred_list))\npred_list_cor = [' '.join(pred_dic[id]) for id in sample_list]\ndf = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})\ndf.to_csv('submission.csv', header=True, index=False)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1e13c5254d34d2b0ca5e9684cc82444f26108f9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}