{"cells":[{"metadata":{"trusted":true,"_uuid":"67d5bdae960ee5f848d0c6cf25c331d5d8513a49"},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom fastai.vision import *\nfrom fastai.metrics import accuracy\nfrom fastai.basic_data import *\nfrom skimage.util import montage\nimport pandas as pd\nfrom torch import optim\nimport re\n\nfrom utils import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4704ee99dae9bb5ea6b994bc8d2e3918fd518513"},"cell_type":"code","source":"# import fastai\n# from fastprogress import force_console_behavior\n# import fastprogress\n# fastprogress.fastprogress.NO_BAR = True\n# master_bar, progress_bar = force_console_behavior()\n# fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21949e6507e6b560a291f2e356f8d478b986c4a5"},"cell_type":"markdown","source":"It is mildly annoying that people are able to get to ~0.61 on the LB using a pretrained CNN on full images, which is a level of performance that has eluded us thus far. Let's see if we can change this!\n\nTo improve the score, these are some of the things we will do:\n* train longer and on bigger images\n* use biggger or newer architecture (though this is slightly annoying as it would make the training run much longer, maybe it can be avoided)\n* use more data augmentation\n* balancing the classes\n* ~~more aggressively tune the threshold for the 'new_whale' label (this is also a great way to get into trouble on the test set)~~\n* retrain on the entire train set (particularly important for whales with just a couple of pictures)\n* if this does not give us adequate level of performance, I will attempt SWA or some other form of ensembling"},{"metadata":{"_uuid":"e3ee9c49e8344fa33e2266d74d38b10f7989a6da"},"cell_type":"markdown","source":"## Prepare data"},{"metadata":{"trusted":true,"_uuid":"5b2ad65966977303b0bb59fefc144068c53e025b"},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2328f6952528646a3394a34282a469ee14920527"},"cell_type":"code","source":"path = Path('../input/')\npath_test = Path('../input/test')\npath_train = Path('../input/train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3c571ddeddc2c8c8a4b9e7c0117114f18d7dcf6"},"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b68864e132e6ca71aed1de06cf59568a814cc51"},"cell_type":"code","source":"im_count = df[df.Id != 'new_whale'].Id.value_counts()\nim_count.name = 'sighting_count'\ndf = df.join(im_count, on='Id')\nval_fns = set(df.sample(frac=1)[(df.Id != 'new_whale') & (df.sighting_count > 1)].groupby('Id').first().Image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"500ac5f62f5cb007480e59587cbc5485c7758dae"},"cell_type":"code","source":"# pd.to_pickle(val_fns, 'data/val_fns')\n#val_fns = pd.read_pickle('data/val_fns')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fb23773caca61cd75094ac245dc1c74e6e4682e"},"cell_type":"code","source":"fn2label = {row[1].Image: row[1].Id for row in df.iterrows()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce2f28eca1ad30bcc08fb092f9d9d903948143b3"},"cell_type":"code","source":"SZ = 224\nBS = 64\nNUM_WORKERS = 0\nSEED=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd0ac710bc712d8906d282028e8abb699054d22d"},"cell_type":"code","source":"path2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4eaefc1d1a1efbf59a4b445aa47295fe65a941f"},"cell_type":"code","source":"data = (\n    ImageItemList\n        .from_df(df[df.Id != 'new_whale'], '../input/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('../input/test'))\n        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='../input')\n        .normalize(imagenet_stats)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e235ab18e2f2f68dae1700625abde2ae261fa84"},"cell_type":"code","source":"data.show_batch(rows=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"399d880382f2969727a2af387d25672251db7cc9"},"cell_type":"code","source":"data.train_ds[1][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56403b2ddc861ee8d5bb12d9786107a80943d6a6"},"cell_type":"code","source":"data.train_ds[1][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14efcbbccc72c4359a9e87be9b01852ce05627fc"},"cell_type":"code","source":"data.train_ds[1][0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f92834e5e60a60416d9bbd7b23dab29dfedb9078"},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true,"_uuid":"ceb6c3912771147500cc04301a93a4ac22dd735e"},"cell_type":"code","source":"name = f'res50-tuned-{SZ}'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93d57f464ab00afea47e2a750e1ff3a6aa8ff7a7"},"cell_type":"markdown","source":"Warning - code below captures the gist of the experiments I ran, but now looking at the results I think I may have deleted some lines where I load weights, etc. Doesn't really matter, this still has all the info I need to train on the full train set."},{"metadata":{"trusted":true,"_uuid":"fb9fd924e4b23bef2ef114d94a17fd6527f5befc"},"cell_type":"code","source":"!git clone https://github.com/radekosmulski/whale","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf434025afb225ef34d84fd71663366b7aba7ae2"},"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/working/whale')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19b5a2f6bf03d8ee5e7654235aab41842f3f0e25"},"cell_type":"code","source":"from whale.utils import map5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cd2a9b76c5196f0d25727fc4fbdb3b09b810690"},"cell_type":"code","source":"MODEL_PATH = \"/tmp/model/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f6334ca835a30cc5b0ff7e8e96ca32816822d55"},"cell_type":"code","source":"# transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n\nlearn = create_cnn(data, models.resnet50, metrics=[accuracy, map5], lin_ftrs=[2048], model_dir=MODEL_PATH)\nlearn.clip_grad();\n\nlearn.fit_one_cycle(12, 1e-2)\n\nlearn.unfreeze()\n\nmax_lr = 1e-3\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(20, lrs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"457e3946561eb4a7e1d12a7508d0e57b9b60a124"},"cell_type":"code","source":"# SZ = 448\n# transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n\nlearn = create_cnn(data, models.resnet50, metrics=[accuracy, map5], lin_ftrs=[2048])\nlearn.clip_grad();\n\nlearn.fit_one_cycle(12, 1e-2)\n\nlearn.unfreeze()\n\nmax_lr = 1e-3\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(20, lrs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5c97d5b4ac526fbb6dace37288be62e980656e9"},"cell_type":"code","source":"SZ = 224 * 2\nBS = 64 // 4\nNUM_WORKERS = 12\nSEED=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55d6059e21f24abb9bd6a28e60190fcf1aa635af"},"cell_type":"code","source":"df = pd.read_csv('data/oversampled_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef4ac29b72ce6ad910390dc72337f469320a5a3a"},"cell_type":"code","source":"data = (\n    ImageItemList\n        .from_df(df[df.Id != 'new_whale'], 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5e5fc276807b66088a0416ebce8588e83c3145d"},"cell_type":"code","source":"%%time\n# SZ = 448\n# transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n# oversampled without val\n\nlearn = create_cnn(data, models.resnet50, metrics=[accuracy, map5], lin_ftrs=[2048])\nlearn.clip_grad();\nlearn.load('small_lr');\nlearn.freeze_to(-1)\n\nlearn.fit_one_cycle(2, 1e-2 / 4)\n\nlearn.unfreeze()\n\nmax_lr = 1e-3 / 4\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(3, lrs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf270b8561f4f087ed2ab25ea6437d18e4cf3634"},"cell_type":"markdown","source":"## Adding new_whale to predictions "},{"metadata":{"trusted":true,"_uuid":"59a6e32cc85d34b175cf7f25aa49bce3b79a4198"},"cell_type":"code","source":"df = pd.read_csv('data/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cbe761e96fff34de4b248b340103e8385004d16"},"cell_type":"code","source":"new_whale_fns = set(df[df.Id == 'new_whale'].sample(frac=1).Image.iloc[:1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d344d92459d8b7954b6f7358067cb4e8aed82e4"},"cell_type":"code","source":"data = (\n    ImageItemList\n        .from_df(df, 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns.union(new_whale_fns))\n        .label_from_func(lambda path: fn2label[path2fn(path)], classes=learn.data.classes)\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False, max_zoom=1, max_warp=0, max_rotate=2), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"658e407a7728fb7bc65443388a16aae5331d70f0"},"cell_type":"code","source":"data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"302a6b2e562da786377e79db8652815835d0530e"},"cell_type":"code","source":"learn.data = data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a57d9dce01b53a58f280dd535b3cde91ead47bd"},"cell_type":"code","source":"preds, _ = learn.get_preds(DatasetType.Valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3148022df52d8c037ecf1cc7cf0cc14052cfdc6"},"cell_type":"code","source":"classes = learn.data.classes + ['new_whale']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07db34940e4b70f0b5bb53f4e0ce1142a0fdce7a"},"cell_type":"code","source":"targs = torch.tensor([classes.index(label.obj) if label else 5004 for label in learn.data.valid_ds.y])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80d8ab0d35db72f03e7b982c1f581246700fb418"},"cell_type":"code","source":"# without predicting new_whale\nmap5(preds, targs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17a45cac00c63e1ce408c58db276e7b26964488c"},"cell_type":"code","source":"preds = torch.cat((preds, torch.ones_like(preds[:, :1])), 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b36ec49ca6b4c442b5a492b1272ec9ceb3b9f3db"},"cell_type":"code","source":"# always predicting new_whale with probability = 1\nmap5(preds, targs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6637d3934febe314a7f1bc2a6e995b9c972b99e6"},"cell_type":"code","source":"%%time\nres = []\nps = np.linspace(0, 1, 51)\nfor p in ps:\n    preds[:, 5004] = p\n    res.append(map5(preds, targs).item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a720f1935cab8ce621c6ad0a8b6590a562f0e22d"},"cell_type":"code","source":"best_p = ps[np.argmax(res)]; best_p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2433514a8a0e48a50794c71415f2baa5dcf6f339"},"cell_type":"code","source":"# %%time\n\n# # alternate way of estimating p, this didn't work\n# # thinking more about this there is no reason this should work with our model,\n# # but can be attempted if we have some better way of predicting new_whale\n# res = []\n# ps = np.linspace(0, 1, 51)\n# for p in ps:\n#     preds[:, 5004] = p\n#     res.append(np.mean([lst.split()[0] == 'new_whale' for lst in top_5_pred_labels(preds, classes)]))\n# \n# best_p = ps[np.argmin(np.abs(np.array(res) - 0.3))]; best_p # I assume the test set contains ~30% of new_whales","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74612c924c273f1b6b5be7762a3da49ff2bbffd7"},"cell_type":"code","source":"preds[:, 5004] = best_p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6879a3c08e6b01b971647925c70659f01ccd07be"},"cell_type":"code","source":"map5(preds, targs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49e89947baf18a4874b58af63e3619bd14c10f72"},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":true,"_uuid":"42fa3e380df7d33de450e2a71f2c39d072030694"},"cell_type":"code","source":"preds, _ = learn.get_preds(DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4f76dc8c68090a9e31c615c4a6bffa5b16e14d0"},"cell_type":"code","source":"preds = torch.cat((preds, torch.ones_like(preds[:, :1])), 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d275c3fd0f08678dad84ee5b78f4affa33b4146"},"cell_type":"code","source":"preds[:, 5004] = best_p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58ba2e187504a6b9e425fe10c9c8134f60a50a73"},"cell_type":"code","source":"def create_submission(preds, data, name, classes=None):\n    if not classes: classes = data.classes\n    sub = pd.DataFrame({'Image': [path.name for path in data.test_ds.x.items]})\n    sub['Id'] = top_5_pred_labels(preds, classes)\n    sub.to_csv(f'{name}.csv.gz', index=False, compression='gzip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f3f82afc3f6b644d5e487a5c14c3a41dabdf0d2"},"cell_type":"code","source":"create_submission(preds, learn.data, name, classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4615006b8cce2d1372f432776a2ef07f041e95a"},"cell_type":"code","source":"pd.read_csv(f'{name}.csv.gz').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da9ceb0aa2730266da40d30875182b179be79655"},"cell_type":"code","source":"pd.read_csv(f'{name}.csv.gz').Id.str.split().apply(lambda x: x[0] == 'new_whale').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de59d4c835cef6ef4547dbe7c6fb3f2a3cd024c7"},"cell_type":"code","source":"!pip3 install kaggle-cli","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8c12deee17b60d9cf7aa29260637944384a7623"},"cell_type":"code","source":"!kaggle competitions submit -c humpback-whale-identification -f subs/{name}.csv.gz -m \"{name}\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}