{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\nfrom fastai.metrics import accuracy\nfrom fastai.basic_data import *\nimport pandas as pd\n\nfrom utils import *\n\ndf = pd.read_csv('data/train.csv')\ndf.head()\n\ndf.Id.value_counts().head()\n\n(df.Id == 'new_whale').mean()\n\n(df.Id.value_counts() == 1).mean()\n\ndf.Id.nunique()\n\ndf.shape\n\nfn2label = {row[1].Image: row[1].Id for row in df.iterrows()}\n\nSZ = 224\nBS = 64\nNUM_WORKERS = 12\nSEED=0\n\ndata = (\n    ImageList\n        .from_folder('data/train-224')\n        .random_split_by_pct(seed=SEED)\n        .label_from_func(lambda path: fn2label[path.name])\n        .add_test(ImageList.from_folder('data/test-224'))\n        .transform(get_transforms(do_flip=False, max_zoom=1, max_warp=0, max_rotate=2), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n)\n\ndata.show_batch(rows=3)\n\nname = f'res50-{SZ}'\n\nlearn = create_cnn(data, models.resnet50, metrics=[accuracy, map5])\n\nlearn.fit_one_cycle(1)\n\nlearn.recorder.plot_losses()\n\nlearn.save(f'{name}-stage-1')\n\nlearn.unfreeze()\n\nlearn.lr_find()\n\nlearn.recorder.plot()\n\nmax_lr = 1e-4\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(5, lrs)\n\nlearn.save(f'{name}-stage-2')\n\nlearn.recorder.plot_losses()\n\npreds, _ = learn.get_preds(DatasetType.Test)\n\nmkdir -p subs\n\ncreate_submission(preds, learn.data, name)\n\npd.read_csv(f'subs/{name}.csv.gz').head()\n\n!kaggle competitions submit -c humpback-whale-identification -f subs/{name}.csv.gz -m \"{name}\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom fastai.vision import *\nfrom fastai.metrics import accuracy\nfrom fastai.basic_data import *\nimport pandas as pd\nimport re\n\nfrom utils import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df = pd.read_csv('data/train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"(df.Id != 'new_whale').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"im_count = df[df.Id != 'new_whale'].Id.value_counts()\nim_count.name = 'sighting_count'\ndf = df.join(im_count, on='Id'); df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"val_fns = set(df.sample(frac=1)[(df.Id != 'new_whale') & (df.sighting_count > 1)].groupby('Id').first().Image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"len(val_fns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"val_fns = val_fns.union(set(df[df.Id == 'new_whale'].sample(frac=1).Image.values[:1000]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"len(val_fns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"fn2label = {row[1].Image: 'new_whale' if row[1].Id == 'new_whale' else 'known_whale' for row in df.iterrows()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"SZ = 224\nBS = 64\nNUM_WORKERS = 12\nSEED=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"path2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data = (\n    ImageItemList\n        .from_df(df, 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False, max_zoom=1, max_warp=0, max_rotate=2), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data.show_batch(rows=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":false},"cell_type":"code","source":"name = f'res50-{SZ}'","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn = create_cnn(data, models.resnet50, metrics=[accuracy])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.fit_one_cycle(4, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.save(f'{name}-stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.fit_one_cycle(4, [1e-6, 1e-5, 1e-4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.save(f'{name}-stage-2')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Accuracy of 77% sounds good, right?"},{"metadata":{"trusted":false},"cell_type":"code","source":"preds, targs = learn.get_preds()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.data.classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# https://en.wikipedia.org/wiki/Precision_and_recall\ntp = ((preds.argmax(1) == 1).long() * targs).sum()\ntn = ((preds.argmax(1) == 0).long() * (targs==0).long()).sum()\nfn = ((preds.argmax(1) == 0).long() * targs).sum()\nfp = ((preds.argmax(1) == 1).long() * (targs==0).long()).sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# recall of new_whale\ntp/(tp+fn).float()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# precision\ntp/(tp+fp).float()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# accuracy\n(tp+tn)/(tp+tn+fp+fn).float()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom fastai.vision import *\nfrom fastai.metrics import accuracy\nfrom fastai.basic_data import *\nfrom skimage.util import montage\nimport pandas as pd\nfrom torch import optim\nimport re\n\nfrom utils import *\n\ndf = pd.read_csv('data/train.csv')\ndf.head()\nim_count = df[df.Id != 'new_whale'].Id.value_counts()\nim_count.name = 'sighting_count'\ndf = df.join(im_count, on='Id')\nval_fns = set(df.sample(frac=1)[(df.Id != 'new_whale') & (df.sighting_count > 1)].groupby('Id').first().Image)\n\n# pd.to_pickle(val_fns, 'data/val_fns')\nval_fns = pd.read_pickle('data/val_fns')\n\nfn2label = {row[1].Image: row[1].Id for row in df.iterrows()}\n\nSZ = 224\nBS = 64\nNUM_WORKERS = 12\nSEED=0\n\npath2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)\n\ndf = df[df.Id != 'new_whale']\n\ndf.shape\n\ndf.sighting_count.max()\n\ndf_val = df[df.Image.isin(val_fns)]\ndf_train = df[~df.Image.isin(val_fns)]\ndf_train_with_val = df\n\ndf_val.shape, df_train.shape, df_train_with_val.shape\n\n%%time\n\nres = None\nsample_to = 15\n\nfor grp in df_train.groupby('Id'):\n    n = grp[1].shape[0]\n    additional_rows = grp[1].sample(0 if sample_to < n  else sample_to - n, replace=True)\n    rows = pd.concat((grp[1], additional_rows))\n    \n    if res is None: res = rows\n    else: res = pd.concat((res, rows))\n        \n%%time\n\nres_with_val = None\nsample_to = 15\n\nfor grp in df_train_with_val.groupby('Id'):\n    n = grp[1].shape[0]\n    additional_rows = grp[1].sample(0 if sample_to < n  else sample_to - n, replace=True)\n    rows = pd.concat((grp[1], additional_rows))\n    \n    if res_with_val is None: res_with_val = rows\n    else: res_with_val = pd.concat((res_with_val, rows))\n\nres.shape, res_with_val.shape\n\npd.concat((res, df_val))[['Image', 'Id']].to_csv('data/oversampled_train.csv', index=False)\nres_with_val[['Image', 'Id']].to_csv('data/oversampled_train_and_val.csv', index=False)\n\ndf = pd.read_csv('data/oversampled_train.csv')\n\ndata = (\n    ImageItemList\n        .from_df(df[df.Id != 'new_whale'], 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False, max_zoom=1, max_warp=0, max_rotate=2), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)\n\ndata\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom fastai.vision import *\nfrom fastai.metrics import accuracy\nfrom fastai.basic_data import *\nfrom skimage.util import montage\nimport pandas as pd\nfrom torch import optim\nimport re\n\nfrom utils import *\n\nimport fastai\nfrom fastprogress import force_console_behavior\nimport fastprogress\nfastprogress.fastprogress.NO_BAR = True\nmaster_bar, progress_bar = force_console_behavior()\nfastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar\n\ndf = pd.read_csv('data/train.csv')\nval_fns = {'69823499d.jpg'}\n\nfn2label = {row[1].Image: row[1].Id for row in df.iterrows()}\npath2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)\n\nname = f'res50-full-train'\n\nSZ = 224\nBS = 64\nNUM_WORKERS = 12\nSEED=0\n\ndata = (\n    ImageItemList\n        .from_df(df[df.Id != 'new_whale'], 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)\n\n%%time\n\nlearn = create_cnn(data, models.resnet50, lin_ftrs=[2048])\nlearn.clip_grad();\n\nlearn.fit_one_cycle(14, 1e-2)\nlearn.save(f'{name}-stage-1')\n\nlearn.unfreeze()\n\nmax_lr = 1e-3\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(24, lrs)\nlearn.save(f'{name}-stage-2')\n\nSZ = 224 * 2\nBS = 64 // 4\nNUM_WORKERS = 12\nSEED=0\n\ndata = (\n    ImageItemList\n        .from_df(df[df.Id != 'new_whale'], 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)\n\n%%time\nlearn = create_cnn(data, models.resnet50, lin_ftrs=[2048])\nlearn.clip_grad();\nlearn.load(f'{name}-stage-2')\nlearn.freeze_to(-1)\n\nlearn.fit_one_cycle(12, 1e-2 / 4)\nlearn.save(f'{name}-stage-3')\n\nlearn.unfreeze()\n\nmax_lr = 1e-3 / 4\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(22, lrs)\nlearn.save(f'{name}-stage-4')\n\n# with oversampling\ndf = pd.read_csv('data/oversampled_train_and_val.csv')\n\ndata = (\n    ImageItemList\n        .from_df(df, 'data/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('data/test'))\n        .transform(get_transforms(do_flip=False), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='data')\n        .normalize(imagenet_stats)\n)\n\n%%time\nlearn = create_cnn(data, models.resnet50, lin_ftrs=[2048])\nlearn.clip_grad();\nlearn.load(f'{name}-stage-4')\nlearn.freeze_to(-1)\n\nlearn.fit_one_cycle(2, 1e-2 / 4)\nlearn.save(f'{name}-stage-5')\n\nlearn.unfreeze()\n\nmax_lr = 1e-3 / 4\nlrs = [max_lr/100, max_lr/10, max_lr]\n\nlearn.fit_one_cycle(3, lrs)\nlearn.save(f'{name}-stage-6')\n\npreds, _ = learn.get_preds(DatasetType.Test)\n\npreds = torch.cat((preds, torch.ones_like(preds[:, :1])), 1)\n\npreds[:, 5004] = 0.06\nclasses = learn.data.classes + ['new_whale']\n\ncreate_submission(preds, learn.data, name, classes)\n\npd.read_csv(f'subs/{name}.csv.gz').head()\n\npd.read_csv(f'subs/{name}.csv.gz').Id.str.split().apply(lambda x: x[0] == 'new_whale').mean()\n\n!kaggle competitions submit -c humpback-whale-identification -f subs/{name}.csv.gz -m \"{name}\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The classifier picks up on a whale not being known 72% of the time. Out of all the whales it identifies as new, only 54% are.\n\nAlas, this seems like a dead end. I could squeeze some LB performance out of this (when combined with another model) but definitely not worth the time at this point."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.0"}},"nbformat":4,"nbformat_minor":1}