{"cells":[{"metadata":{"trusted":true,"_uuid":"9495ee2fa938baba558faa4fb7223da5cb7e0f6a"},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom fastai.vision import *\nfrom fastai.metrics import accuracy\nfrom fastai.basic_data import *\nfrom skimage.util import montage\nimport pandas as pd\nfrom torch import optim\nimport re\n\nfrom utils import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac60deefdb7f16c401da9206505ca43e453fad54"},"cell_type":"markdown","source":"## Prepare data"},{"metadata":{"trusted":true,"_uuid":"ec358049eda156b12e2b192ca80ecf9785c35a71"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83c7353d84d8eb700fb73087161fdbb279eb8013"},"cell_type":"code","source":"im_count = df[df.Id != 'new_whale'].Id.value_counts()\nim_count.name = 'sighting_count'\ndf = df.join(im_count, on='Id')\nval_fns = set(df.sample(frac=1)[(df.Id != 'new_whale') & (df.sighting_count > 1)].groupby('Id').first().Image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3485f7e434a52db50b252137cc532350e70023f1"},"cell_type":"code","source":"# pd.to_pickle(val_fns, 'data/val_fns')\n#val_fns = pd.read_pickle('data/val_fns')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f6ae95f67a1953894d2f01360f176c4b563fb5e"},"cell_type":"code","source":"fn2label = {row[1].Image: row[1].Id for row in df.iterrows()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ebf72553918d7e93669276f9f381445632ea8db"},"cell_type":"code","source":"SZ = 224\nBS = 64\nNUM_WORKERS = 0\nSEED=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4007fb435e58e2cabc0141ea1265ce9086e83d0f"},"cell_type":"code","source":"path2fn = lambda path: re.search('\\w*\\.jpg$', path).group(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34e48abe3a801776d2f373c03de427e456d22f56"},"cell_type":"code","source":"df = df[df.Id != 'new_whale']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a0ecf90a00a0da0ed0752db96ca87376341a646"},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3349e62591b8cbfa244b10cd0e109bf2185eee8"},"cell_type":"code","source":"df.sighting_count.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d8e79327831cd6245f19b3e90b60c4fce2e3c21"},"cell_type":"code","source":"df_val = df[df.Image.isin(val_fns)]\ndf_train = df[~df.Image.isin(val_fns)]\ndf_train_with_val = df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eeb9d99a6e66bd4c9d9c6723e7a8e9e2e0b4f30d"},"cell_type":"code","source":"df_val.shape, df_train.shape, df_train_with_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"360430177835f68a6f2bfea1d82d6a9975e45541"},"cell_type":"code","source":"%%time\n\nres = None\nsample_to = 15\n\nfor grp in df_train.groupby('Id'):\n    n = grp[1].shape[0]\n    additional_rows = grp[1].sample(0 if sample_to < n  else sample_to - n, replace=True)\n    rows = pd.concat((grp[1], additional_rows))\n    \n    if res is None: res = rows\n    else: res = pd.concat((res, rows))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ed5fe4716fc9f2b3a9bc939260f99c9202c30ad"},"cell_type":"code","source":"%%time\n\nres_with_val = None\nsample_to = 15\n\nfor grp in df_train_with_val.groupby('Id'):\n    n = grp[1].shape[0]\n    additional_rows = grp[1].sample(0 if sample_to < n  else sample_to - n, replace=True)\n    rows = pd.concat((grp[1], additional_rows))\n    \n    if res_with_val is None: res_with_val = rows\n    else: res_with_val = pd.concat((res_with_val, rows))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b85805051d8c034883a957cd2341b2cfa29e2109"},"cell_type":"code","source":"res.shape, res_with_val.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd2877b16effd07d1337cc8a9a44ea4a33992fc3"},"cell_type":"markdown","source":"Our training set increased 6-fold, but that is still an amount of data that is okay. I don't think it makes sense to worry about breaking up the data into smaller epochs."},{"metadata":{"trusted":true,"_uuid":"229d7b5db4436fe7a6d096ccd2d2a6c93e272cb0"},"cell_type":"code","source":"pd.concat((res, df_val))[['Image', 'Id']].to_csv('oversampled_train.csv', index=False)\nres_with_val[['Image', 'Id']].to_csv('oversampled_train_and_val.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fc0293e3afef5bbc6a5d97bac4f17840a37f8711"},"cell_type":"markdown","source":"The naming here is not very fortunate, but the idea is that `oversampled_train` has single entries for images in `val_fns` and `oversampled_train_and_val` is both `val` and `train` combined. Meaning, `oversampled_train_and_val` is one we might want to use when retraining on the entire train set."},{"metadata":{"trusted":true,"_uuid":"9243c5df585ddecedec6ac2a1b531e66dd439885"},"cell_type":"code","source":"df = pd.read_csv('oversampled_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6136c0c078806bf91770e91066ed77d5f3caf399"},"cell_type":"code","source":"data = (\n    ImageItemList\n        .from_df(df[df.Id != 'new_whale'], '../input/train', cols=['Image'])\n        .split_by_valid_func(lambda path: path2fn(path) in val_fns)\n        .label_from_func(lambda path: fn2label[path2fn(path)])\n        .add_test(ImageItemList.from_folder('../input/test'))\n        .transform(get_transforms(do_flip=False, max_zoom=1, max_warp=0, max_rotate=2), size=SZ, resize_method=ResizeMethod.SQUISH)\n        .databunch(bs=BS, num_workers=NUM_WORKERS, path='../input')\n        .normalize(imagenet_stats)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54aed13c97de9c373cdabc63b31345eb27d0a388"},"cell_type":"code","source":"data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08365c1d9b3b6e0538ec2d39d54ad089ba6210c1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ad2fe2d7c6a7cd768390c31e1b79cde1da4652e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.0"}},"nbformat":4,"nbformat_minor":1}