{"cells":[{"metadata":{},"cell_type":"markdown","source":"**The purpose of this notebook is to create simple augmentation dataset for minority classes**\n\nThe better option is to include the augmentation step in the data generator for the model. However, I wanted to share this notebook just to illustrate how some simple augmentation can be done","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom PIL import Image\n\nfrom PIL import ImageFilter,ImageEnhance,ImageChops,ImageOps\nfrom PIL import ImageEnhance\nfrom tqdm import tqdm\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DIR = '../input/landmark-retrieval-2020/'\n#path_to_train = DIR + '/train/'\ntrain = pd.read_csv(\"../input/landmark-retrieval-2020/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Now we get paths of all images using**\n\n\n\nWe use the kernel : https://www.kaggle.com/derinformatiker/landmark-retrieval-all-paths . PLease upvote this kernel if you found the path extraction piece helpful like I did","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_paths(sub):\n    index = [\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"a\",\"b\",\"c\",\"d\",\"e\",\"f\"]\n\n    paths = []\n\n    for a in index:\n        for b in index:\n            for c in index:\n                try:\n                    paths.extend([f\"../input/landmark-retrieval-2020/{sub}/{a}/{b}/{c}/\" + x for x in os.listdir(f\"../input/landmark-retrieval-2020/{sub}/{a}/{b}/{c}\")])\n                except:\n                    pass\n\n    return paths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = train\n\nrows = []\nfor i in tqdm(range(len(train))):\n    row = train.iloc[i]\n    path  = list(row[\"id\"])[:3]\n    temp = row[\"id\"]\n    row[\"id\"] = f\"../input/landmark-retrieval-2020/train/{path[0]}/{path[1]}/{path[2]}/{temp}.jpg\"\n    rows.append(row[\"id\"])\n    \nrows = pd.DataFrame(rows)\ntrain_path[\"id\"] = rows","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k =train[['id','landmark_id']].groupby(['landmark_id']).agg({'id':'count'})\nk.rename(columns={'id':'Count_class'}, inplace=True)\nk.reset_index(level=(0), inplace=True)\nfreq_ct_df = pd.DataFrame(k)\nfreq_ct_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.merge(train,freq_ct_df, on = ['landmark_id'], how='left')\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels_lt3 = train_labels[train_labels['Count_class']<3]\ntrain_labels_lt3.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##lets take a sample of 100 images\naug_sample = train_labels_lt3.sample(100)\nimg_list = aug_sample['id'].tolist()\nid_list = aug_sample['landmark_id'].tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels_aug = pd.DataFrame(columns=['id','landmark_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def update_file_path(filename, sufx):\n    parts = filename.split('.')\n    return \"\".join(parts[:-1])+ '_' + sufx + '.' + parts[-1]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Creating some sample transformations. One can add many more in a similar manner\nfor imagefile, IdFile in zip(img_list, id_list):\n    im=Image.open(imagefile)\n    im_blur=im.filter(ImageFilter.GaussianBlur)\n    im_unsharp=im.filter(ImageFilter.UnsharpMask(radius=2, percent=150, threshold=3))\n    im_edgeenhance = im.filter(ImageFilter.EDGE_ENHANCE_MORE)\n    #im_invert = im.transpose(Image.FLIP_LEFT_RIGHT)\n    im_rot30=im.rotate(30)\n    os.chdir('../working/')\n    _blur_path=update_file_path(imagefile, 'bl')\n    im_blur.save(\"\".join(os.path.splitext(os.path.basename(_blur_path))))\n    _unsharp_path=update_file_path(imagefile, 'un')\n    im_unsharp.save(\"\".join(os.path.splitext(os.path.basename(_unsharp_path))))\n    _edgeenhance_path=update_file_path(imagefile, 'edgenh')\n    im_edgeenhance.save(\"\".join(os.path.splitext(os.path.basename(_edgeenhance_path))))\n    _imrot30_path=update_file_path(imagefile, 'rot30')\n    im_rot30.save(\"\".join(os.path.splitext(os.path.basename(_imrot30_path))))\n    train_labels_aug= train_labels_aug.append({'id' : \"\".join(os.path.splitext(os.path.basename(_blur_path))) , 'landmark_id' : IdFile} , ignore_index=True)\n    train_labels_aug = train_labels_aug.append({'id' : \"\".join(os.path.splitext(os.path.basename(_unsharp_path))), 'landmark_id' : IdFile} , ignore_index=True)\n    train_labels_aug = train_labels_aug.append({'id' : \"\".join(os.path.splitext(os.path.basename(_edgeenhance_path))) , 'landmark_id' : IdFile} , ignore_index=True)\n    train_labels_aug = train_labels_aug.append({'id' : \"\".join(os.path.splitext(os.path.basename(_imrot30_path))) , 'landmark_id' : IdFile} , ignore_index=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels_aug.to_csv('train_labels_aug.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"**If you found this even a little helpful, an upvote would be massively appreciated. Cheers!!**\n\nThanks to the Google and Kaggle team for creating this competition every year.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}