{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n'''\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-09T00:38:25.116957Z","iopub.execute_input":"2022-02-09T00:38:25.117352Z","iopub.status.idle":"2022-02-09T00:38:25.149208Z","shell.execute_reply.started":"2022-02-09T00:38:25.117236Z","shell.execute_reply":"2022-02-09T00:38:25.148363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport albumentations as A\n\n## other packages\nfrom termcolor import colored\nfrom colorama import Fore, Back, Style\n# colored output\ny_ = Fore.YELLOW\nr_ = Fore.RED\ng_ = Fore.GREEN\nb_ = Fore.BLUE\nm_ = Fore.MAGENTA\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:25:53.598124Z","iopub.execute_input":"2022-02-09T04:25:53.598804Z","iopub.status.idle":"2022-02-09T04:25:56.207930Z","shell.execute_reply.started":"2022-02-09T04:25:53.598676Z","shell.execute_reply":"2022-02-09T04:25:56.207134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img_path = '/kaggle/input/happy-whale-and-dolphin/train_images'\ntest_img_path = '/kaggle/input/happy-whale-and-dolphin/test_images'\ntrain_csv_path = '/kaggle/input/happy-whale-and-dolphin/train.csv'","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:17.764260Z","iopub.execute_input":"2022-02-09T04:26:17.764525Z","iopub.status.idle":"2022-02-09T04:26:17.768357Z","shell.execute_reply.started":"2022-02-09T04:26:17.764497Z","shell.execute_reply":"2022-02-09T04:26:17.767771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Training CSV Data","metadata":{}},{"cell_type":"code","source":"trainDF = pd.read_csv(train_csv_path)\ntrainDF.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:20.242726Z","iopub.execute_input":"2022-02-09T04:26:20.243284Z","iopub.status.idle":"2022-02-09T04:26:20.352850Z","shell.execute_reply.started":"2022-02-09T04:26:20.243246Z","shell.execute_reply":"2022-02-09T04:26:20.351948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's fix some issues with the species column:","metadata":{}},{"cell_type":"code","source":"trainDF.species.replace({\"globis\": \"short_finned_pilot_whale\",\n                          \"pilot_whale\": \"short_finned_pilot_whale\",\n                          \"kiler_whale\": \"killer_whale\",\n                          \"bottlenose_dolpin\": \"bottlenose_dolphin\"}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:23.661584Z","iopub.execute_input":"2022-02-09T04:26:23.661861Z","iopub.status.idle":"2022-02-09T04:26:23.688102Z","shell.execute_reply.started":"2022-02-09T04:26:23.661833Z","shell.execute_reply":"2022-02-09T04:26:23.687340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA\n\n### Unique Species","metadata":{}},{"cell_type":"code","source":"print('Number of unique species: ', trainDF['species'].nunique())\nprint('Species Names: ', trainDF['species'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:26.990907Z","iopub.execute_input":"2022-02-09T04:26:26.991199Z","iopub.status.idle":"2022-02-09T04:26:27.010440Z","shell.execute_reply.started":"2022-02-09T04:26:26.991143Z","shell.execute_reply":"2022-02-09T04:26:27.009837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of images per species","metadata":{}},{"cell_type":"code","source":"trainDF['species'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:29.907277Z","iopub.execute_input":"2022-02-09T04:26:29.907907Z","iopub.status.idle":"2022-02-09T04:26:29.925881Z","shell.execute_reply.started":"2022-02-09T04:26:29.907873Z","shell.execute_reply":"2022-02-09T04:26:29.925037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nsns.countplot(x='species',data=trainDF, order = trainDF['species'].value_counts().index)\nplt.xticks(rotation=90)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:33.110183Z","iopub.execute_input":"2022-02-09T04:26:33.110490Z","iopub.status.idle":"2022-02-09T04:26:33.709942Z","shell.execute_reply.started":"2022-02-09T04:26:33.110458Z","shell.execute_reply":"2022-02-09T04:26:33.709361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of animals (unique animals) per species","metadata":{}},{"cell_type":"code","source":"trainDF.groupby(['species'])['individual_id'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:40.365337Z","iopub.execute_input":"2022-02-09T04:26:40.365796Z","iopub.status.idle":"2022-02-09T04:26:40.393586Z","shell.execute_reply.started":"2022-02-09T04:26:40.365743Z","shell.execute_reply":"2022-02-09T04:26:40.393068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_count = trainDF.groupby(['species'])['individual_id'].nunique().sort_values(ascending=False)\nid_count.index, id_count.values","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:26:49.747625Z","iopub.execute_input":"2022-02-09T04:26:49.748386Z","iopub.status.idle":"2022-02-09T04:26:49.773786Z","shell.execute_reply.started":"2022-02-09T04:26:49.748346Z","shell.execute_reply":"2022-02-09T04:26:49.773168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.bar(x = id_count.index, height=id_count.values)\nplt.xticks(rotation=90)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:27:22.336631Z","iopub.execute_input":"2022-02-09T04:27:22.336927Z","iopub.status.idle":"2022-02-09T04:27:22.710325Z","shell.execute_reply.started":"2022-02-09T04:27:22.336892Z","shell.execute_reply":"2022-02-09T04:27:22.709389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of unique ID's","metadata":{}},{"cell_type":"code","source":"trainDF['individual_id'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:39:08.046237Z","iopub.execute_input":"2022-02-09T04:39:08.046516Z","iopub.status.idle":"2022-02-09T04:39:08.069956Z","shell.execute_reply.started":"2022-02-09T04:39:08.046487Z","shell.execute_reply":"2022-02-09T04:39:08.069273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training and Test Images","metadata":{}},{"cell_type":"code","source":"# Function to get image paths from train and test directory\ndef getImagePaths(path):\n    image_names = []\n    for dirname, _, filenames in os.walk(path):\n        for filename in filenames:\n            fullpath = os.path.join(dirname, filename)\n            image_names.append(fullpath)\n    return image_names\n\n# Function to display multiple images\ndef display_multiple_img(image_paths, rows, cols, title):\n    fig,ax = plt.subplots(nrows=rows, ncols=cols, figsize=(16,8))\n    plt.suptitle(title, fontsize=20)\n    for ind, img_path in enumerate(image_paths):\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        try:\n            ax.ravel()[ind].imshow(image)\n            ax.ravel()[ind].set_axis_off()\n        except:\n            continue;\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:52:09.784858Z","iopub.execute_input":"2022-02-09T04:52:09.785126Z","iopub.status.idle":"2022-02-09T04:52:09.793808Z","shell.execute_reply.started":"2022-02-09T04:52:09.785099Z","shell.execute_reply":"2022-02-09T04:52:09.792935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_paths = getImagePaths(train_img_path)\ntest_images_paths = getImagePaths(test_img_path)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:48:24.719859Z","iopub.execute_input":"2022-02-09T04:48:24.720137Z","iopub.status.idle":"2022-02-09T04:49:26.301584Z","shell.execute_reply.started":"2022-02-09T04:48:24.720107Z","shell.execute_reply":"2022-02-09T04:49:26.300366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'{y_}Number of train images: ' + f'{g_} {len(train_images_paths)}\\n')\nprint(f'{y_}Number of test images: ' + f'{g_} {len(test_images_paths)}\\n')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:50:36.675538Z","iopub.execute_input":"2022-02-09T04:50:36.676344Z","iopub.status.idle":"2022-02-09T04:50:36.682611Z","shell.execute_reply.started":"2022-02-09T04:50:36.676293Z","shell.execute_reply":"2022-02-09T04:50:36.681648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_multiple_img(train_images_paths[0:25],5,5,'Train images')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:52:13.915994Z","iopub.execute_input":"2022-02-09T04:52:13.917103Z","iopub.status.idle":"2022-02-09T04:52:26.146074Z","shell.execute_reply.started":"2022-02-09T04:52:13.917054Z","shell.execute_reply":"2022-02-09T04:52:26.145222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_multiple_img(test_images_paths[0:25],5,5,'Test images')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:53:16.198499Z","iopub.execute_input":"2022-02-09T04:53:16.198768Z","iopub.status.idle":"2022-02-09T04:53:28.906094Z","shell.execute_reply.started":"2022-02-09T04:53:16.198739Z","shell.execute_reply":"2022-02-09T04:53:28.905307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Images of most frequent species","metadata":{}},{"cell_type":"code","source":"def n_most_frequent(df, col, n, most=True):\n    if most:\n        return df[col].value_counts()[:n].index.tolist()\n    else:\n        return df[col].value_counts()[-n:].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:59:25.711096Z","iopub.execute_input":"2022-02-09T04:59:25.711421Z","iopub.status.idle":"2022-02-09T04:59:25.717604Z","shell.execute_reply.started":"2022-02-09T04:59:25.711386Z","shell.execute_reply":"2022-02-09T04:59:25.716482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF['species'].value_counts()[:5].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-02-09T04:59:55.798023Z","iopub.execute_input":"2022-02-09T04:59:55.798320Z","iopub.status.idle":"2022-02-09T04:59:55.812100Z","shell.execute_reply.started":"2022-02-09T04:59:55.798288Z","shell.execute_reply":"2022-02-09T04:59:55.811512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m_freq_species = n_most_frequent(trainDF, 'species', 5, True)\nl_freq_species = n_most_frequent(trainDF, 'species', 5, False)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T05:07:57.148651Z","iopub.execute_input":"2022-02-09T05:07:57.148938Z","iopub.status.idle":"2022-02-09T05:07:57.168997Z","shell.execute_reply.started":"2022-02-09T05:07:57.148909Z","shell.execute_reply":"2022-02-09T05:07:57.167976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainDF[trainDF['species']==m_freq_species[0]]['image'].values","metadata":{"execution":{"iopub.status.busy":"2022-02-09T05:12:15.441858Z","iopub.execute_input":"2022-02-09T05:12:15.442297Z","iopub.status.idle":"2022-02-09T05:12:15.458424Z","shell.execute_reply.started":"2022-02-09T05:12:15.442263Z","shell.execute_reply":"2022-02-09T05:12:15.457430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec = m_freq_species[0]\nz = trainDF[trainDF['species']==spec]['image'].values.tolist()\nz[0:9]","metadata":{"execution":{"iopub.status.busy":"2022-02-09T05:19:53.342254Z","iopub.execute_input":"2022-02-09T05:19:53.342750Z","iopub.status.idle":"2022-02-09T05:19:53.361329Z","shell.execute_reply.started":"2022-02-09T05:19:53.342719Z","shell.execute_reply":"2022-02-09T05:19:53.360305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for spec in m_freq_species:\n    z = trainDF[trainDF['species']==spec]['image'].values.tolist()\n    z9 = z[0:9]\n    fullpaths = [os.path.join(train_img_path, x) for x in z9]\n    display_multiple_img(fullpaths,3,3,spec)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T05:22:17.713221Z","iopub.execute_input":"2022-02-09T05:22:17.713537Z","iopub.status.idle":"2022-02-09T05:22:40.699783Z","shell.execute_reply.started":"2022-02-09T05:22:17.713504Z","shell.execute_reply":"2022-02-09T05:22:40.699098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"def plot_augmentations(images, titles, sup_title):\n    fig,axes = plt.subplots(figsize=(20,16), nrows=3, ncols=4, squeeze=False)\n    \n    for indx, (img, title) in enumerate(zip(images, titles)):\n        axes[indx//4][indx%4].imshow(img)\n        axes[indx//4][indx%4].set_title(title, fontsize=15)\n        \n    plt.tight_layout()\n    fig.suptitle(sup_title, fontsize=20)\n    fig.subplots_adjust(wspace=0.2, hspace=0.2, top=0.93)\n    axes[2,2].set_visible(False)\n    axes[2,3].set_visible(False)\n    plt.show()\n\ndef augment(paths, data):\n    albumentations = [A.RandomSunFlare(p=0.02), A.RandomFog(p=1), A.RandomBrightness(p=1),\n                      A.Rotate(p=1, limit=9), A.RGBShift(p=1), A.RandomSnow(p=0.02),\n                      A.HorizontalFlip(p=1), A.RandomContrast(limit=0.5,p=1),\n                      A.HueSaturationValue(p=1, hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=50)]\n    titles = [\"RandomSunFlare\",\"RandomFog\",\"RandomBrightnessContrast\",\n                       \"Rotate\", \"RGBShift\", \"RandomSnow\",\"HorizontalFlip\", \"RandomContrast\",\"HSV\"]\n    for i in paths:\n        image_path = i\n        image_name = image_path.split(\"/\")[4].split(\".\")[0]\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        # RESIZE IMAGE\n        image = cv2.resize(image, (224,224))\n        \n        images = []\n        for aug_type in albumentations:\n            augmented_img = aug_type(image=image)['image']\n            images.append(augmented_img)\n        \n        titles.insert(0,\"original\")\n        images.insert(0,image)\n        sup_title = 'Image Augmentation for '+ data+ \" - \" +image_name \n        plot_augmentations(images, titles, sup_title)\n        titles.remove('original')\n        \naugment(train_images_paths[0:2],'train')","metadata":{"execution":{"iopub.status.busy":"2022-02-09T05:37:20.174825Z","iopub.execute_input":"2022-02-09T05:37:20.175359Z","iopub.status.idle":"2022-02-09T05:37:25.096212Z","shell.execute_reply.started":"2022-02-09T05:37:20.175316Z","shell.execute_reply":"2022-02-09T05:37:25.095146Z"},"trusted":true},"execution_count":null,"outputs":[]}]}