{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport plotly.express as px\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-02T01:14:13.327378Z","iopub.execute_input":"2022-02-02T01:14:13.327776Z","iopub.status.idle":"2022-02-02T01:14:13.331717Z","shell.execute_reply.started":"2022-02-02T01:14:13.327746Z","shell.execute_reply":"2022-02-02T01:14:13.330816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read the data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/happy-whale-and-dolphin/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-01T23:55:34.522213Z","iopub.execute_input":"2022-02-01T23:55:34.522470Z","iopub.status.idle":"2022-02-01T23:55:34.594530Z","shell.execute_reply.started":"2022-02-01T23:55:34.522438Z","shell.execute_reply":"2022-02-01T23:55:34.593585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Species Distribution\n\nLet's look at the distribution of the different speicies in the training set","metadata":{}},{"cell_type":"code","source":"species_count = train.groupby('species')['individual_id'].agg(species_count= 'count').reset_index()\nno_of_individuals = train.groupby('individual_id')['image'].agg(no_of_individuals='count').reset_index()\n\ntrain = pd.merge(train, species_count, on=['species'], how='inner')\ntrain = pd.merge(train, no_of_individuals, on=['individual_id'], how='inner')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T23:55:34.595732Z","iopub.execute_input":"2022-02-01T23:55:34.595975Z","iopub.status.idle":"2022-02-01T23:55:34.697043Z","shell.execute_reply.started":"2022-02-01T23:55:34.595947Z","shell.execute_reply":"2022-02-01T23:55:34.696206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at the distribution of different species","metadata":{}},{"cell_type":"code","source":"px.bar(species_count.sort_values(by='species_count', ascending=False), x='species', y='species_count', color='species')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:14:26.481786Z","iopub.execute_input":"2022-02-02T00:14:26.482158Z","iopub.status.idle":"2022-02-02T00:14:26.694111Z","shell.execute_reply.started":"2022-02-02T00:14:26.482121Z","shell.execute_reply":"2022-02-02T00:14:26.693380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's plot the same using a pie chart","metadata":{}},{"cell_type":"code","source":"px.pie(train, names='species')","metadata":{"execution":{"iopub.status.busy":"2022-02-01T23:56:17.165903Z","iopub.execute_input":"2022-02-01T23:56:17.166248Z","iopub.status.idle":"2022-02-01T23:56:17.612056Z","shell.execute_reply.started":"2022-02-01T23:56:17.166213Z","shell.execute_reply":"2022-02-01T23:56:17.611004Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observations\n1. Obviously there are some duplicates in the species name with different spellings\n    - bottlenose_dolphin and bottlenose_dolpin\n    - killer_whale and kiler_whale\n2. Almost half of the data comes from the three species bottlenose_dolphin, beluga and humpback_whale\n3. Blue whale, false_killer_whale and dusky_dolphin constitute almost 1/5th of the entrire dataset\n4. Almost 15 species have less than 1% of the data, with frasiers_dolphin having the least amount of data (just 14 images)","metadata":{}},{"cell_type":"markdown","source":"#### Check if there are any individual_id's having different naming conventions\n","metadata":{}},{"cell_type":"code","source":"set(train[train['species']=='killer_whale']['individual_id'].unique()).intersection(set(train[train['species']=='kiler_whale']['individual_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:42:09.422815Z","iopub.execute_input":"2022-02-02T00:42:09.423212Z","iopub.status.idle":"2022-02-02T00:42:09.452720Z","shell.execute_reply.started":"2022-02-02T00:42:09.423173Z","shell.execute_reply":"2022-02-02T00:42:09.451327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nset(train[train['species']=='bottlenose_dolphin']['individual_id'].unique()).intersection(set(train[train['species']=='bottlenose_dolpin']['individual_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:42:57.356154Z","iopub.execute_input":"2022-02-02T00:42:57.356700Z","iopub.status.idle":"2022-02-02T00:42:57.387800Z","shell.execute_reply.started":"2022-02-02T00:42:57.356645Z","shell.execute_reply":"2022-02-02T00:42:57.386742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like we do not have any overlap :)","metadata":{}},{"cell_type":"markdown","source":"## Individuals Distribution","metadata":{}},{"cell_type":"code","source":"px.bar(train.groupby('species')['individual_id'].nunique().reset_index().rename(columns={'individual_id':'no_of_individuals'}).sort_values(by='no_of_individuals', ascending=False), \n       x='species', y='no_of_individuals', color='species')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:59:06.245808Z","iopub.execute_input":"2022-02-02T00:59:06.246100Z","iopub.status.idle":"2022-02-02T00:59:06.450054Z","shell.execute_reply.started":"2022-02-02T00:59:06.246069Z","shell.execute_reply":"2022-02-02T00:59:06.449349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(train[train.species=='bottlenose_dolphin'], x='individual_id', y='no_of_individuals', title='bottlenose_dolphin distribution')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:30:17.842443Z","iopub.execute_input":"2022-02-02T00:30:17.842750Z","iopub.status.idle":"2022-02-02T00:30:17.994754Z","shell.execute_reply.started":"2022-02-02T00:30:17.842716Z","shell.execute_reply":"2022-02-02T00:30:17.994056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(train[train.species=='kiler_whale'], x='individual_id', y='no_of_individuals', title='Killer whale distribution')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:30:47.015149Z","iopub.execute_input":"2022-02-02T00:30:47.016043Z","iopub.status.idle":"2022-02-02T00:30:47.106585Z","shell.execute_reply.started":"2022-02-02T00:30:47.015976Z","shell.execute_reply":"2022-02-02T00:30:47.105673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(species_count.sort_values(by='species_count', ascending=False), x='species', y='species_count', color='species')","metadata":{"execution":{"iopub.status.busy":"2022-02-02T00:13:52.057874Z","iopub.execute_input":"2022-02-02T00:13:52.058194Z","iopub.status.idle":"2022-02-02T00:13:52.277335Z","shell.execute_reply.started":"2022-02-02T00:13:52.058160Z","shell.execute_reply":"2022-02-02T00:13:52.276482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize some sample images of different species","metadata":{}},{"cell_type":"code","source":"# function to display multiple images: Modified from (Thanks to https://www.kaggle.com/ruchi798/and-identification-eda-augmentation)\n\ndef path(group,group_type):\n    PATH = \"../input/happy-whale-and-dolphin/train_images\"\n    \n    #species\n    if group_type=='species':\n        z = train['image'][train['species']==group].values \n    \n    #ID\n    if group_type=='id':\n        z = train['image'][train['individual_id']==group].values \n   \n    image_names = []\n    for filename in z:\n        fullpath = os.path.join(PATH, filename)\n        image_names.append(fullpath)\n    return image_names\n\n\n\ndef display_multiple_imgs(group, group_type, rows, cols):\n    \n    image_paths = path(group, group_type)\n    image_paths = np.random.choice(image_paths, rows*cols)\n    \n    figure, ax = plt.subplots(nrows=rows,ncols=cols,figsize=(16,8))\n    plt.suptitle(group, fontsize=20)\n    for ind,image_path in enumerate(image_paths):\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) \n        image = cv2.resize(image, (1200, 800))\n        try:\n            ax.ravel()[ind].imshow(image)\n            ax.ravel()[ind].set_axis_off()\n        except:\n            continue;\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:21:41.223859Z","iopub.execute_input":"2022-02-02T01:21:41.224175Z","iopub.status.idle":"2022-02-02T01:21:41.236429Z","shell.execute_reply.started":"2022-02-02T01:21:41.224144Z","shell.execute_reply":"2022-02-02T01:21:41.235357Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for species in train['species'].unique():\n    print('\\n\\n')\n    display_multiple_imgs(species, 'species', 3, 3)","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:23:26.659076Z","iopub.execute_input":"2022-02-02T01:23:26.659919Z","iopub.status.idle":"2022-02-02T01:24:39.998913Z","shell.execute_reply.started":"2022-02-02T01:23:26.659860Z","shell.execute_reply":"2022-02-02T01:24:39.998184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Will keep the notebook updated with new EDA","metadata":{"execution":{"iopub.status.busy":"2022-02-02T01:17:55.642266Z","iopub.execute_input":"2022-02-02T01:17:55.642591Z","iopub.status.idle":"2022-02-02T01:17:55.903956Z","shell.execute_reply.started":"2022-02-02T01:17:55.642560Z","shell.execute_reply":"2022-02-02T01:17:55.902873Z"}}}]}