{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**The objective of this competition is to identify individual dolphins and whales with a dataset of their images.\nTherefore, in this EDA, I am going through the photos and individual information provided to try to get the whole picture of our data.**","metadata":{}},{"cell_type":"code","source":"# Imports\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport seaborn as sns\nfrom PIL import Image\nfrom tqdm import tqdm\nimport random \nimport cv2","metadata":{"execution":{"iopub.status.busy":"2022-04-10T02:07:56.933818Z","iopub.execute_input":"2022-04-10T02:07:56.935704Z","iopub.status.idle":"2022-04-10T02:07:59.138851Z","shell.execute_reply.started":"2022-04-10T02:07:56.935667Z","shell.execute_reply":"2022-04-10T02:07:59.137510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/happy-whale-and-dolphin/train.csv\")\ntest = pd.read_csv(\"../input/happy-whale-and-dolphin/sample_submission.csv\")\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:54.933299Z","iopub.execute_input":"2022-04-09T13:18:54.934007Z","iopub.status.idle":"2022-04-09T13:18:55.191152Z","shell.execute_reply.started":"2022-04-09T13:18:54.933955Z","shell.execute_reply":"2022-04-09T13:18:55.190308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.192809Z","iopub.execute_input":"2022-04-09T13:18:55.193048Z","iopub.status.idle":"2022-04-09T13:18:55.203809Z","shell.execute_reply.started":"2022-04-09T13:18:55.193017Z","shell.execute_reply":"2022-04-09T13:18:55.202595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**First, check if there is any missing values**","metadata":{}},{"cell_type":"code","source":"#check for missing values\ntrain.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.205386Z","iopub.execute_input":"2022-04-09T13:18:55.205619Z","iopub.status.idle":"2022-04-09T13:18:55.237602Z","shell.execute_reply.started":"2022-04-09T13:18:55.205592Z","shell.execute_reply":"2022-04-09T13:18:55.236698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.23956Z","iopub.execute_input":"2022-04-09T13:18:55.239948Z","iopub.status.idle":"2022-04-09T13:18:55.257053Z","shell.execute_reply.started":"2022-04-09T13:18:55.23991Z","shell.execute_reply":"2022-04-09T13:18:55.255941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Adding the file path to our image dataframe below.**","metadata":{}},{"cell_type":"code","source":"train_img_dir = \"../input/happy-whale-and-dolphin/train_images/\"\ntest_img_dir = \"../input/happy-whale-and-dolphin/test_images/\"\ntrain[\"path\"] =  train_img_dir + train[\"image\"]\ntest[\"path\"] = test_img_dir + test[\"image\"]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.259333Z","iopub.execute_input":"2022-04-09T13:18:55.259885Z","iopub.status.idle":"2022-04-09T13:18:55.297185Z","shell.execute_reply.started":"2022-04-09T13:18:55.25984Z","shell.execute_reply":"2022-04-09T13:18:55.296524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check the species name**","metadata":{}},{"cell_type":"code","source":"species = train[\"species\"].unique()\nprint(\"number of unique species: \", train[\"species\"].nunique())\nprint(\"names of species: \", train[\"species\"].unique())\nspecies.sort()\nprint(\"------------------------------------------\")\nprint(train[\"species\"].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.298375Z","iopub.execute_input":"2022-04-09T13:18:55.299181Z","iopub.status.idle":"2022-04-09T13:18:55.328819Z","shell.execute_reply.started":"2022-04-09T13:18:55.29912Z","shell.execute_reply":"2022-04-09T13:18:55.328085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Cleaning miss-spelling species name\n### Rename with the major species name so each one has a \"dolphin/whale\" at the end\ntrain[\"species\"].replace(\n    {\n        \"bottlenose_dolpin\" : \"bottlenose_dolphin\",\n        \"kiler_whale\" : \"killer_whale\",\n        \"beluga\" : \"beluga_whale\",\n        \"globis\" : \"globis_whale\"\n    },\n    inplace = True\n)\nspecies = train[\"species\"].unique()\nprint(\"number of unique species: \", train[\"species\"].nunique())\nprint(\"names of species: \", train[\"species\"].unique())","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.329905Z","iopub.execute_input":"2022-04-09T13:18:55.330469Z","iopub.status.idle":"2022-04-09T13:18:55.370311Z","shell.execute_reply.started":"2022-04-09T13:18:55.330435Z","shell.execute_reply":"2022-04-09T13:18:55.369297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create label for class as dolphin/whale\ntrain['labels'] = train[\"species\"].map(lambda x : \"dolphin\" if \"dolphin\" in x else \"whale\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.372155Z","iopub.execute_input":"2022-04-09T13:18:55.37268Z","iopub.status.idle":"2022-04-09T13:18:55.401826Z","shell.execute_reply.started":"2022-04-09T13:18:55.372631Z","shell.execute_reply":"2022-04-09T13:18:55.400929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_train_image(dataset,title):\n    plt.figure(figsize = (15,10))\n    plt.suptitle(title, fontsize=30)\n    for i, sp in enumerate(dataset.species.unique()):\n        plt.subplot(4,7,i+1)\n        imgs = dataset[dataset[\"species\"] == sp].reset_index(drop = True)\n        random_pick_img_path = imgs.loc[random.randint(0, len(imgs)-1),'path']\n        img = Image.open(random_pick_img_path)\n        plt.imshow(img)\n        plt.axis(\"off\")\n        plt.title(sp)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.638394Z","iopub.execute_input":"2022-04-09T13:18:55.6387Z","iopub.status.idle":"2022-04-09T13:18:55.647781Z","shell.execute_reply.started":"2022-04-09T13:18:55.638661Z","shell.execute_reply":"2022-04-09T13:18:55.646563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Looking through photos randomly for every species in our dataset\nplot_train_image(train,\"Train Images\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:18:55.71465Z","iopub.execute_input":"2022-04-09T13:18:55.715129Z","iopub.status.idle":"2022-04-09T13:19:10.350084Z","shell.execute_reply.started":"2022-04-09T13:18:55.715091Z","shell.execute_reply":"2022-04-09T13:19:10.349213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function for retreving the width and height for every photo.\ndef get_image_size(train):\n    widths, heights = [], []\n    for path in tqdm(train[\"path\"]):\n        width, height = Image.open(path).size\n        widths.append(width)\n        heights.append(height)\n\n    train[\"widths\"] = widths\n    train[\"heights\"] = heights\n    train[\"dimensions\"] = train['widths'] * train['heights']\n","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:19:44.708833Z","iopub.execute_input":"2022-04-09T13:19:44.709526Z","iopub.status.idle":"2022-04-09T13:19:44.714826Z","shell.execute_reply.started":"2022-04-09T13:19:44.709482Z","shell.execute_reply":"2022-04-09T13:19:44.714268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.to_csv(\"train_df_with_imgs.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:19:47.694261Z","iopub.execute_input":"2022-04-09T13:19:47.69475Z","iopub.status.idle":"2022-04-09T13:19:48.051497Z","shell.execute_reply.started":"2022-04-09T13:19:47.694698Z","shell.execute_reply":"2022-04-09T13:19:48.050591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The above data processing takes around ten minutes so I have saved the result for later use.\ntrain = pd.read_csv(\"../input/happy-whale-and-dolphin-eda/train_df_with_imgs.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T02:08:03.475704Z","iopub.execute_input":"2022-04-10T02:08:03.476018Z","iopub.status.idle":"2022-04-10T02:08:03.696526Z","shell.execute_reply.started":"2022-04-10T02:08:03.475986Z","shell.execute_reply":"2022-04-10T02:08:03.695664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20,8))\nsns.scatterplot(x = \"widths\", y = \"heights\", data =train)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T02:08:06.864484Z","iopub.execute_input":"2022-04-10T02:08:06.864798Z","iopub.status.idle":"2022-04-10T02:08:07.385399Z","shell.execute_reply.started":"2022-04-10T02:08:06.864762Z","shell.execute_reply":"2022-04-10T02:08:07.384577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we actually have multiple sizes of photos in our dataset.","metadata":{}},{"cell_type":"code","source":"print(\"Imgae Max Height: \", train[\"heights\"].max())\nprint(\"Image Min Height: \", train[\"heights\"].min())\nprint(\"Image Max Width: \", train[\"widths\"].max())\nprint(\"Image Min Width: \", train[\"widths\"].min())","metadata":{"execution":{"iopub.status.busy":"2022-04-10T02:11:54.562254Z","iopub.execute_input":"2022-04-10T02:11:54.563001Z","iopub.status.idle":"2022-04-10T02:11:54.570998Z","shell.execute_reply.started":"2022-04-10T02:11:54.562959Z","shell.execute_reply":"2022-04-10T02:11:54.570128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Image size distribution\nfig, (ax1, ax2, ax3) = plt.subplots(3,1, figsize= (30,20))\n\nfig.suptitle(\"Image Size distribution on Species\", size = 30, weight = \"bold\")\n\nv1 = sns.violinplot(data = train, x = \"species\", y = \"widths\", ax=ax1, hue=\"labels\")\n\nax1.set_title(\"Width\", y = 0.97, size = 15, weight = \"bold\")\nax1.set_xlabel(\"\")\nax1.set_ylabel(\"Width\",size=13, weight = \"bold\")\nax1.set_xticklabels(ax1.get_xticklabels(),rotation = 45, ha = 'right')\n\nv2 = sns.violinplot(data = train, x = \"species\", y = \"heights\", ax = ax2, hue = \"labels\")\nax2.set_title(\"Height\", y = 0.97, size = 15, weight = \"bold\")\nax2.set_xlabel(\"\")\nax2.set_ylabel(\"Height\",size=13, weight = \"bold\")\nax2.set_xticklabels(ax2.get_xticklabels(),rotation = 45, ha = 'right')\n\nv3 = sns.violinplot(data = train, x = \"species\", y = \"dimensions\", ax = ax3, hue = \"labels\")\nax3.set_title(\"Dimensions\", y = 0.97, size = 15, weight = \"bold\")\nax3.set_xlabel(\"\")\nax3.set_ylabel(\"Dimensions\",size=13, weight = \"bold\")\nax3.set_xticklabels(ax3.get_xticklabels(),rotation = 45, ha = 'right')\n\nsns.despine(left=True, bottom=True)\nplt.subplots_adjust(\n    left=None, \n    bottom=None, \n    right=None, \n    top=0.90, \n    wspace=None, \n    hspace=0.5\n);","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-04-10T02:13:01.870871Z","iopub.execute_input":"2022-04-10T02:13:01.871731Z","iopub.status.idle":"2022-04-10T02:13:05.266555Z","shell.execute_reply.started":"2022-04-10T02:13:01.871679Z","shell.execute_reply":"2022-04-10T02:13:05.265776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I was a bit curious after seeing that the photos of gray whales all have the same size. So I print out some random pictures of this species below to take a look.","metadata":{}},{"cell_type":"code","source":"tmp = train[train[\"species\"] == \"gray_whale\"].reset_index(drop=True)\nplt.figure(figsize=(20,10))\nfor i, n in enumerate(random.sample(range(0,len(tmp)-1), 10)):\n    im = Image.open(tmp.loc[n,\"path\"])\n    plt.subplot(2,5,i+1)\n    plt.axis(\"off\")\n    plt.imshow(im)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T02:12:15.357563Z","iopub.execute_input":"2022-04-10T02:12:15.358341Z","iopub.status.idle":"2022-04-10T02:12:26.920410Z","shell.execute_reply.started":"2022-04-10T02:12:15.358296Z","shell.execute_reply":"2022-04-10T02:12:26.919793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(16,8))\nwhales = train[train[\"labels\"] == \"whale\"]\ndolphins = train[train[\"labels\"] == \"dolphin\"]\nassert (len(whales)+len(dolphins))==len(train)\n\nsns.countplot(\n    y = \"species\",\n    data = whales,\n    order = whales[\"species\"].value_counts().index,\n    ax = ax[0]\n)\n\nax[0].set_title('Whales')\nax[0].set_ylabel(None)\n\nsns.countplot(\n    y=\"species\", \n    data=dolphins, \n    ax=ax[1], \n    order = dolphins[\"species\"].value_counts().index,\n    palette=\"RdYlGn\"\n)\nax[1].set_title('Dolphins')\nax[1].set_ylabel(None)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:20:05.832162Z","iopub.execute_input":"2022-04-09T13:20:05.832774Z","iopub.status.idle":"2022-04-09T13:20:06.447033Z","shell.execute_reply.started":"2022-04-09T13:20:05.83274Z","shell.execute_reply":"2022-04-09T13:20:06.446051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pie chart\nplt.figure(figsize=(8,8))\nlabel_cnt = train.groupby([\"labels\"]).size().reset_index(name =\"counts\")\nplt.pie(label_cnt[\"counts\"], labels= label_cnt[\"labels\"],autopct='%1.1f%%',colors = sns.color_palette('Paired')[0:9],\n        shadow=True, startangle=90)\nplt.legend(loc = \"upper left\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:20:06.449827Z","iopub.execute_input":"2022-04-09T13:20:06.450168Z","iopub.status.idle":"2022-04-09T13:20:06.614814Z","shell.execute_reply.started":"2022-04-09T13:20:06.450123Z","shell.execute_reply":"2022-04-09T13:20:06.613689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In our dataset, the amount of photos varies by individials and by classes(whale/dolphin)","metadata":{}},{"cell_type":"markdown","source":"let's take a look at individuals now.\n","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:20:06.616735Z","iopub.execute_input":"2022-04-09T13:20:06.617111Z","iopub.status.idle":"2022-04-09T13:20:06.637676Z","shell.execute_reply.started":"2022-04-09T13:20:06.617064Z","shell.execute_reply":"2022-04-09T13:20:06.636466Z"}}},{"cell_type":"code","source":"individuals = train[\"individual_id\"].value_counts().head(10)\ntop_ten = pd.DataFrame({'individual_id':individuals.index, 'frequency' : individuals.values})\nplt.figure(figsize = (12,4))\nplt.bar(top_ten[\"individual_id\"], top_ten[\"frequency\"], width = 0.8, color=(0.2, 0.4, 0.6), zorder =4)\nplt.xticks(rotation=45)\nplt.yticks(rotation=45)\nplt.ylabel(\"frequency\")\nplt.xlabel(\"Individual Ids\")\nplt.title(\"Top 10 Individual Ids used by frequency\")\nplt.grid(visible = True, color ='grey',linestyle ='-', linewidth = 0.9,alpha = 0.2, zorder=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:20:06.63956Z","iopub.execute_input":"2022-04-09T13:20:06.640011Z","iopub.status.idle":"2022-04-09T13:20:06.927389Z","shell.execute_reply.started":"2022-04-09T13:20:06.63995Z","shell.execute_reply":"2022-04-09T13:20:06.926369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (20, 10))\nsns.kdeplot(np.log(train.loc[train['labels'] == 'whale']['individual_id'].value_counts()))\nsns.kdeplot(np.log(train.loc[train['labels'] == 'dolphin']['individual_id'].value_counts()))\nplt.xlabel(\"Scaled Counts for Individual Occurences\", fontsize=15)\nplt.ylabel(\"Density\", fontsize=15)\nplt.legend(labels = ['whale', 'dolphin'], prop= {'size': 20})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:20:06.928622Z","iopub.execute_input":"2022-04-09T13:20:06.929109Z","iopub.status.idle":"2022-04-09T13:20:07.371524Z","shell.execute_reply.started":"2022-04-09T13:20:06.929075Z","shell.execute_reply":"2022-04-09T13:20:07.370855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T13:20:07.372554Z","iopub.execute_input":"2022-04-09T13:20:07.373326Z","iopub.status.idle":"2022-04-09T13:20:07.386911Z","shell.execute_reply.started":"2022-04-09T13:20:07.373269Z","shell.execute_reply":"2022-04-09T13:20:07.385945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"References:\n\nhttps://www.kaggle.com/code/sahamed/eda-visualization-augmentation\n\nhttps://www.kaggle.com/code/kayvanshah/eda-whale-dolphin-identification\n\nReally grateful for these shared notebooks. I have learnt a lot and hope my version can be helpful to others as well.","metadata":{}}]}