{"cells":[{"metadata":{"_uuid":"aca8f0bc5ea4c8c1ead263d848239b1b49801044"},"cell_type":"markdown","source":"# Basic data exploration:\n\n1. distribution of images per whale\n1. viewing some images (same whale, different whale, 'new_whale')\n1. distribution of image resolution between train & test\n1. duplicate image analysis by perceptual hash"},{"metadata":{"trusted":true,"_uuid":"f023932152e00324c49788561b0c413477d52a51"},"cell_type":"code","source":"# used ideas from:\n# https://www.kaggle.com/mmrosenb/whales-an-exploration \n# https://www.kaggle.com/stehai/duplicate-images","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport os\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [14, 9]\n\nimport collections\nfrom PIL import Image\n\nDIR = \"../input\"\n\ntrain = pd.read_csv(os.path.join(DIR, \"train.csv\"))\ntest = pd.read_csv(os.path.join(DIR, \"sample_submission.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4207bec4457a2fe58b66aeace06cedfd92bc3835"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b91c069c462d8aa2c4d81c22bb7c514d70f9468f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1edd205d22a2d52e1896d665e4b04a042391c06"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a51f095f9e5efe38c2b37934d1146ddf5adf1ee"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44c58efa6a8043042a140c3b875eeda05b2ae59a"},"cell_type":"markdown","source":"## Distribution of images per whale is highly skewed.\n\n1. 2000+ whales have just one image\n2. Single whale with most images have 73 of them\n3. Images dsitribution:\n  1. almost 30% comes from whales with 4 or less images\n  1. almost 40% comes from 'new_whale' group\n  1. the rest 30% comes from whales with 5-73 images\n"},{"metadata":{"trusted":true,"_uuid":"584664f772898cf0da7d739e1b978c61720aa1ec"},"cell_type":"code","source":"train['Id'].value_counts()[:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c68f8bbafa40382b4372e03ec68098ae0eabcc0"},"cell_type":"code","source":"counted = train.groupby(\"Id\").count().rename(columns={\"Image\":\"image_count\"})\ncounted.loc[counted[\"image_count\"] > 80,'image_count'] = 80\nplt.figure()\nsns.countplot(data=counted, x=\"image_count\")\nplt.show()\nsns.distplot(counted[\"image_count\"], norm_hist=True, kde=False, hist_kws={'cumulative': True})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6f6edf11d9ac020a80807bcc9cd68485a299072"},"cell_type":"code","source":"image_count_for_whale = train.groupby(\"Id\", as_index=False).count().rename(columns={\"Image\":\"image_count\"})\nwhale_count_for_image_count = image_count_for_whale.groupby(\"image_count\", as_index=False).count().rename(columns={\"Id\":\"whale_count\"})\nwhale_count_for_image_count['image_total_count'] = whale_count_for_image_count['image_count'] * whale_count_for_image_count['whale_count']\nwhale_count_for_image_count['image_total_count_cum'] = whale_count_for_image_count[\"image_total_count\"].cumsum() / len(train)\nsns.barplot(x='image_count',y='image_total_count_cum',data=whale_count_for_image_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a610fe51e0e3fdb632f70d29938e8a2739adc555"},"cell_type":"code","source":"whale_count_for_image_count[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c719f2a56f7b70841be6d4214767b369212fe65"},"cell_type":"code","source":"whale_count_for_image_count[-3:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a70cb8f482d9bf4ac60cd27cc69e03251c7f7c0b"},"cell_type":"markdown","source":"# Let's see some images\n\n#### Some images of 'new_whale'"},{"metadata":{"trusted":true,"_uuid":"618c315b0065423fb0d89ab86d0feb2803991dc2"},"cell_type":"code","source":"fig = plt.figure(figsize = (20, 15))\nfor idx, img_name in enumerate(train[train['Id'] == 'new_whale']['Image'][:12]):\n    y = fig.add_subplot(3, 4, idx+1)\n    img = cv2.imread(os.path.join(DIR,\"train\",img_name))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    y.imshow(img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ab60c77896ace5a3644ec600445fbdf50be92b2"},"cell_type":"markdown","source":"    #### Now some pictures of whales that have just 1 image: quite a large variance in colors"},{"metadata":{"trusted":true,"_uuid":"a94a39aab2d80a58beb45cfd5d7db293dc8ccfc5"},"cell_type":"code","source":"single_whales = train['Id'].value_counts().index[-12:]\nfig = plt.figure(figsize = (20, 15))\n\nfor widx, whale in enumerate(single_whales):\n    for idx, img_name in enumerate(train[train['Id'] == whale]['Image'][:1]):\n        axes = widx + idx + 1\n        y = fig.add_subplot(3, 4, axes)\n        img = cv2.imread(os.path.join(DIR,\"train\",img_name))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        y.imshow(img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94022cc7175e539b97efb524e510c235645b433a"},"cell_type":"markdown","source":"#### Below: each row shows pictures of one whale. I think it's quite easy to at least see similiar appearence there"},{"metadata":{"trusted":true,"_uuid":"071a4b57ddd4de06f8bbab168de3903d76348562"},"cell_type":"code","source":"topN=5\ntop_whales = train['Id'].value_counts().index[1:1+topN]\nfig = plt.figure(figsize = (20, 5*topN))\n\nfor widx, whale in enumerate(top_whales):\n    for idx, img_name in enumerate(train[train['Id'] == whale]['Image'][:4]):\n        axes = widx*4 + idx+1\n        y = fig.add_subplot(topN, 4, axes)\n        img = cv2.imread(os.path.join(DIR,\"train\",img_name))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        y.imshow(img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"52423bb6b1fa39654f91e68260d3da0dc808fd7c"},"cell_type":"markdown","source":"# Resolutions\n\n#### over 7000 unique resolutions but 39 most popular cover ~45% images (both in train and in test)"},{"metadata":{"trusted":true,"_uuid":"c1107cc3865eeea714027c596f6b0235af596b28"},"cell_type":"code","source":"imageSizes_train = collections.Counter([Image.open(f'{DIR}/train/{filename}').size\n                        for filename in os.listdir(f\"{DIR}/train\")])\nimageSizes_test = collections.Counter([Image.open(f'{DIR}/test/{filename}').size\n                        for filename in os.listdir(f\"{DIR}/test\")])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"a616eed4c4179088ffc9be49065aa1e38fb94802"},"cell_type":"code","source":"def isdf(imageSizes):\n    imageSizeFrame = pd.DataFrame(list(imageSizes.most_common()),columns = [\"imageDim\",\"count\"])\n    imageSizeFrame['fraction'] = imageSizeFrame['count'] / sum(imageSizes.values())\n    imageSizeFrame['count_cum'] = imageSizeFrame['count'].cumsum()\n    imageSizeFrame['count_cum_fraction'] = imageSizeFrame['count_cum'] / sum(imageSizes.values())\n    return imageSizeFrame\n\ntrain_isdf = isdf(imageSizes_train)\ntrain_isdf['set'] = 'train'\ntest_isdf = isdf(imageSizes_test)\ntest_isdf['set'] = 'test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c715493bf9b602ef72982d64060fb9b641e542e"},"cell_type":"code","source":"isizes = train_isdf.merge(test_isdf, how=\"outer\", on=\"imageDim\")\nisizes['total_count'] = isizes['count_x'] + isizes['count_y']\ndims_order = isizes.sort_values('total_count', ascending=False)[['imageDim']]\nlen(dims_order)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49869b33cc46d88d9e9722cb7f3b0a973d94da68"},"cell_type":"code","source":"isizes = pd.concat([train_isdf, test_isdf])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49f78f7f469c440850aec5606a99724cf204f6c6"},"cell_type":"code","source":"isizes.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6729c47a67ab16f8be9094fc0001399c9a9e850"},"cell_type":"code","source":"isizes.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ca9e2049b12347b48ad8e17a02f35abc7b6b94c"},"cell_type":"code","source":"popularSizes = isizes[isizes['fraction'] > 0.002]\npopularSizes.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6321fe75e7bff19ddacc52938d7875a5bc9df80"},"cell_type":"code","source":"popularSizes.groupby('set').max()['count_cum_fraction']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"becfc1c181a6aefa5241176219f4b7b96c9825b2"},"cell_type":"code","source":"sns.barplot(x='imageDim',y='fraction',data = popularSizes, hue=\"set\")\n_ = plt.xticks(rotation=45)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f51271ec145eb7a10a69982e56b906d4d241e1a"},"cell_type":"markdown","source":"# Duplicates\n\n1. 1 duplicate in train set\n1. 3 duplicates between train and test\n1. totally different than in playground dataset: \n  1. [playground duplicates](https://www.kaggle.com/stehai/duplicate-images)\n  1. [solution that used duplicate information](https://www.kaggle.com/martinpiotte/whale-recognition-model-with-score-0-78563)\n"},{"metadata":{"trusted":true,"_uuid":"6abc22ffaacf6dadd6885f59fc740e37c3e58a6c"},"cell_type":"code","source":"import imagehash\n\ndef getImageMetaData(file_path):\n    with Image.open(file_path) as img:\n        img_hash = imagehash.phash(img)\n        return img.size, img.mode, img_hash\n\ndef get_img_duplicates_info(df, dataset):\n    \n    m = df.Image.apply(lambda x: getImageMetaData(os.path.join(DIR, dataset, x)))\n    df[\"Hash\"] = [str(i[2]) for i in m]\n    df[\"Shape\"] = [i[0] for i in m]\n    df[\"Mode\"] = [str(i[1]) for i in m]\n    df[\"Length\"] = df[\"Shape\"].apply(lambda x: x[0]*x[1])\n    df[\"Ratio\"] = df[\"Shape\"].apply(lambda x: x[0]/x[1])\n    df[\"New_Whale\"] = df.Id == \"new_whale\"\n    \n    \n    img_counts = df.Id.value_counts().to_dict()\n    df[\"Id_Count\"] = df.Id.apply(lambda x: img_counts[x])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07f65ec29510a8d97350a1e7f38bfe496b9f0d03"},"cell_type":"code","source":"train_dups = get_img_duplicates_info(train, \"train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"728d7a475e0047375157ed91d980f4adc81f2d62"},"cell_type":"code","source":"train_dups.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbc155b629bc48201377cea4c30f10fb0ca414cc"},"cell_type":"code","source":"t = train_dups.Hash.value_counts()\nt = t.loc[t>1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"720243e4b50b1d8471ad51ac52ff09b6a66936cd"},"cell_type":"code","source":"\"Duplicate hashes: {}\".format(len(t))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f28aa6f1f5596b6f6a72732e7daec3658d8f508"},"cell_type":"code","source":"t","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e361a02e8a97695e6e0c5338c20629f7964cd66e"},"cell_type":"code","source":"t.index[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dab68e572c801912d926ba92667d939e9f84899d"},"cell_type":"code","source":"train_dups[train_dups['Hash'] == t.index[0]].head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c660e94985ee932db2e94c664862837e83ced791"},"cell_type":"markdown","source":"#### The only duplicate found in train dataset comes from the same whale.\n"},{"metadata":{"trusted":true,"_uuid":"f32a2923c224e55ae0f21984d4f10c750d958396"},"cell_type":"code","source":"fig = plt.figure(figsize = (20, 10))\nfor idx, img_name in enumerate(train_dups[train_dups['Hash'] == t.index[0]]['Image'][:2]):\n    y = fig.add_subplot(3, 4, idx+1)\n    img = cv2.imread(os.path.join(DIR,\"train\",img_name))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    y.imshow(img)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"940dcec0ed506eaaf890fb70d6d0e5fb636a17b0"},"cell_type":"code","source":"test_dups = get_img_duplicates_info(test, \"test\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3fbe4fc5d5252dd9a4c40f045b69c3f6c9e0df7d"},"cell_type":"code","source":"test_d = test_dups.Hash.value_counts()\ntest_d = test_d.loc[test_d>1]\n\"Duplicate hashes in test: {}\".format(len(test_d))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb9dbd4f633a1b38c704cbbd74fbe87716b649f2"},"cell_type":"code","source":"common_hashes = test_dups.merge(train_dups, how=\"inner\", on=\"Hash\", suffixes=(\"_test\",\"_train\"))\ncommon_hashes.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9268d0067ebaedc39fd9b068e407ebfc0270891e"},"cell_type":"code","source":"\"Duplicate hashes between train and test: {}\".format(len(common_hashes))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e27d259d2f8a930b7a3975dda56d11f1ebde327"},"cell_type":"markdown","source":"### below each row shows images with the same pHash, left column from train, right from test"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"29b9e3f151cec163b112c31adca66414017a1e01"},"cell_type":"code","source":"fig = plt.figure(figsize = (10, 10))\n\nfor idx, images in enumerate(common_hashes[['Image_train','Image_test']].values):\n    y = fig.add_subplot(len(common_hashes),2, idx*2+1)\n    img = cv2.imread(os.path.join(DIR,\"train\",images[0]))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    y.imshow(img)\n\n    y = fig.add_subplot(len(common_hashes),2, idx*2+2)\n    img = cv2.imread(os.path.join(DIR,\"test\",images[1]))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    y.imshow(img)\n\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcefb56352f21b4f08e133c657c494afbce18228"},"cell_type":"code","source":"# train duplicates - to remove:\ntrain_to_remove = train_dups[train_dups['Hash'] == t.index[0]].drop_duplicates('Hash')[['Image']]\ntrain_to_remove.to_csv(\"train_remove.csv\",index=False)\ntrain_to_remove.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e123dae6b1d8a5c39c8898c5dafc324c851ac9e1"},"cell_type":"code","source":"# easy answers in test:\neasy_peasy = common_hashes[['Image_test','Id_train']]\neasy_peasy.to_csv(\"test_easy.csv\", index=False)\neasy_peasy.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}