{"cells":[{"metadata":{"_uuid":"c5991b4aa3e589b34e4f3325afa931cc686f0657"},"cell_type":"markdown","source":"## Separating gray and color image\nThis kernel explores class imbalance between gray and color images. <br>\nThe proportion of \"new whale\" class is considerably different between gray and color images. <br>\nSo, I seperated training and test images according to gray and color images. <br>\nThis kernel could be useful for someone who want to make each model of gray and color images. <br>"},{"metadata":{"trusted":true,"_uuid":"2e36ea939769c7a48db2119f37b35cba58f98bf4"},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport cv2\nimport os\nfrom PIL import Image\n\nfrom matplotlib.pyplot import imshow\nfrom IPython.display import HTML","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a49b5b8e673568ddc481d7c1c3c07ee9f67b0321"},"cell_type":"code","source":"img_train_path = os.path.abspath('../input/train')\nimg_test_path = os.path.abspath('../input/test')\ncsv_train_path = os.path.abspath('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a054d13846727a4be008ea68b387932a554fd54f"},"cell_type":"code","source":"df = pd.read_csv(csv_train_path)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bf102a657ca675100e486dd9aa084aca796f741"},"cell_type":"code","source":"df['Image_path'] = [os.path.join(img_train_path,whale) for whale in df['Image']]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58113dc6b9245fb74735bcc79d50d85bc3fbb758"},"cell_type":"markdown","source":"### Training data"},{"metadata":{"trusted":true,"_uuid":"7feb94589489cea32402aa9a182a708d97957729"},"cell_type":"code","source":"img = Image.open(df['Image_path'][5])\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5daee02d33eeb0e62e2e964d1c82314442f5de39"},"cell_type":"code","source":"len(df)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"880a3930d3fe72f8b446aea9dd76587c868d6d27"},"cell_type":"code","source":"df.Id.value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c612798484fe6648ff0fe892d906e7de6521c1a4"},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"7109fbf94a54f358db68be2dccc2ea51ce64c5da"},"cell_type":"code","source":"# For color image, shape[1] = 3 \n# For gray image, shape[1] = null\ngray_flag = []\nfor i, row in tqdm(df.iterrows()) : \n    img = Image.open(row['Image_path'])\n    try : \n        if np.array(img.getdata()).shape[1] == 3 :\n            gray_flag.append(True)\n        else : \n            gray_flag.append(\"Error\")\n    except : \n        gray_flag.append(False)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe70541105c6e63edc669741db816131a6dc76fe"},"cell_type":"code","source":"df['is_color'] = gray_flag","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"badec9954985465c9e94515e572ddf1645c54a9d"},"cell_type":"code","source":"df['is_color'].value_counts().plot(kind='bar') \nplt.xticks(np.arange(2), ['Color', 'Gray'])\nplt.show()\nprint(df['is_color'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb764eacf296174ffb1bea4374f75c8813146d8f"},"cell_type":"markdown","source":"### Examples of color and gray image"},{"metadata":{"trusted":true,"_uuid":"56126cc33060cea2895192c1f176629383ec3e14"},"cell_type":"code","source":"color_file_path = df[df['is_color'] == True]['Image_path'][1:5]\ngray_file_path = df[df['is_color'] == False]['Image_path'][1:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5022340f99248977eaa4f639685ed455362d7b9"},"cell_type":"code","source":"for file in color_file_path :\n    img = Image.open(file)\n    plt.imshow(img)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43548347a195852c16e84923e02cdb2ee3817947"},"cell_type":"code","source":"for file in gray_file_path :\n    img = Image.open(file)\n    plt.imshow(img)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63eff3ec591a85557a103485feeeba99b6c663ac"},"cell_type":"markdown","source":"### Class distribution "},{"metadata":{"trusted":true,"_uuid":"9bcb8ac60ef36c94d02f25b99123e163ed0a4b3b"},"cell_type":"code","source":"df[df['is_color'] == True].Id.value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49cfee7b75f93aff1695d73904488e1bfb31fb51"},"cell_type":"code","source":"df[df['is_color'] == False].Id.value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"239889f7448bfc6f599b341148b26ba0031c97e6"},"cell_type":"code","source":"# Proportion of new whale is different\ncolor_df = df[df['is_color'] == True]\ngray_df = df[df['is_color'] == False]\n\nprint(\"Color images\")\nprint(len(color_df[color_df['Id'] == \"new_whale\"]) , '/' , len(color_df))\nprint(float(len(color_df[color_df['Id'] == \"new_whale\"]) / len(color_df)))\n\nprint(\"Gray images\")\nprint(len(gray_df[gray_df['Id'] == \"new_whale\"]) , '/' , len(gray_df))\nprint(float(len(gray_df[gray_df['Id'] == \"new_whale\"]) / len(gray_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca9018ef80472f35d0e2adc775536aa93ad49207"},"cell_type":"code","source":"df.loc [df['Id'] == \"new_whale\", \"new_whale\"] = \"new_whale\" \ndf.loc [df['Id'] != \"new_whale\", \"new_whale\"] = \"other\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b37966cd1572ad1f711188be489b49319d3b64b"},"cell_type":"code","source":"freq_df = df.groupby([\"is_color\",\"new_whale\"])[\"new_whale\"].count().unstack(\"new_whale\")\nfreq_df.plot(kind='bar', figsize=(10,5))\n\nplt.title(\"New whale x Color \")\nplt.xticks(np.arange(2), ['Gray', 'Color'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84d9f63964a9e314f051678064df6cc8f4f76ed2"},"cell_type":"code","source":"color_df.to_csv(\"train_color.csv\", index=False)\ngray_df.to_csv(\"train_gray.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e9742b6f346e93087aef1e3d730c8bd4934872a"},"cell_type":"markdown","source":"### Test data"},{"metadata":{"trusted":true,"_uuid":"320f7114fd6bfb6ffd0ab99ddecce825869ef83d"},"cell_type":"code","source":"submission = pd.read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00dd40263cfc7c587b2147d6c64d3d1db7a32918"},"cell_type":"code","source":"test_image_paths= [os.path.join(img_test_path, whale) for whale in submission['Image']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8386eccca534ec012c92943fa03bf69bc8bb922b"},"cell_type":"code","source":"# For color image, shape[1] = 3 w\n# For gray image, shape[1] = null\ngray_flag_test = []\nfor i, path in tqdm(enumerate(test_image_paths)) : \n    img = Image.open(path)\n    try : \n        if np.array(img.getdata()).shape[1] == 3 :\n            gray_flag_test.append(True)\n        else : \n            gray_flag_test.append(\"Error\")\n    except : \n        gray_flag_test.append(False)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41f2fea8eca223f63e9f9e0249254fb95a5a398e"},"cell_type":"code","source":"test_df = submission.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"951c6458a91156601e5a0c2492ac19dd07d24f62"},"cell_type":"code","source":"test_df['is_color'] = gray_flag_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80e7f2f1b4ba4edba3b0ae97446ec123b6fb3548"},"cell_type":"code","source":"freq_df = test_df.groupby([\"is_color\"])[\"is_color\"].count()\nfreq_df.plot(kind='bar', figsize=(10,5))\nplt.xticks(np.arange(2), ['Gray', 'Color'])\nplt.title(\"Color\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"349d4bd102ad5e49f81069e5fb36358217792fc1"},"cell_type":"code","source":"color_test_df = test_df[test_df['is_color'] == True]\ngrah_test_df = test_df[test_df['is_color'] == False]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fa3357ff54ca2b0fbe227330e2ed4289c4ab4ea"},"cell_type":"code","source":"color_test_df.to_csv(\"test_color.csv\", index=False)\ngrah_test_df.to_csv(\"test_gray.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}