{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"img_size = 300\ndef crop_image_from_gray(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n        mask = gray_img>tol\n        \n        check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0))].shape[0]\n        if (check_shape == 0): # image is too dark so that we crop out everything,\n            return img # return original image\n        else:\n            img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0))]\n            img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0))]\n            img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0))]\n            img = np.stack([img1,img2,img3],axis=-1)\n        return img\ndef preprocess_image(img_file):\n    img = cv2.imread(img_file)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img = crop_image_from_gray(img)\n    img = cv2.resize(img, (img_size,img_size))\n    return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Blurry image\nIn this competition, not all images have good quality. Ths blurry image (such as index 8) is difficult to extract features from it. Therefore, we may need to drop the blur images before we train the model.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig=plt.figure(figsize=(8, 8))\nimage_path = train_df.loc[8,'id_code']\nimage_id = train_df.loc[8,'diagnosis']\nimg = preprocess_image(f'../input/train_images/{image_path}.png')\nplt.title(f'diagnosis:{image_id} index:{8}')\nplt.imshow(img)\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Variance of the Laplacian\nTo automaticlly drop blurry image, we can employ the laplacian kernel and use its variance to represent image's amount of blur."},{"metadata":{"trusted":true},"cell_type":"code","source":"def isClear(img, threshold = 60):\n    return cv2.Laplacian(img, cv2.CV_64F).var() > threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_samples(df, columns=4, rows=3):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n    for i in range(columns*rows):\n        idx = np.random.randint(0, len(df)-1, 1)[0]\n        image_path = df.loc[idx,'id_code']\n        image_id = df.loc[idx,'diagnosis']\n        img = preprocess_image(f'../input/train_images/{image_path}.png')\n        fig.add_subplot(rows, columns, i+1)\n        plt.title(f'diagnosis:{image_id}   isclear:{isClear(img)}')\n        plt.imshow(img)\n    plt.tight_layout()\ndisplay_samples(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Drop the blurry image"},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nblur_list = []\nblur_list_id = []\nstart_time = time.time();\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    img = preprocess_image(f'../input/train_images/{image_id}.png')\n    if(not isClear(img)):\n        blur_list.append(i)\n        blur_list_id.append(image_id)\ntrain_df = train_df.drop(blur_list)\nprint(f'Cost: {time.time() - start_time}:.3% seconds');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Droped items:{len(blur_list_id)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Display the dropped images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_blurry_samples(df, img_id_list, columns=4, rows=3):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n    for i in range(columns*rows):\n        img = preprocess_image(f'../input/train_images/{img_id_list[i]}.png')\n        fig.add_subplot(rows, columns, i+1)\n        plt.title(f'index:{i}  isclear:{isClear(img)}')\n        plt.imshow(img)\n    plt.tight_layout()\ndisplay_blurry_samples(train_df, blur_list_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}