{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"7d9e51ff-f9dc-f508-a9a6-228c04ea7f62"},"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d8352420-2231-441e-b98e-57cde48bbee1"},"outputs":[],"source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.mlab as mlab\nimport matplotlib.image as mpimg\nimport numpy as np\nfrom PIL import Image\nfrom scipy import sparse\nfrom scipy import ndimage\nfrom scipy.ndimage import gaussian_filter\nfrom skimage import data\nfrom skimage import img_as_float\nfrom skimage import morphology, measure\nfrom skimage.color import label2rgb\n\n%matplotlib inline"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"86789576-22e9-a994-2c09-a3f2d2ca73e9"},"outputs":[],"source":"jpeg_threshold = 0.15\n\n\ndef find_all_dots(raw_path, dot_path, expected_counts, jpeg_threshold=0.15, mask_dilation=5):\n    raw_image = mpimg.imread(raw_path)\n    dot_image = mpimg.imread(dot_path)\n    \n    # Convert to floats. Will save us headache later.\n    raw_image = raw_image.astype(float)\n    dot_image = dot_image.astype(float)\n    raw_image = raw_image / raw_image.max()\n    dot_image = dot_image / dot_image.max()\n\n    # Dot images have some black artifacts. Let's mask those out.\n    dot_norm = np.linalg.norm(dot_image, axis=2)\n    threshold = (dot_norm.max() - dot_norm.min()) * 0.005\n    initial_mask_1d = dot_norm <= threshold\n\n    for i in range(mask_dilation):\n        initial_mask_1d = ndimage.binary_dilation(initial_mask_1d)\n\n    # Broadcast to 3d for true image mask.\n    _, initial_mask = np.broadcast_arrays(dot_image, initial_mask_1d[..., None])\n\n    # Remove the background. Mask with the original image.\n    dot_diff = np.linalg.norm(dot_image - raw_image, axis=2)\n    dot_diff[initial_mask[:, :, 0]] = 0\n\n    # Remove jpeg artifact noise.\n    dot_diff[dot_diff < jpeg_threshold] = 0\n\n    def mask_image(m, mask, c):\n        mask_1d = mask < 0.01\n        _, mask_3d = np.broadcast_arrays(m, mask_1d[..., None])\n        m[mask_3d] = c\n\n    dots = dot_image.copy()\n    mask_image(dots, dot_diff, 0)\n\n    # Other ways to consider removing noise.\n    dot_diff_eroded = dot_diff > 0.01\n    dot_diff_eroded = morphology.closing(dot_diff_eroded)\n    dot_diff_eroded = ndimage.binary_erosion(dot_diff_eroded)\n    #dot_diff_eroded = ndimage.binary_erosion(dot_diff_eroded)\n    dots = dot_image.copy()\n    mask_image(dots, dot_diff_eroded, 0)\n\n    labeled_dots, label_count = morphology.label(dot_diff_eroded, return_num=True, connectivity=2)\n    print('Labels={}, Expected={}'.format(label_count, expected_counts))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"01bcc8d5-3e64-bc6d-681f-14fa3f760975"},"outputs":[],"source":"values = pd.read_csv('../input/Train/train.csv')\n\ndef check_counts(i, threshold=0.15, dilation=5):\n    expected_counts = sum(values.iloc[i][1:])\n    raw_path = '../input/Train/{}.jpg'.format(i)\n    dot_path = '../input/TrainDotted/{}.jpg'.format(i)\n    find_all_dots(raw_path, dot_path, expected_counts, threshold, dilation)\n    \nfor i in range(11):\n    check_counts(i)"},{"cell_type":"markdown","metadata":{"_cell_guid":"3fdc5ef2-71bf-cbcb-688b-37f913d0e09e"},"source":"Looks like we are doing fairly well on all but 3. One of them we missed two. And two of them are just completely off. We'll have to dig into #3 and #9 to find out why they found so many when they shouldn't have."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7c940d1d-3558-bbd7-e097-9ea49413b640"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}