{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "13c8434a-aa38-8f6b-d98f-c19cc8791296"
      },
      "source": [
        "# Possibly corrupted images\n",
        "\n",
        "Found an image that can cause some troubles in training phase : '1339', 'Type_1'\n",
        "\n",
        "- PIL opens it as an array of objects \n",
        "- OpenCV opens normally but gives a warning: `Premature end of JPEG file`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0eca4da7-b1e2-e162-170c-33c73ccfb18b"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "from glob import glob\n",
        "\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "\n",
        "TRAIN_DATA = \"../input/train\"\n",
        "type_1_files = glob(os.path.join(TRAIN_DATA, \"Type_1\", \"*.jpg\"))\n",
        "type_1_ids = np.array([s[len(os.path.join(TRAIN_DATA, \"Type_1\"))+1:-4] for s in type_1_files])\n",
        "type_2_files = glob(os.path.join(TRAIN_DATA, \"Type_2\", \"*.jpg\"))\n",
        "type_2_ids = np.array([s[len(os.path.join(TRAIN_DATA, \"Type_2\"))+1:-4] for s in type_2_files])\n",
        "type_3_files = glob(os.path.join(TRAIN_DATA, \"Type_3\", \"*.jpg\"))\n",
        "type_3_ids = np.array([s[len(os.path.join(TRAIN_DATA, \"Type_3\"))+1:-4] for s in type_3_files])\n",
        "\n",
        "print(len(type_1_files), len(type_2_files), len(type_3_files))\n",
        "print(\"Type 1\", type_1_ids[:10])\n",
        "print(\"Type 2\", type_2_ids[:10])\n",
        "print(\"Type 3\", type_3_ids[:10])\n",
        "\n",
        "TEST_DATA = \"../input/test\"\n",
        "test_files = glob(os.path.join(TEST_DATA, \"*.jpg\"))\n",
        "test_ids = np.array([s[len(TEST_DATA)+1:-4] for s in test_files])\n",
        "print(len(test_ids))\n",
        "print(test_ids[:10])\n",
        "\n",
        "ADDITIONAL_DATA = \"../input/additional\"\n",
        "additional_type_1_files = glob(os.path.join(ADDITIONAL_DATA, \"Type_1\", \"*.jpg\"))\n",
        "additional_type_1_ids = np.array([s[len(os.path.join(ADDITIONAL_DATA, \"Type_1\"))+1:-4] for s in additional_type_1_files])\n",
        "additional_type_2_files = glob(os.path.join(ADDITIONAL_DATA, \"Type_2\", \"*.jpg\"))\n",
        "additional_type_2_ids = np.array([s[len(os.path.join(ADDITIONAL_DATA, \"Type_2\"))+1:-4] for s in additional_type_2_files])\n",
        "additional_type_3_files = glob(os.path.join(ADDITIONAL_DATA, \"Type_3\", \"*.jpg\"))\n",
        "additional_type_3_ids = np.array([s[len(os.path.join(ADDITIONAL_DATA, \"Type_3\"))+1:-4] for s in additional_type_3_files])\n",
        "\n",
        "print(len(additional_type_1_files), len(additional_type_2_files), len(additional_type_2_files))\n",
        "print(\"Type 1\", additional_type_1_ids[:10])\n",
        "print(\"Type 2\", additional_type_2_ids[:10])\n",
        "print(\"Type 3\", additional_type_3_ids[:10])\n",
        "\n",
        "\n",
        "def get_filename(image_id, image_type):\n",
        "    \"\"\"\n",
        "    Method to get image file path from its id and type   \n",
        "    \"\"\"\n",
        "    if image_type == \"Type_1\" or \\\n",
        "        image_type == \"Type_2\" or \\\n",
        "        image_type == \"Type_3\":\n",
        "        data_path = os.path.join(TRAIN_DATA, image_type)\n",
        "    elif image_type == \"Test\":\n",
        "        data_path = TEST_DATA\n",
        "    elif image_type == \"AType_1\" or \\\n",
        "          image_type == \"AType_2\" or \\\n",
        "          image_type == \"AType_3\":\n",
        "        data_path = os.path.join(ADDITIONAL_DATA, image_type)\n",
        "    else:\n",
        "        raise Exception(\"Image type '%s' is not recognized\" % image_type)\n",
        "\n",
        "    ext = 'jpg'\n",
        "    return os.path.join(data_path, \"{}.{}\".format(image_id, ext))\n",
        "\n",
        "def _get_image_data_pil(image_id, image_type):\n",
        "    \"\"\"\n",
        "    Method to get image data as np.array specifying image id and type\n",
        "    \"\"\"\n",
        "    fname = get_filename(image_id, image_type)\n",
        "    try:\n",
        "        img = Image.open(fname)\n",
        "    except Exception as e:\n",
        "        assert False, \"Failed to read image : %s, %s. Error message: %s\" % (image_id, image_type, e)\n",
        "    return np.asarray(img)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4e37e7b3-44df-f61c-1d0b-4a637facce48"
      },
      "outputs": [],
      "source": [
        "#type_ids = (type_1_ids, type_2_ids, type_3_ids, test_ids)\n",
        "#image_types = [\"Type_1\", \"Type_2\", \"Type_3\", \"Test\"]\n",
        "\n",
        "#corrupted_image_id_type_list = []\n",
        "#for ids, image_type in zip(type_ids, image_types):\n",
        "#    for image_id in ids:\n",
        "#        img = _get_image_data_pil(image_id, image_type)\n",
        "#        if img.dtype.kind is not 'u':\n",
        "#            corrupted_image_id_type_list.append((image_id, image_type))\n",
        "# print(len(corrupted_image_id_type_list), corrupted_image_id_type_list[:10])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "1c4b5884-a21b-e771-0101-5fdc3c84fc60"
      },
      "source": [
        "There is only one possible corrupted image : \n",
        "'1339', 'Type_1'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c621c4da-478b-7e82-f4b0-4e6e6f66ecec"
      },
      "outputs": [],
      "source": [
        "corrupted_image_id_type_list = [('1339', 'Type_1')]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c76d93b6-c56b-c95d-fd84-59d1c28908fc"
      },
      "outputs": [],
      "source": [
        "import cv2\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def _get_image_data_opencv(image_id, image_type):\n",
        "    \"\"\"\n",
        "    Method to get image data as np.array specifying image id and type\n",
        "    \"\"\"\n",
        "    fname = get_filename(image_id, image_type)\n",
        "    img = cv2.imread(fname)\n",
        "    assert img is not None, \"Failed to read image : %s, %s\" % (image_id, image_type)\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "    return img"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "57134f95-fffa-ea4d-d1fd-2daba87b6b88"
      },
      "outputs": [],
      "source": [
        "n = 5\n",
        "for i in range(len(corrupted_image_id_type_list)):\n",
        "    image_id, image_type = corrupted_image_id_type_list[i]\n",
        "    img = _get_image_data_opencv(image_id, image_type)\n",
        "    if i % n == 0:\n",
        "        plt.figure(figsize=(20,10))\n",
        "        plt.suptitle(\"Possibly corrupted images\")\n",
        "    plt.subplot(1,n,i % n + 1)\n",
        "    plt.imshow(img)\n",
        "    plt.title(\"%s / %s\" % (image_id, image_type))   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "127667f6-24dd-99e8-6e46-8c10c2df25b9"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}