{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "a24e0671-8d6e-4f13-9e2e-29716da3cd01"
      },
      "source": [
        "## Data exploration of CCS Kaggle competition\n",
        "https://www.kaggle.com/c/intel-mobileodt-cervical-cancer-screening\n",
        "We see cv2 is better than PIL in terms of reading jpg in."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a38ab3c8-6639-e0d4-f113-e240534e3e92"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import PIL\n",
        "import matplotlib.pyplot as plt\n",
        "import keras"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "082602b1-880d-ea81-6a4a-ec06127c397f"
      },
      "outputs": [],
      "source": [
        "def read_jpg(path):\n",
        "    return np.asarray(PIL.Image.open(path),dtype=np.uint8)    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f20adca8-0f99-f284-f358-df62665c34a0"
      },
      "outputs": [],
      "source": [
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "757c23ec-cec3-b435-d42a-2ad735441750"
      },
      "outputs": [],
      "source": [
        "tr_ty1 = [x for x in os.listdir('../input/train/Type_1') if '.jpg' in x]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "6c8316ba-3ec9-c178-9c50-ff3f7de1c89d"
      },
      "source": [
        "PIL failed in reading one of the jpg's in train/Type_1."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "86195b7e-edcc-ccd6-da86-fd2ffc1f07e3"
      },
      "outputs": [],
      "source": [
        "for j,f in enumerate(tr_ty1):\n",
        "    img = read_jpg(path='../input/train/Type_1/'+f)\n",
        "    print(j,f,img.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e4a1addf-7f18-eadb-7a9c-abece9d1c6ac"
      },
      "outputs": [],
      "source": [
        "# Something wrong /w this particular image\n",
        "read_jpg('../input/train/Type_1/'+tr_ty1[60])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2fc3af1b-be37-91b9-e775-92baf71ffa0b"
      },
      "outputs": [],
      "source": [
        "read_jpg('../input/train/Type_1/'+tr_ty1[61])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6232e74d-b501-f879-5f03-39ab35cf3f8d"
      },
      "outputs": [],
      "source": [
        "tr_ty1[61]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "67b10cea-2282-869e-3b24-38c5bd011aab"
      },
      "outputs": [],
      "source": [
        "import cv2\n",
        "def get_image_data(fname):\n",
        "    img = cv2.imread(fname)\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "    return img"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bac6abec-ad09-4d98-b7bd-13bf3cfe4bf3"
      },
      "outputs": [],
      "source": [
        "img = get_image_data('../input/train/Type_1/'+tr_ty1[61])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b2b62e29-1293-49c0-daa9-1e5a4545585c"
      },
      "outputs": [],
      "source": [
        "plt.imshow(img)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "47578d79-3feb-6564-1ca8-a3c49fff785b"
      },
      "outputs": [],
      "source": [
        "for j,f in enumerate(tr_ty1):\n",
        "    img = get_image_data(fname='../input/train/Type_1/'+f)\n",
        "    print(j,f,img.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b632ed9f-2bc8-e279-3269-95357dfc22ad"
      },
      "outputs": [],
      "source": [
        "# The following is from previous version of testing notebook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "24971687-83da-7a28-660a-0619d178ab4c"
      },
      "outputs": [],
      "source": [
        "len([x for x in os.listdir('../input/train/Type_3') if '.jpg' in x])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4b28422a-4173-1203-ee96-875e3adb11b3"
      },
      "outputs": [],
      "source": [
        "[x for x in os.listdir('../input/train/Type_1') if '.jpg' not in x] + \\\n",
        "[x for x in os.listdir('../input/train/Type_2') if '.jpg' not in x] + \\\n",
        "[x for x in os.listdir('../input/train/Type_3') if '.jpg' not in x]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c073f801-7ba4-b7c6-6f1d-3722dc8d2b9f"
      },
      "outputs": [],
      "source": [
        "len([x for x in os.listdir('../input/additional/Type_1') if '.jpg' in x]), \\\n",
        "len([x for x in os.listdir('../input/additional/Type_2') if '.jpg' in x]), \\\n",
        "len([x for x in os.listdir('../input/additional/Type_3') if '.jpg' in x])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "6622fa4e-0000-d591-ae6d-49f922f45074"
      },
      "source": [
        "Therefore, there are 250+1191 type_1, 781+3567 type_2, and 450+1976 type_3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7b3d6074-7143-9cec-a49a-7b055af55244"
      },
      "outputs": [],
      "source": [
        "[x for x in os.listdir('../input/additional/Type_1') if '.jpg' not in x] + \\\n",
        "[x for x in os.listdir('../input/additional/Type_2') if '.jpg' not in x] + \\\n",
        "[x for x in os.listdir('../input/additional/Type_3') if '.jpg' not in x]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "eeec7424-f5b0-1837-7a4f-8787ae0ca36b"
      },
      "outputs": [],
      "source": [
        "test_files = os.listdir('../input/test/')\n",
        "len(test_files)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cd4f9c5e-c30f-0044-ffa8-3665e445e79d"
      },
      "outputs": [],
      "source": [
        "[x for x in test_files if 'jpg' not in x]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7cc6ca17-9c87-0f77-25e1-ba9a981b6477"
      },
      "outputs": [],
      "source": [
        "#plt.imshow('../input/test/0.jpg') # Doesn't work directly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3d3ed5fd-b10b-3105-4ad2-8aefe9d3c5b4"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "650b45aa-1741-5679-86f2-ab58e7d377e9"
      },
      "outputs": [],
      "source": [
        "img = read_jpg('../input/test/0.jpg')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6ef8a19c-519d-825e-c9b8-85cd7b148054"
      },
      "outputs": [],
      "source": [
        "plt.imshow(img)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bfc1c318-f680-9289-898b-0876439aa2b8"
      },
      "outputs": [],
      "source": [
        "img.shape\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "191fff8e-6155-cfaa-4aef-d5ec63446219"
      },
      "outputs": [],
      "source": [
        "plt.imshow(read_jpg('../input/test/1.jpg'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ad2172dd-6be2-6ae2-9360-47a83a478d0e"
      },
      "outputs": [],
      "source": [
        "plt.imshow(read_jpg('../input/test/2.jpg'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7335051f-b600-c7bb-bb4c-00a8936960a6"
      },
      "outputs": [],
      "source": [
        "plt.imshow(read_jpg('../input/test/3.jpg'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "49c3f788-f155-d019-3864-6d59a0942cd0"
      },
      "outputs": [],
      "source": [
        "subm = pd.read_csv('../input/sample_submission.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ec6eff36-0a6d-483d-7a5b-4fe13bfb46b7"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0e05b300-3d04-fafe-1bb9-657232294242"
      },
      "outputs": [],
      "source": [
        "subm.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "32e8c916-61fd-2b22-ce02-7bc6ed67c2c4"
      },
      "outputs": [],
      "source": [
        "subm.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6246d480-73ba-d914-6910-dbf3bcecf1da"
      },
      "outputs": [],
      "source": [
        "0.168805\t+ 0.527346\t+ 0.303849"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c54f5f1a-ac60-46c0-5894-a82c89505884"
      },
      "outputs": [],
      "source": [
        "subm.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "57e30744-f9ee-b3a6-675b-704ce2dc155e"
      },
      "outputs": [],
      "source": [
        "os.listdir('../config')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f7d61d91-286a-01e6-e094-a47b2f526ed5"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ba77a3fb-82c9-8737-3f68-efae6902516e"
      },
      "outputs": [],
      "source": [
        "os.listdir('../lib')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "896436a4-6a73-76c4-16c1-697c48437dbb"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "10126a9d-48bf-e365-ce33-dc7f3647999f"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "557b16d8-2881-0ca9-14e1-58e85ef061e7"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d72b66e2-5330-b4a8-cac6-32d924daea2e"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e4b77a5a-281d-f733-6ec8-2ed24e7ae384"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}