{"cells": [{"metadata": {"_uuid": "38855a6eb624f9ea41c8c0b4992ae16a850cdf7c", "_cell_guid": "67fc0e35-93a4-407b-9870-586781caae64"}, "cell_type": "markdown", "source": ["## Carvana Challenge EDA"]}, {"metadata": {"_uuid": "2a2dc143a44c7bcc98fd506371cfca42d6aaca03", "_cell_guid": "a640f830-cfbd-4ad0-be82-610ab706a15e"}, "cell_type": "markdown", "source": ["This kernel is very much inspired by [this one](https://www.kaggle.com/vfdev5/data-visualization/notebook)"]}, {"metadata": {"_uuid": "361e2d72e4f31933df2a5ea10a51a150d8556e60", "_cell_guid": "f2ffe53b-dca5-4b94-ab52-a59c07081011"}, "cell_type": "markdown", "source": ["Add required imports"]}, {"metadata": {"_uuid": "06e13be28617d598cdd23bdf2b6bbe55d922ed31", "_cell_guid": "94653491-d72e-43ca-a549-9896bc0130a2", "collapsed": true}, "source": ["import pandas as pd\n", "import os\n", "\n", "from PIL import Image\n", "import cv2\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import matplotlib.image as mpimg\n", "import seaborn as sns\n", "\n", "%matplotlib inline"], "cell_type": "code", "outputs": [], "execution_count": 1}, {"metadata": {"_uuid": "4a1540ca29f7d9c1f0c4881699cb95af038074d0", "_cell_guid": "ea45c93b-a7bc-4997-a855-7200a550a6d4"}, "cell_type": "markdown", "source": ["Start by adding the helper files to the python path"]}, {"metadata": {"_uuid": "754e81101f460c960665e0e79935719941598b4b", "_cell_guid": "688fe656-2fa6-4837-ab96-3dc19a4c5655", "collapsed": true}, "source": ["def download_dataset():\n", "    \"\"\"\n", "    Downloads the dataset and return the input paths\n", "    :return: [train_data, test_data, metadata_csv, train_masks_csv, train_masks_data]\n", "    \"\"\"\n", "    competition_name = \"carvana-image-masking-challenge\"\n", "\n", "    destination_path = \"../input/\"\n", "    files = [\"train.zip\", \"test.zip\", \"metadata.csv.zip\", \"train_masks.csv.zip\", \"train_masks.zip\"]\n", "    datasets_path = [\"../input/train\", \"../input/test\", \"../input/metadata.csv\", \"../input/train_masks.csv\",\n", "                    \"../input/train_masks\"]\n", "    is_datasets_present = True\n", "\n", "    # If the folders already exists then the files may already be extracted\n", "    # This is a bit hacky but it's sufficient for our needs\n", "    for dir_path in datasets_path:\n", "        if not os.path.exists(dir_path):\n", "            is_datasets_present = False\n", "\n", "    if not is_datasets_present:\n", "        pass\n", "        #\n", "        # I usually download my dataset with my home made tool on my local PC:\n", "        # https://github.com/EKami/kaggle-data-downloader\n", "        # But here on Kaggle we already have all the dataset present.\n", "        #\n", "        # Put your Kaggle user name and password in a $KAGGLE_USER and $KAGGLE_PASSWD env vars respectively\n", "        # downloader = KaggleDataDownloader(os.getenv(\"KAGGLE_USER\"), os.getenv(\"KAGGLE_PASSWD\"), competition_name)\n", "        #\n", "        # for file in files:\n", "        #    output_path = downloader.download_dataset(file, destination_path)\n", "        #    downloader.decompress(output_path, destination_path)\n", "        #    os.remove(output_path)\n", "    else:\n", "        print(\"All datasets are present.\")\n", "        \n", "    return datasets_path"], "cell_type": "code", "outputs": [], "execution_count": 2}, {"metadata": {"_uuid": "c662b8a4474554ed655d8ef4e9af62d3bf68081d", "_cell_guid": "f2937477-50d8-4e2c-94e9-44aca95fba55"}, "cell_type": "markdown", "source": ["### Download the dataset"]}, {"metadata": {"outputHidden": false, "_uuid": "9e3e00ede9bcebdcdb5f75b53a9eff38771dd92c", "_cell_guid": "a0b21fd2-d0fc-4285-9af7-317d7d416139", "inputHidden": false}, "source": ["train_data, test_data, metadata_csv, train_masks_csv, train_masks_data = download_dataset()"], "cell_type": "code", "outputs": [], "execution_count": 3}, {"metadata": {"_uuid": "2e9c1bc434507b233b48569feacfdbc135f522ba", "_cell_guid": "6eb65221-7479-41f4-835f-4664298de82e"}, "cell_type": "markdown", "source": ["Show csv data informations:"]}, {"metadata": {"_uuid": "d66405cd470408b2630b6eb24f058e13108b1ebe", "_cell_guid": "0fda0137-366b-4e53-8ee5-6fde66071d1f", "collapsed": true}, "source": ["metadata_df = pd.read_csv(metadata_csv)\n", "train_masks_df = pd.read_csv(train_masks_csv)"], "cell_type": "code", "outputs": [], "execution_count": 4}, {"metadata": {"_uuid": "2a8397c47a25dc9d4bb823a2b9bbfa632ebac40b", "_cell_guid": "2742f303-a264-48e9-a188-09d20da4df81"}, "source": ["metadata_df.head()"], "cell_type": "code", "outputs": [], "execution_count": 5}, {"metadata": {"_uuid": "5bcf4536ba80ccfce08d6ee66ed74132411906af", "_cell_guid": "84aaf81e-451a-4924-a0a5-acd413a14634"}, "source": ["train_masks_df.head()"], "cell_type": "code", "outputs": [], "execution_count": 6}, {"metadata": {"_uuid": "e73aff1cb8112597cb4aa06f3f85424d577f2f97", "_cell_guid": "dab7c2b0-bba3-40d9-9bbd-b270c66e5b25"}, "cell_type": "markdown", "source": ["Count train/test data:"]}, {"metadata": {"_uuid": "ebaaa0a2dce1d9e99c7c1c5e4ec3adbd777c7f67", "_cell_guid": "2e39d27e-3f24-4f5a-b494-70825ec5550d"}, "source": ["train_files = os.listdir(train_data)\n", "test_files = os.listdir(test_data)\n", "train_masks_files = os.listdir(train_masks_data)\n", "print(\"Train files count: {}\\nTrain masks files count: {}\\nTest files count: {}\"\n", "      .format(len(train_files), len(train_masks_files), len(test_files)))"], "cell_type": "code", "outputs": [], "execution_count": 7}, {"metadata": {"_uuid": "2c93e8fb7649f8281848c7e068bcfb6f999bb02a", "_cell_guid": "9fb21b7c-d384-47b7-ad75-35f02ea6e42a"}, "cell_type": "markdown", "source": ["Get unique ids (each car has 16 images from different angles):"]}, {"metadata": {"outputHidden": false, "_uuid": "d75efe1bed7c77e7a1bf702cbbcdcc4768d0cc15", "_cell_guid": "bd37c03d-ba58-411f-871f-516d75409760", "inputHidden": false}, "source": ["train_ids = list(set(t.split(\"_\")[0] for t in train_files))\n", "masks_ids = list(set(t.split(\"_\")[0] for t in train_masks_files))\n", "test_ids = list(set(t.split(\"_\")[0] for t in test_files))\n", "\n", "print(\"Train files unique ids count: {}\\nTest files unique ids count: {}\".format(len(train_ids), len(test_ids)))\n", "assert len(train_ids) * 16 == len(train_files)\n", "assert len(test_ids) * 16 == len(test_files)"], "cell_type": "code", "outputs": [], "execution_count": 8}, {"metadata": {"_uuid": "5644dac770a9b2cc7931df29be3ee24505e83af9", "_cell_guid": "c27fc887-d80a-424e-a273-2df77799114a"}, "cell_type": "markdown", "source": ["## Utility functions\n", "\n", "Create a function to get car images path from their ID and another one to open the image and return a numpy matrix."]}, {"metadata": {"outputHidden": false, "_uuid": "8b47780786d81238027b9187a9fe5fc5bc8e43fd", "_cell_guid": "44453840-a057-4955-af3e-32756a40d1f8", "inputHidden": false, "collapsed": true}, "source": ["def get_car_image_files(car_image_id, get_mask=False):\n", "    if get_mask:\n", "        if car_image_id in masks_ids:\n", "            return [train_masks_data + \"/\" + s for s in train_masks_files if car_image_id in s]\n", "        else:\n", "            raise Exception(\"No mask with this ID found\")\n", "    elif car_image_id in train_ids:\n", "        return [train_data + \"/\" + s for s in train_files if car_image_id in s]\n", "    elif car_image_id in test_ids:\n", "        return [test_data + \"/\" + s for s in test_files if car_image_id in s]\n", "    raise Exception(\"No image with this ID found\")\n", "    \n", "def get_image_matrix(image_path):\n", "    img = Image.open(image_path)\n", "    return np.asarray(img, dtype=np.uint8)"], "cell_type": "code", "outputs": [], "execution_count": 9}, {"metadata": {"_uuid": "80b88a836c9f0e73f6d8cc56a4ad4bdfda096e12", "_cell_guid": "2952a6db-c4dd-4ac3-b2f4-8cdacb3917a3", "collapsed": true}, "cell_type": "markdown", "source": ["## Display a single car with its mask"]}, {"metadata": {"_uuid": "edc9013dc14edea6331bf624c739cb55094f706b", "_cell_guid": "95f8b5c6-f482-4f91-8c64-b98f46f145e6"}, "source": ["image_id = train_ids[0]\n", "\n", "plt.figure(figsize=(20, 20))\n", "img = get_image_matrix(get_car_image_files(image_id)[0])\n", "mask = get_image_matrix(get_car_image_files(image_id, True)[0])\n", "img_masked = cv2.bitwise_and(img, img, mask=mask)\n", "\n", "print(\"Image shape: {} | image type: {} | mask shape: {} | mask type: {}\"\n", "      .format(img.shape, img.dtype, mask.shape, mask.dtype) )\n", "\n", "plt.subplot(131)\n", "plt.imshow(img)\n", "plt.subplot(132)\n", "plt.imshow(mask)\n", "plt.subplot(133)\n", "plt.imshow(img_masked);"], "cell_type": "code", "outputs": [], "execution_count": 10}, {"metadata": {"_uuid": "72483825545e32a27001291fda710d1090fd35c1", "_cell_guid": "d82fc83b-94da-4f91-acfb-e6261b869b0a"}, "cell_type": "markdown", "source": ["## Check train mask DataFrame\n", "Check that the retrived mask indices corresponds to the ones in the csv file by checking the rle signatures"]}, {"metadata": {"_uuid": "cc5cfea5db84579458f740fdd0a4f5defe5f1f9b", "_cell_guid": "1f572e77-6550-4191-bd62-9f8c7891a290"}, "source": ["def rle_encode(mask_image):\n", "    pixels = mask_image.flatten()\n", "    # We avoid issues with '1' at the start or end (at the corners of \n", "    # the original image) by setting those pixels to '0' explicitly.\n", "    # We do not expect these to be non-zero for an accurate mask, \n", "    # so this should not harm the score.\n", "    pixels[0] = 0\n", "    pixels[-1] = 0\n", "    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n", "    runs[1::2] = runs[1::2] - runs[:-1:2]\n", "    return runs\n", "\n", "def rle_to_string(runs):\n", "    return ' '.join(str(x) for x in runs)\n", "\n", "file_name = get_car_image_files(image_id)[0].split(\"/\")[-1]\n", "mask_rle = train_masks_df[train_masks_df['img'] == file_name][\"rle_mask\"].iloc[0]\n", "assert rle_to_string(rle_encode(mask)) == mask_rle, \"Mask rle don't match\"\n", "print(\"Mask rle match!\")"], "cell_type": "code", "outputs": [], "execution_count": null}, {"metadata": {"_uuid": "28d0673bdae8300d747cfa833e711c11caf53a6d", "_cell_guid": "41b2d81f-ef5f-47f9-bbfb-42e39607a328"}, "cell_type": "markdown", "source": ["## Display few random cars from train dataset"]}, {"metadata": {"_uuid": "d59dbcc28794e1400fadac0dc2d4f0cef8dc325e", "_cell_guid": "27b5f126-107e-41a2-81c2-9d6020ef95ba", "scrolled": false}, "source": ["images_path = [get_car_image_files(id) for id in train_ids[:5]]\n", "\n", "for i, angles in enumerate(images_path):\n", "    _, axs = plt.subplots(4, 4, figsize=(14, 10))  #  figsize=(20, 20)\n", "    plt.rc('axes', grid=False)\n", "    plt.subplots_adjust(wspace=0, hspace=0)\n", "    axs = axs.ravel()\n", "    \n", "    for j, img_path in enumerate(angles):\n", "        img = mpimg.imread(img_path)\n", "        axs[j].axis('off')\n", "        axs[j].imshow(img);"], "cell_type": "code", "outputs": [], "execution_count": null}, {"metadata": {"_uuid": "6bf75ff05c503c93db167fa805f77baa52f8fc16", "_cell_guid": "36d3f4ee-35a5-4ed3-92c9-02c71e22bc3e"}, "cell_type": "markdown", "source": ["## How many different cars in the dataset"]}, {"metadata": {"_uuid": "5dd0648f2ed78a59a5195f6323383f922c28a589", "_cell_guid": "b4ec07a7-dec4-403a-a556-f2d931d6f36a", "collapsed": true}, "source": ["plt.figure(figsize=(12, 10))\n", "sns.countplot(y=\"make\", data=metadata_df);"], "cell_type": "code", "outputs": [], "execution_count": null}], "metadata": {"nteract": {"version": "0.2.0"}, "kernel_info": {"name": "rik_ssh_ekami_192_168_1_150_tensorflowremote"}, "kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "name": "python", "version": "3.6.1", "file_extension": ".py", "codemirror_mode": {"version": 3, "name": "ipython"}}}, "nbformat": 4, "nbformat_minor": 1}