{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "1d37e6af-04fe-9d0e-dad8-f6b2d69e6f95"
      },
      "source": [
        "Understanding Philipp Schmidt notebook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "113cfa2e-f42f-caf4-2b45-9408a934f47a"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import matplotlib.pyplot as plt\n",
        "#from skimage.io import imread, imshow\n",
        "from scipy import misc\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input/train\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "43ed04db-b841-6ada-6029-d95b3f557a0e"
      },
      "outputs": [],
      "source": [
        "from glob import glob\n",
        "total_cervix_images = []\n",
        "\n",
        "for path in glob(\"../input/train/*\"):\n",
        "    cervix_images = sorted(glob(path+\"/*\"))\n",
        "    total_cervix_images += cervix_images\n",
        "    \n",
        "total_cervix_images = pd.DataFrame({'imagepath':total_cervix_images})\n",
        "total_cervix_images['filetype'] = total_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\n",
        "total_cervix_images['type'] = total_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\n",
        "\n",
        "total_cervix_images.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "0ad250b8-84c6-f78e-d2ed-bcf31f6ddab9"
      },
      "outputs": [],
      "source": [
        "type_aggregation = total_cervix_images.groupby(['type','filetype']).agg('count')\n",
        "type_aggregation_p = type_aggregation.apply(lambda row: 1.0*row['imagepath']/total_cervix_images.shape[0], axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "985cc058-00d3-c5a9-12ef-6146732b8fd9"
      },
      "outputs": [],
      "source": [
        "fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10,8))\n",
        "type_aggregation.plot.barh(ax=axes[0])\n",
        "axes[0].set_xlabel(\"image count\")\n",
        "type_aggregation_p.plot.barh(ax=axes[1])\n",
        "axes[1].set_xlabel(\"training size fraction\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aa9a93dd-da3a-bc69-9f6e-d52f5a4e2b83"
      },
      "outputs": [],
      "source": [
        "fig = plt.figure(figsize= (12,8))\n",
        "i = 1\n",
        "\n",
        "for t in sorted(total_cervix_images['type'].unique()):\n",
        "    ax = fig.add_subplot(1,3,i)    \n",
        "    i += 1\n",
        "    f = total_cervix_images[total_cervix_images['type']==t]['imagepath'].values[0]\n",
        "    plt.imshow(plt.imread(f))\n",
        "    plt.title(\"Sample cervix {}\".format(t))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "543fd8c0-8d75-34e2-716f-1eb6ce234328"
      },
      "outputs": [],
      "source": [
        "from collections import defaultdict\n",
        "images = defaultdict(list)\n",
        "\n",
        "for t in total_cervix_images['type'].unique():\n",
        "    sample_counter = 0\n",
        "    for _,row in total_cervix_images[total_cervix_images['type']==t].iterrows():\n",
        "        try:\n",
        "            print(row.imagepath.values[0])\n",
        "            image = misc.imread(row.imagepath)\n",
        "            images[t].append(img)\n",
        "            sample_counter += 1\n",
        "        except:\n",
        "            print(\"imread failed for image {}\".format(row.imagepath))\n",
        "            print(\"sample counter = {}\".format(sample_counter))\n",
        "        if sample_counter > 35:\n",
        "            break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "fb0d558b-f444-74fd-dd9c-3a74d5aff6f2"
      },
      "outputs": [],
      "source": [
        "dfs = []\n",
        "for t in all_cervix_images['type'].unique():\n",
        "    t_ = pd.DataFrame(\n",
        "        {\n",
        "            'nrows': list(map(lambda i: i.shape[0], images[t])),\n",
        "            'ncols': list(map(lambda i: i.shape[1], images[t])),\n",
        "            'nchans': list(map(lambda i: i.shape[2], images[t])),\n",
        "            'type': t\n",
        "        }\n",
        "    )\n",
        "    dfs.append(t_)\n",
        "\n",
        "shapes_df = pd.concat(dfs, axis=0)\n",
        "shapes_df_grouped = shapes_df.groupby(by=['nchans', 'ncols', 'nrows', 'type']).size().reset_index().sort_values(['type', 0], ascending=False)\n",
        "shapes_df_grouped"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2dd64cd1-43f3-dd83-cbca-e9aa657b683c"
      },
      "outputs": [],
      "source": ""
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "de6e9e3a-836c-0cf5-c83f-40bd46dbfb45"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}