{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"040abf30-9f01-8300-f7ae-3671d783287a"},"source":"Understanding Philipp Schmidt notebook"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"95b52bf0-310a-6ca0-30a5-892de024e2b0"},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n#from skimage.io import imread, imshow\nfrom scipy import misc\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input/train\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"785e7d51-f363-5402-071a-7f02615cc392"},"outputs":[],"source":"from glob import glob\ntotal_cervix_images = []\n\nfor path in glob(\"../input/train/*\"):\n    cervix_images = sorted(glob(path+\"/*\"))\n    total_cervix_images += cervix_images\n    \ntotal_cervix_images = pd.DataFrame({'imagepath':total_cervix_images})\ntotal_cervix_images['filetype'] = total_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\ntotal_cervix_images['type'] = total_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\n\ntotal_cervix_images.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9f24db56-3db6-9974-9f4d-22b98d4033f8"},"outputs":[],"source":"type_aggregation = total_cervix_images.groupby(['type','filetype']).agg('count')\ntype_aggregation_p = type_aggregation.apply(lambda row: 1.0*row['imagepath']/total_cervix_images.shape[0], axis=1)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"abe838c4-ebff-721f-20f8-ab04e4c0c7cb"},"outputs":[],"source":"fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10,8))\ntype_aggregation.plot.barh(ax=axes[0])\naxes[0].set_xlabel(\"image count\")\ntype_aggregation_p.plot.barh(ax=axes[1])\naxes[1].set_xlabel(\"training size fraction\")"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2c217db1-b791-25cf-713c-453ef91fe3af"},"outputs":[],"source":"fig = plt.figure(figsize= (12,8))\ni = 1\n\nfor t in sorted(total_cervix_images['type'].unique()):\n    ax = fig.add_subplot(1,3,i)    \n    i += 1\n    f = total_cervix_images[total_cervix_images['type']==t]['imagepath'].values[0]\n    plt.imshow(plt.imread(f))\n    plt.title(\"Sample cervix {}\".format(t))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"86519e97-33a4-a982-6087-95142aa2b01d"},"outputs":[],"source":"from collections import defaultdict\nimages = defaultdict(list)\n\nfor t in total_cervix_images['type'].unique():\n    sample_counter = 0\n    for _,row in total_cervix_images[total_cervix_images['type']==t].iterrows():\n        try:\n            image = misc.imread(row.imagepath)\n            images[t].append(image)\n            sample_counter += 1\n        except:\n            print(\"imread failed for image {}\".format(row.imagepath))\n            print(\"sample counter = {}\".format(sample_counter))\n        if sample_counter > 35:\n            break"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d68b82da-0288-0467-0fe9-8fc4726499f1"},"outputs":[],"source":"dfs = []\nfor t in total_cervix_images['type'].unique():\n    t_ = pd.DataFrame(\n        {\n            'nrows': list(map(lambda i: i.shape[0], images[t])),\n            'ncols': list(map(lambda i: i.shape[1], images[t])),\n            'nchans': list(map(lambda i: i.shape[2], images[t])),\n            'type': t\n        }\n    )\n    dfs.append(t_)\n\nshapes_df = pd.concat(dfs, axis=0)\nshapes_df_grouped = shapes_df.groupby(by=['nchans', 'ncols', 'nrows', 'type']).size().reset_index().sort_values(['type', 0], ascending=False)\nshapes_df_grouped"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"513d03e1-72f6-524f-6f7d-8bd208445a76"},"outputs":[],"source":"import cv2"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"00221b31-87c8-19be-29e9-01d06edd0b85"},"outputs":[],"source":"def transform_image(img, rescaled_dim, to_gray=False):\n    resized = cv2.resize(img, (rescaled_dim,rescaled_dim), cv2.INTER_LINEAR)\n    \n    if to_gray:\n        cv2.cvtColor(resized, cv2.COLOR2GRAY).astype('float')\n    else:\n        resized = resized.astype('float')\n        \n    normalized = cv2.normalize(resized, None, 0.0, 1.0, cv2.NORM_MINMAX)\n    timg = normalized.reshape(1,np.prod(normalized.shape))\n    return timg/np.linalg.norm(timg)\n    "},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2d0c3d35-e077-ff8c-190e-d5cfcbb0bdb6"},"outputs":[],"source":"rescaled_dim = 100\nall_images = []\nall_image_types = []"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5da34bc9-574e-2c3c-9b6f-5ccbef34c977"},"outputs":[],"source":"for t in total_cervix_images['type'].unique():\n    all_images = all_images + images[t]\n    all_image_types = all_image_types + len(images[t])*[t]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5b3b3b1a-8608-53e0-3013-62114e514322"},"outputs":[],"source":"gray_all_images_as_vecs = [transform_image(img, rescaled_dim) for img in all_images]\ngray_imgs_mat = np.asarray(gray_all_images_as_vecs).squeeze()\nall_image_types = np.array(all_image_types)\ngray_imgs_mat.shape, all_image_types.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b4202b70-f92b-9197-ce13-ad6b37ef601b"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}