{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b2c36c81-3bb2-0554-7590-29683c95e7d1"},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom skimage.io import imread, imshow\nimport cv2\n\n%matplotlib inline\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input/train\"]).decode(\"utf8\"))\nfrom glob import glob\nbasepath = '../input/train/'"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f6a4b3ab-c2eb-9f88-4658-3a1a2356ee92"},"outputs":[],"source":" def getCervixImagesTraining():\n    \n\n\n    all_cervix_images = []\n\n    for path in sorted(glob(basepath + \"*\")):\n        #print(path)\n        cervix_type = path.split(\"/\")[-1]\n        # print(cervix_type)\n        cervix_images = sorted(glob(basepath + cervix_type + \"/*\"))\n        all_cervix_images = all_cervix_images + cervix_images\n    all_cervix_images = pd.DataFrame({'imagepath': all_cervix_images})\n    all_cervix_images['filetype'] = all_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\n    all_cervix_images['type'] = all_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\n    return all_cervix_images\nall_cervix_images=getCervixImagesTraining()\nall_cervix_images.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"027aeced-f169-82d2-5991-dd2ca97fbce1"},"outputs":[],"source":"def plotbasicGraph():\n    print('We have a total of {} images in the whole dataset'.format(all_cervix_images.shape[0]))\n    type_aggregation = all_cervix_images.groupby(['type', 'filetype']).agg('count')\n    print(type_aggregation)\n    type_aggregation_p = type_aggregation.apply(lambda row: 1.0*row['imagepath']/all_cervix_images.shape[0], axis=1)\n    print(type_aggregation_p)\n    fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10, 8))\n\n    type_aggregation.plot.barh(ax=axes[0])\n    axes[0].set_xlabel(\"image count\")\n    type_aggregation_p.plot.barh(ax=axes[1])\n    axes[1].set_xlabel(\"training size fraction\") \n\nplotbasicGraph()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b17757c4-caa9-0ad6-207a-3fbe2c52c096"},"outputs":[],"source":"fig = plt.figure(figsize=(12,8))\n\ni = 1\nfor t in all_cervix_images['type'].unique():\n    ax = fig.add_subplot(1,3,i)\n    i+=1\n    f = all_cervix_images[all_cervix_images['type'] == t]['imagepath'].values[0]\n    plt.imshow(plt.imread(f))\n    plt.title('sample for cervix {}'.format(t))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0eb8812f-6e07-be96-7828-d7bd371c7623"},"outputs":[],"source":"from collections import defaultdict\n\nimages = defaultdict(list)\n\nfor t in all_cervix_images['type'].unique():\n    sample_counter = 0\n    for _, row in all_cervix_images[all_cervix_images['type'] == t].iterrows():\n        #print('reading image {}'.format(row.imagepath))\n        try:\n            img = imread(row.imagepath)\n            sample_counter +=1\n            images[t].append(img)\n        except:\n            print('image read failed for {}'.format(row.imagepath))\n        if sample_counter > 35:\n            break"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5da00831-57b0-4949-689d-6ece78e82216"},"outputs":[],"source":"dfs = []\nfor t in all_cervix_images['type'].unique():\n    t_ = pd.DataFrame(\n        {\n            'nrows': list(map(lambda i: i.shape[0], images[t])),\n            'ncols': list(map(lambda i: i.shape[1], images[t])),\n            'nchans': list(map(lambda i: i.shape[2], images[t])),\n            'type': t\n        }\n    )\n    print(t_)\n    dfs.append(t_)\n\nshapes_df = pd.concat(dfs, axis=0)\nshapes_df_grouped = shapes_df.groupby(by=['nchans', 'ncols', 'nrows', 'type']).size().reset_index().sort_values(['type', 0], ascending=False)\nshapes_df_grouped"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"91297dde-24cd-46ca-473c-c0cba81c84fd"},"outputs":[],"source":"shapes_df_grouped['size_with_type'] = shapes_df_grouped.apply(lambda row: '{}-{}-{}'.format(row.ncols, row.nrows, row.type), axis=1)\nshapes_df_grouped = shapes_df_grouped.set_index(shapes_df_grouped['size_with_type'].values)\nshapes_df_grouped['count'] = shapes_df_grouped[[0]]\n\nplt.figure(figsize=(10,8))\n#shapes_df_grouped['count'].plot.barh(figsize=(10,8))\nsns.barplot(x=\"count\", y=\"size_with_type\", data=shapes_df_grouped)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5f469ba9-56d8-cd42-1742-954d075fd4f7"},"outputs":[],"source":"def transform_image(img, rescaled_dim, to_gray=False):\n    resized = cv2.resize(img, (rescaled_dim, rescaled_dim), cv2.INTER_LINEAR)\n    #print(resized)\n    if to_gray:\n        resized = cv2.cvtColor(resized, cv2.COLOR_RGB2GRAY).astype('float')\n    else:\n        resized = resized.astype('float')\n\n    normalized = cv2.normalize(resized, None, 0.0, 1.0, cv2.NORM_MINMAX)\n    #print(normalized)\n    normalized.shape\n    timg = normalized.reshape(1, np.prod(normalized.shape))\n    #print(timg)\n    return timg/np.linalg.norm(timg)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ec8dbdc8-dcf3-4720-4f9f-3ad7816845fb"},"outputs":[],"source":"rescaled_dim = 100\nall_images = []\nall_image_types = []\n\nfor t in all_cervix_images['type'].unique():\n    all_images = all_images + images[t]\n    all_image_types = all_image_types + len(images[t])*[t]\n    #print(len(images[t])*[t])\n\n#print(all_images  )\n#print(all_image_types )\ntransform_image(all_images[0], rescaled_dim)\n\n    \n "},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ded046c8-a2be-793e-bc47-6adc7836b5a2"},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a5dbd55a-9bac-a2cb-5d2d-e917ef66dc28"},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5b7e9dd8-e4a3-eca5-7845-144bb3ebdc05"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}