{"cells": [{"execution_count": null, "outputs": [], "source": ["import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from skimage.io import imread, imshow\n", "import cv2\n", "from glob import glob\n", "\n", "%matplotlib inline"], "cell_type": "code", "metadata": {"_cell_guid": "34411b42-5e6a-061a-e240-2d77d17c8a75", "_uuid": "509c6e2f364f3791a55ac003030f27f13011407f"}}, {"execution_count": null, "outputs": [], "source": ["# Paths Imagens\n", "\n", "basepath = '../input/train/'\n", "\n", "all_cervix_images = []\n", "teste = []\n", "\n", "for path in sorted(glob(basepath + \"*\")):\n", "    cervix_type = path.split(\"/\")[-1]\n", "    cervix_images = sorted(glob(basepath + cervix_type + \"/*\"))\n", "    all_cervix_images = all_cervix_images + cervix_images\n", "    teste.append(cervix_images)\n", "\n", "all_cervix_images = pd.DataFrame({'imagepath': all_cervix_images})\n", "all_cervix_images['filetype'] = all_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\n", "all_cervix_images['type'] = all_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\n", "all_cervix_images.head()"], "cell_type": "code", "metadata": {"_cell_guid": "afa38fc1-86f1-7fa1-adee-47ab23c25481", "_uuid": "ca64758190fedd212e5bb5f34420e05913294214"}}, {"execution_count": null, "outputs": [], "source": ["# Contagem de Imagens\n", "\n", "print('We have a total of {} images in the whole dataset'.format(all_cervix_images.shape[0]))\n", "type_aggregation = all_cervix_images.groupby(['type', 'filetype']).agg('count')\n", "type_aggregation_p = type_aggregation.apply(lambda row: 1.0*row['imagepath']/all_cervix_images.shape[0], axis=1)\n", "\n", "fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10, 8))\n", "\n", "type_aggregation.plot.barh(ax=axes[0])\n", "axes[0].set_xlabel(\"image count\")\n", "type_aggregation_p.plot.barh(ax=axes[1])\n", "axes[1].set_xlabel(\"training size fraction\")"], "cell_type": "code", "metadata": {"_cell_guid": "05553ee2-4a8a-52cf-adea-4dbd7444b886", "_uuid": "faa33b791753d247c9538afea72cb9733bf18024"}}, {"execution_count": null, "outputs": [], "source": ["#Pr\u00e9via das Imagens\n", "\n", "fig = plt.figure(figsize=(12,8))\n", "\n", "i = 1\n", "for t in all_cervix_images['type'].unique():\n", "    ax = fig.add_subplot(1,3,i)\n", "    i+=1\n", "    f = all_cervix_images[all_cervix_images['type'] == t]['imagepath'].values[0]\n", "    plt.imshow(plt.imread(f))\n", "    plt.title('sample for cervix {}'.format(t))"], "cell_type": "code", "metadata": {"_cell_guid": "d22a5d40-ab0b-65bb-36bf-37b850d0ad52", "_uuid": "6fe72e98aef8f47bff822b50f71c225e22c045e4"}}, {"execution_count": null, "outputs": [], "source": ["# Subconjo dos tipos\n", "\n", "from collections import defaultdict\n", "\n", "images = defaultdict(list)\n", "\n", "for t in all_cervix_images['type'].unique():\n", "    sample_counter = 0\n", "    for _, row in all_cervix_images[all_cervix_images['type'] == t].iterrows():\n", "        #print('reading image {}'.format(row.imagepath))\n", "        try:\n", "            img = imread(row.imagepath)\n", "            sample_counter +=1\n", "            images[t].append(img)\n", "        except:\n", "            print('image read failed for {}'.format(row.imagepath))\n", "        if sample_counter > 35:\n", "            break"], "cell_type": "code", "metadata": {"_cell_guid": "a1083d08-2173-24c3-768a-cf1eca53e8f9", "_uuid": "b8139a41e37f5e13659e377bdecf68fa36b0c829"}}, {"execution_count": null, "outputs": [], "source": ["dfs = []\n", "for t in all_cervix_images['type'].unique():\n", "    t_ = pd.DataFrame(\n", "        {\n", "            'nrows': list(map(lambda i: i.shape[0], images[t])),\n", "            'ncols': list(map(lambda i: i.shape[1], images[t])),\n", "            'nchans': list(map(lambda i: i.shape[2], images[t])),\n", "            'type': t\n", "        }\n", "    )\n", "    dfs.append(t_)\n", "\n", "shapes_df = pd.concat(dfs, axis=0)\n", "shapes_df_grouped = shapes_df.groupby(by=['nchans', 'ncols', 'nrows', 'type']).size().reset_index().sort_values(['type', 0], ascending=False)\n", "shapes_df_grouped"], "cell_type": "code", "metadata": {"_cell_guid": "b9e0facc-5b86-cb64-13d7-9e73c78b72e7", "_uuid": "d18d27a65998cf981c289fde82e7fd22e5e42676"}}, {"execution_count": null, "outputs": [], "source": ["#Gr\u00e1fico com o tamanho das imagens\n", "\n", "shapes_df_grouped['size_with_type'] = shapes_df_grouped.apply(lambda row: '{}-{}-{}'.format(row.ncols, row.nrows, row.type), axis=1)\n", "shapes_df_grouped = shapes_df_grouped.set_index(shapes_df_grouped['size_with_type'].values)\n", "shapes_df_grouped['count'] = shapes_df_grouped[[0]]\n", "\n", "plt.figure(figsize=(10,8))\n", "#shapes_df_grouped['count'].plot.barh(figsize=(10,8))\n", "sns.barplot(x=\"count\", y=\"size_with_type\", data=shapes_df_grouped)"], "cell_type": "code", "metadata": {"_cell_guid": "a223746e-7e5b-25bb-93c5-da20d6d221ad", "_uuid": "88e72857043e9b0739580c0708659509d8847553"}}, {"execution_count": null, "outputs": [], "source": ["def transform_image(img, rescaled_dim, to_gray=False):\n", "    resized = cv2.resize(img, (rescaled_dim, rescaled_dim), cv2.INTER_LINEAR)\n", "\n", "    if to_gray:\n", "        resized = cv2.cvtColor(resized, cv2.COLOR_RGB2GRAY).astype('float')\n", "    else:\n", "        resized = resized.astype('float')\n", "\n", "    normalized = cv2.normalize(resized, None, 0.0, 1.0, cv2.NORM_MINMAX)\n", "    timg = normalized.reshape(1, np.prod(normalized.shape))\n", "\n", "    return timg/np.linalg.norm(timg)\n", "\n", "rescaled_dim = 100\n", "\n", "all_images = []\n", "all_image_types = []\n", "\n", "for t in all_cervix_images['type'].unique():\n", "    all_images = all_images + images[t]\n", "    all_image_types = all_image_types + len(images[t])*[t]\n", "\n", "# - normalize each uint8 image to the value interval [0, 1] as float image\n", "# - rgb to gray\n", "# - downsample image to rescaled_dim X rescaled_dim\n", "# - L2 norm of each sample = 1\n", "gray_all_images_as_vecs = [transform_image(img, rescaled_dim) for img in all_images]\n", "\n", "gray_imgs_mat = np.array(gray_all_images_as_vecs).squeeze()\n", "all_image_types = np.array(all_image_types)\n", "gray_imgs_mat.shape, all_image_types.shape"], "cell_type": "code", "metadata": {"_cell_guid": "f9614bb1-a967-41f9-c5c7-3cb81eee52ee", "_uuid": "045b7048c6a10bb033124d5c27b814e1000d6c3c"}}, {"execution_count": null, "outputs": [], "source": ["from sklearn.manifold import TSNE\n", "tsne = TSNE(\n", "    n_components=3,\n", "    init='random', # pca\n", "    random_state=101,\n", "    method='barnes_hut',\n", "    n_iter=500,\n", "    verbose=2\n", ").fit_transform(gray_imgs_mat)"], "cell_type": "code", "metadata": {"_cell_guid": "7e2cd59d-870e-3743-10fc-39f4387b589f", "_uuid": "c7d29cab70855618a418bd9f537f2f27660c6d71"}}, {"execution_count": null, "outputs": [], "source": ["for t in all_cervix_images['type'].unique():\n", "    tsne_t = tsne[np.where(all_image_types == t), :][0]\n", "    plt.scatter(tsne_t[:, 0], tsne_t[:, 1])\n", "plt.legend(all_cervix_images['type'].unique())"], "cell_type": "code", "metadata": {"_cell_guid": "fc3db54c-5446-01b3-a142-6d898463edf2", "_uuid": "afc1671711666a92fabe1ddc0f17ce57d2747407"}}, {"execution_count": null, "outputs": [], "source": ["from matplotlib.offsetbox import OffsetImage, AnnotationBbox\n", "def imscatter(x, y, images, ax=None, zoom=0.01):\n", "    ax = plt.gca()\n", "    images = [OffsetImage(image, zoom=zoom) for image in images]\n", "    artists = []\n", "    for x0, y0, im0 in zip(x, y, images):\n", "        ab = AnnotationBbox(im0, (x0, y0), xycoords='data', frameon=False)\n", "        artists.append(ax.add_artist(ab))\n", "    ax.update_datalim(np.column_stack([x, y]))\n", "    ax.autoscale()\n", "    #return artists\n", "\n", "nimgs = 60\n", "plt.figure(figsize=(10,8))\n", "imscatter(tsne[0:nimgs,0], tsne[0:nimgs,1], all_images[0:nimgs])"], "cell_type": "code", "metadata": {"_cell_guid": "a8e66bea-ba61-e81c-d898-35c92964413d", "_uuid": "549bcb857f33b1db07090ef89168a9c1bfd753c0"}}, {"execution_count": null, "outputs": [], "source": ["pal = sns.color_palette(\"hls\", 3)\n", "sns.palplot(pal)"], "cell_type": "code", "metadata": {"_cell_guid": "6df504e8-5f3e-078f-761f-55a9575af529", "_uuid": "1e48cd2a1c76b8cc1621dfdf94d6820abb7297b5"}}, {"execution_count": null, "outputs": [], "source": ["from scipy.spatial.distance import pdist, squareform\n", "\n", "sq_dists = squareform(pdist(gray_imgs_mat))\n", "\n", "all_image_types = list(all_image_types)\n", "\n", "d = {\n", "    'Type_1': pal[0],\n", "    'Type_2': pal[1],\n", "    'Type_3': pal[2]\n", "}\n", "\n", "# translate each sample to its color\n", "colors = list(map(lambda t: d[t], all_image_types))\n", "\n", "sns.clustermap(\n", "    sq_dists,\n", "    figsize=(12,12),\n", "    row_colors=colors, col_colors=colors,\n", "    cmap=plt.get_cmap('viridis')\n", ")"], "cell_type": "code", "metadata": {"_cell_guid": "682d782f-e31a-343b-95eb-610ca45d0410", "_uuid": "8f3833e56d002312015cab8c9f236268e9e822a1"}}, {"execution_count": null, "outputs": [], "source": ["mask = np.zeros_like(sq_dists, dtype=np.bool)\n", "mask[np.triu_indices_from(mask)] = True\n", "\n", "plt.figure(figsize=(12,12))\n", "sns.heatmap(sq_dists, cmap=plt.get_cmap('viridis'), square=True, mask=mask)"], "cell_type": "code", "metadata": {"_cell_guid": "f38a38f0-61a9-0faf-5839-a63d701b14e9", "_uuid": "6314fe7d167f35cf57f74ab2a9d9ce09f6bb1630"}}, {"execution_count": null, "outputs": [], "source": ["# upper triangle of matrix set to np.nan\n", "sq_dists[np.triu_indices_from(mask)] = np.nan\n", "sq_dists[0, 0] = np.nan\n", "\n", "fig = plt.figure(figsize=(12,8))\n", "# maximally dissimilar image\n", "ax = fig.add_subplot(1,3,1)\n", "maximally_dissimilar_image_idx = np.nanargmax(np.nanmean(sq_dists, axis=1))\n", "plt.imshow(all_images[maximally_dissimilar_image_idx])\n", "plt.title('maximally dissimilar')\n", "\n", "# maximally similar image\n", "ax = fig.add_subplot(1,3,2)\n", "maximally_similar_image_idx = np.nanargmin(np.nanmean(sq_dists, axis=1))\n", "plt.imshow(all_images[maximally_similar_image_idx])\n", "plt.title('maximally similar')\n", "\n", "# now compute the mean image\n", "ax = fig.add_subplot(1,3,3)\n", "mean_img = gray_imgs_mat.mean(axis=0).reshape(rescaled_dim, rescaled_dim, 3)\n", "plt.imshow(cv2.normalize(mean_img, None, 0.0, 1.0, cv2.NORM_MINMAX))\n", "plt.title('mean image')"], "cell_type": "code", "metadata": {"_cell_guid": "df5b6d99-9a60-3a6a-4d29-1dff20b50486", "_uuid": "c631a6e5bf484828d12c785819ae3e3fd3477966"}}, {"execution_count": null, "outputs": [], "source": ["from sklearn.preprocessing import LabelEncoder\n", "from sklearn.preprocessing import Normalizer\n", "y = LabelEncoder().fit_transform(all_image_types).reshape(-1)\n", "X = gray_imgs_mat # no need for normalizing, we already did this earlier Normalizer().fit_transform(gray_imgs_mat)\n", "X.shape, y.shape"], "cell_type": "code", "metadata": {"_cell_guid": "92118bcf-9d64-cddf-3326-190f59dc5c4b", "_uuid": "36011f336f702fd1333daf960e755038e92374c6"}}, {"execution_count": null, "outputs": [], "source": ["from sklearn.linear_model import LogisticRegression\n", "from sklearn.model_selection import GridSearchCV, train_test_split\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n", "\n", "X_train.shape, X_test.shape, y_train.shape, y_test.shape"], "cell_type": "code", "metadata": {"_cell_guid": "e5a1679b-078f-9b74-8322-e5bb08aaae88", "_uuid": "491947448cf186ab904a34e1d6b52da4d4c7722e"}}, {"execution_count": null, "outputs": [], "source": ["y_train, y_test"], "cell_type": "code", "metadata": {"_cell_guid": "ef99a002-dffb-47a8-98fa-349f5c6aaf24", "_uuid": "0abe3fc6670df0109d321cf82a4fb38e638a0f93"}}, {"execution_count": null, "outputs": [], "source": ["clf = LogisticRegression()\n", "grid = {\n", "    'C': [1e-9, 1e-6, 1e-3, 1e0],\n", "    'penalty': ['l1', 'l2']\n", "}\n", "cv = GridSearchCV(clf, grid, scoring='neg_log_loss', n_jobs=-1, verbose=1)\n", "cv.fit(X_train, y_train)"], "cell_type": "code", "metadata": {"_cell_guid": "7cea418d-b73d-8f4f-a594-b2ec37769a04", "_uuid": "14c471936594b78b0ada1f6c90f28b269ef82f27"}}, {"execution_count": null, "outputs": [], "source": ["y_test_hat_p = cv.predict_proba(X_test)"], "cell_type": "code", "metadata": {"_cell_guid": "c70960e5-13af-5aee-31c0-e6379bced58a", "_uuid": "d765efa998b76c26ec6fff4ad1e954868065c535"}}, {"execution_count": null, "outputs": [], "source": ["from sklearn.metrics import confusion_matrix\n", "\n", "y_test_hat = cv.predict(X_test)\n", "\n", "data = [\n", "    go.Heatmap(\n", "        z=confusion_matrix(y_test, y_test_hat),\n", "        x=[0, 1, 2],\n", "        y=[0, 1, 2],\n", "        colorscale='Viridis',\n", "        text = True ,\n", "        opacity = 1.0\n", "    )\n", "]\n", "\n", "layout = go.Layout(\n", "    title='Test Confusion matrix',\n", "    xaxis = dict(ticks='', nticks=36),\n", "    yaxis = dict(ticks='' ),\n", "    width = 900, height = 700,\n", "    \n", ")\n", "\n", "\n", "fig = go.Figure(data=data, layout=layout)\n", "py.iplot(fig, filename='labelled-heatmap')"], "cell_type": "code", "metadata": {"_cell_guid": "908d3644-64a8-436c-9cfa-a2909219be2a", "_uuid": "f91be9b6307f11b8e9893e19f399234b94d02b35"}}, {"execution_count": null, "outputs": [], "source": "", "cell_type": "code", "metadata": {"_cell_guid": "4f1e1a8b-5a77-b1e1-548d-c2764972f5ac", "_uuid": "7a5d57897299644fab7015b50f468af43ffa7dbc"}}, {"execution_count": null, "outputs": [], "source": ["import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from skimage.io import imread, imshow\n", "import cv2\n", "\n", "%matplotlib inline\n", "import plotly.offline as py\n", "py.init_notebook_mode(connected=True)\n", "import plotly.graph_objs as go\n", "import plotly.tools as tls\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input/train\"]).decode(\"utf8\"))"], "cell_type": "code", "metadata": {"_cell_guid": "e95eaa83-557c-1a7c-936f-50416fafe1a4", "_uuid": "856fad8a0e2768009368b82f60abe053d782f03a"}}, {"execution_count": null, "outputs": [], "source": ["from glob import glob\n", "basepath = '../input/train/'\n", "\n", "all_cervix_images = []\n", "\n", "for path in sorted(glob(basepath + \"*\")):\n", "    cervix_type = path.split(\"/\")[-1]\n", "    cervix_images = sorted(glob(basepath + cervix_type + \"/*\"))\n", "    all_cervix_images = all_cervix_images + cervix_images\n", "\n", "all_cervix_images = pd.DataFrame({'imagepath': all_cervix_images})\n", "all_cervix_images['filetype'] = all_cervix_images.apply(lambda row: row.imagepath.split(\".\")[-1], axis=1)\n", "all_cervix_images['type'] = all_cervix_images.apply(lambda row: row.imagepath.split(\"/\")[-2], axis=1)\n", "all_cervix_images.head()"], "cell_type": "code", "metadata": {"_cell_guid": "30455336-38fa-c327-4fc8-e147f7a19d14", "_uuid": "f829a754ccb62349dd769c0d213cebfa03b7e8cf"}}, {"execution_count": null, "outputs": [], "source": "", "cell_type": "code", "metadata": {"_cell_guid": "66b8a028-3e3b-2fbb-01a4-fd48909913e2", "_uuid": "bf16e334362b0090d2f30d013441afaa65570a2e"}}, {"execution_count": null, "outputs": [], "source": ["def transform_image(img, rescaled_dim, to_gray=False):\n", "    resized = cv2.resize(img, (rescaled_dim, rescaled_dim), cv2.INTER_LINEAR)\n", "\n", "    if to_gray:\n", "        resized = cv2.cvtColor(resized, cv2.COLOR_RGB2GRAY).astype('float')\n", "    else:\n", "        resized = resized.astype('float')\n", "\n", "    normalized = cv2.normalize(resized, None, 0.0, 1.0, cv2.NORM_MINMAX)\n", "    timg = normalized.reshape(1, np.prod(normalized.shape))\n", "\n", "    return timg/np.linalg.norm(timg)\n", "\n", "rescaled_dim = 100\n", "\n", "all_images = []\n", "all_image_types = []\n", "\n", "for t in all_cervix_images['type'].unique():\n", "    all_images = all_images + images[t]\n", "    all_image_types = all_image_types + len(images[t])*[t]\n", "\n", "# - normalize each uint8 image to the value interval [0, 1] as float image\n", "# - rgb to gray\n", "# - downsample image to rescaled_dim X rescaled_dim\n", "# - L2 norm of each sample = 1\n", "gray_all_images_as_vecs = [transform_image(img, rescaled_dim) for img in all_images]\n", "\n", "gray_imgs_mat = np.array(gray_all_images_as_vecs).squeeze()\n", "all_image_types = np.array(all_image_types)\n", "gray_imgs_mat.shape, all_image_types.shape"], "cell_type": "code", "metadata": {"_cell_guid": "ca1eccdd-cbb5-a8fa-a76a-32a019ad8a35", "_uuid": "73e449581c15eec78212ccece4d15b597d935463"}}, {"execution_count": null, "outputs": [], "source": ["import matplotlib.pyplot as plt\n", "\n", "from skimage.feature import hog\n", "from skimage import data, color, exposure\n", "\n", "\n", "image = color.rgb2gray(data.astronaut())\n", "\n", "fd, hog_image = hog(image, orientations=8, pixels_per_cell=(16, 16),\n", "                    cells_per_block=(1, 1), visualise=True)\n", "\n", "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4), sharex=True, sharey=True)\n", "\n", "ax1.axis('off')\n", "ax1.imshow(image, cmap=plt.cm.gray)\n", "ax1.set_title('Input image')\n", "ax1.set_adjustable('box-forced')\n", "\n", "# Rescale histogram for better display\n", "hog_image_rescaled = exposure.rescale_intensity(hog_image, in_range=(0, 0.02))\n", "\n", "ax2.axis('off')\n", "ax2.imshow(hog_image_rescaled, cmap=plt.cm.gray)\n", "ax2.set_title('Histogram of Oriented Gradients')\n", "ax1.set_adjustable('box-forced')\n", "plt.show()"], "cell_type": "code", "metadata": {"_cell_guid": "f96c6e82-29ce-71ff-c75c-e3c42f38a77d", "_uuid": "22365d38440b9ba06ff75a5aa13ae5b25ff8f520"}}, {"execution_count": null, "outputs": [], "source": "", "cell_type": "code", "metadata": {"_cell_guid": "312dd378-a03d-5d85-5518-51a7d6225c2b", "_uuid": "95b9c1f53c0b809d6b4cc5284de2afab7ab2be67"}}], "nbformat": 4, "metadata": {"_change_revision": 0, "_is_fork": false, "kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}, "language_info": {"name": "python", "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "version": "3.6.0", "mimetype": "text/x-python", "file_extension": ".py"}}, "nbformat_minor": 0}