{"cells":[{"metadata":{},"cell_type":"markdown","source":"#Making an attempt to see the topological structure of the skin cancer image data with PCA as lens","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import kmapper as km","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df=pd.read_csv(\"../input/jpeg-melanoma-256x256/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_imgs_pos =[]\ntrain_imgs_neg = []\ntrain_img_name_pos=[]\ntrain_img_name_neg=[]\n\nbase_path=\"../input/jpeg-melanoma-256x256/train\"\n# pos_count=0\n# neg_count =0\nfor idx,img in enumerate(train_df['image_name']) :\n    image_name=img\n    img_path = os.path.join(base_path,image_name+\".jpg\")\n    \n           \n    #Add negative samples when target is zero\n    if (train_df['target'].iloc[idx]==0) and (len(train_imgs_neg)<10000):\n        img = cv2.imread(img_path,0)\n        img = cv2.resize(img,(128,128))\n        img = np.array(img).reshape(-1,16384)\n        train_imgs_neg.append(img.flatten()) \n        train_img_name_neg.append(image_name)\n        \n     #Add psositive samples when target is one   \n    elif (train_df['target'].iloc[idx]==1) and (len(train_imgs_pos)<500):\n        img = cv2.imread(img_path,0)\n        img = cv2.resize(img,(128,128))\n        img = np.array(img).reshape(-1,16384)\n        train_imgs_pos.append(img.flatten()) \n        train_img_name_pos.append(image_name)\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_img_name_pos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_labels=np.ones(500).reshape(-1,1)\nneg_labels=np.zeros(10000).reshape(-1,1)\nprint(len(pos_labels),len(neg_labels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nli_1=train_img_name_pos\nli_2=train_img_name_neg\nli_1.extend(li_2)\ncombined_img_names=li_1\nprint(len(combined_img_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lis_1=train_imgs_pos\nlis_2=train_imgs_neg\nlis_1.extend(lis_2)\ncombined=lis_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(combined))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nfrom sklearn import ensemble\nimport kmapper as km\nfrom kmapper.plotlyviz import *\nfrom sklearn.decomposition import PCA\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y1 = np.vstack((pos_labels,neg_labels))\n\ny2=combined_img_names\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ydf=pd.DataFrame(y1,columns=['label'])\nydf['img_names']=y2\nydf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lens2 = mapper.fit_transform(combined, projection=PCA(n_components=1))\nprint(lens2)\nprint(len(lens2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#single component is covering as much as 60% variance,so im considering only one component.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a 1-D lens with the First Principal Component from PCA\nX=combined\nmapper = km.KeplerMapper(verbose=0)\nlens = mapper.fit_transform(X, projection=PCA(0.6))\n\nscomplex = mapper.map(lens,\n                      combined,\n                      nr_cubes=15,\n                      overlap_perc=0.7,\n                      clusterer=sklearn.cluster.KMeans(n_clusters=2,\n                                                       random_state=3471))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pl_brewer = [[0.0, '#006837'],\n             [0.1, '#1a9850'],\n             [0.2, '#66bd63'],\n             [0.3, '#a6d96a'],\n             [0.4, '#d9ef8b'],\n             [0.5, '#ffffbf'],\n             [0.6, '#fee08b'],\n             [0.7, '#fdae61'],\n             [0.8, '#f46d43'],\n             [0.9, '#d73027'],\n             [1.0, '#a50026']]\ncolor_function = lens [:,0] - lens[:,0].min()\nmy_colorscale = pl_brewer\nkmgraph,  mapper_summary, colorf_distribution = get_mapper_graph(scomplex,\n                                                                 color_function,\n                                                                 color_function_name='Distance to x-min',\n                                                                 colorscale=my_colorscale)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# assign to node['custom_tooltips']  the node label (0 for benign, 1 for malignant)\ny=ydf.index\nfor node in kmgraph['nodes']:\n    node['custom_tooltips'] = y[scomplex['nodes'][node['name']]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# assign to node['custom_tooltips']  the node label (0 for benign, 1 for malignant)\ny=ydf.label\nfor node in kmgraph['nodes']:\n    node['custom_tooltips'] = y[scomplex['nodes'][node['name']]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bgcolor = 'rgba(10,10,10, 0.9)'\ny_gridcolor = 'rgb(150,150,150)'# on a black background the gridlines are set on  grey","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nplotly_graph_data = plotly_graph(kmgraph, graph_layout='fr', colorscale=my_colorscale,\n                                 factor_size=2.5, edge_linewidth=0.5)\nlayout = plot_layout(title='Topological network representing the<br>Skin cancer  dataset',\n                     width=620, height=570,\n                     annotation_text=get_kmgraph_meta(mapper_summary),\n                     bgcolor=bgcolor)\n\nfw_graph = go.FigureWidget(data=plotly_graph_data, layout=layout)\nfw_hist = node_hist_fig(colorf_distribution, bgcolor=bgcolor,\n                        y_gridcolor=y_gridcolor)\nfw_summary = summary_fig(mapper_summary, height=300)\ndashboard = hovering_widgets(kmgraph,\n                             fw_graph,\n                             ctooltips=True, # ctooltips = True, because we assigned a label to each\n                                             #cluster member\n                             bgcolor=bgcolor,\n                             y_gridcolor=y_gridcolor,\n                             member_textbox_width=600)\n\n#Update the fw_graph colorbar, setting its title:\n\nfw_graph.data[1].marker.colorbar.title = 'dist to<br>x-min'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport plotly.graph_objs as go\nfrom ipywidgets import (HBox, VBox)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VBox([fw_graph, HBox([fw_summary, fw_hist])])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dashboard","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nplotly_graph_data = plotly_graph(kmgraph, graph_layout='fr', colorscale=my_colorscale,\n                                 factor_size=2.5, edge_linewidth=0.5)\nlayout = plot_layout(title='Topological network representing the<br>Skin cancer  dataset',\n                     width=620, height=570,\n                     annotation_text=get_kmgraph_meta(mapper_summary),\n                     bgcolor=bgcolor)\n\nfw_graph = go.FigureWidget(data=plotly_graph_data, layout=layout)\nfw_hist = node_hist_fig(colorf_distribution, bgcolor=bgcolor,\n                        y_gridcolor=y_gridcolor)\nfw_summary = summary_fig(mapper_summary, height=300)\ndashboard = hovering_widgets(kmgraph,\n                             fw_graph,\n                             ctooltips=True, # ctooltips = True, because we assigned a label to each\n                                             #cluster member\n                             bgcolor=bgcolor,\n                             y_gridcolor=y_gridcolor,\n                             member_textbox_width=600)\n\n#Update the fw_graph colorbar, setting its title:\n\nfw_graph.data[1].marker.colorbar.title = 'dist to<br>x-min'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dashboard","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the simplicial complex\ngraph = mapper.map(lens,\n                   X,\n                   cover=km.Cover(n_cubes=15, perc_overlap=0.7),\n                   clusterer=sklearn.cluster.KMeans(n_clusters=2,\n                                                    random_state=1618033))\n\ny=ydf.label\n# Visualization\nmapper.visualize(graph,\n                 path_html=\"Skin-cancer.html\",\n                 title=\"Melanoma skin Cancer Dataset\",\n                 custom_tooltips=y)\n\n\n# import matplotlib.pyplot as plt\nkm.draw_matplotlib(graph)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Visualization\ny=ydf.index\nmapper.visualize(graph,\n                 path_html=\"Skin-cancer_index.html\",\n                 title=\"Melanoma skin Cancer Dataset\",\n                 custom_tooltips=y)\n\n\n# import matplotlib.pyplot as plt\nkm.draw_matplotlib(graph)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#print the map using image names with each cluster","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# assign to node['custom_tooltips']  the node label (0 for benign, 1 for malignant)\ny=ydf.img_names\nfor node in kmgraph['nodes']:\n    node['custom_tooltips'] = y[scomplex['nodes'][node['name']]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Visualization\ny=ydf.img_names\nmapper.visualize(graph,\n                 path_html=\"Skin-cancer_img_names.html\",\n                 title=\"Melanoma skin Cancer Dataset\",\n                 custom_tooltips=y)\n\n\n# import matplotlib.pyplot as plt\nkm.draw_matplotlib(graph)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# upon reading the html file saved in the output folder and cluster 14 the following images are the one which are always same irrespective of the sample size changes,we can see these images all of them are torso and upper body,we can visulaize the images in following cells,the understanding is if any image which has very near features as this can be malignment with more confidence.Only image 2974155 is benin,but it is quite similar to other images.However this information will be helpful for the Domain experts,in this case Doctors.\n# All other clusters has majority the benin and very small portion positive cases\n \n# ISIC_2337907.jpg\n# ISIC_3341710.jpg\n# ISIC_4230049.jpg\n# ISIC_2974155.jpg","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"mal_img1=cv2.imread(\"../input/jpeg-melanoma-256x256/train/ISIC_2337907.jpg\")\n\nmal_img1=cv2.cvtColor(mal_img1, cv2.COLOR_BGR2RGB ) \nplt.imshow(mal_img1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mal_img2=cv2.imread(\"../input/jpeg-melanoma-256x256/train/ISIC_3341710.jpg\")\nmal_img2=cv2.cvtColor(mal_img2, cv2.COLOR_BGR2RGB ) \nplt.imshow(mal_img2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mal_img3=cv2.imread(\"../input/jpeg-melanoma-256x256/train/ISIC_4230049.jpg\")\nmal_img3=cv2.cvtColor(mal_img3, cv2.COLOR_BGR2RGB ) \nplt.imshow(mal_img3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ben_img1=cv2.imread(\"../input/jpeg-melanoma-256x256/train/ISIC_2974155.jpg\")\nben_img1=cv2.cvtColor(ben_img1, cv2.COLOR_BGR2RGB ) \n\nplt.imshow(ben_img1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}