{"cells":[{"metadata":{},"cell_type":"markdown","source":"TO DO:\n\nIMPORTERE:\n- importere all dataen: DONE\n- remove duplicates: DONE\n\nVISUALIZATION:\n- images \n- malignant, benign: DONE\n- age\n- gender: DONE\n- affected areas \n- mean and std?? \n\nPREPROCESSING:\n- make train, validation, and test set\n- resize\n- rescale\n\nDATA AUGMENTATION:\n- random transformation"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfrom keras.layers import Input, Lambda, Dense, Flatten\nfrom keras.models import Model\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.vgg16 import preprocess_input\nfrom keras.preprocessing import image\nfrom keras.models import Sequential\nfrom glob import glob\nimport matplotlib.pyplot as plt\n\nfrom keras.optimizers import Adam, SGD, RMSprop\nimport tensorflow as tf\nimport cv2\nimport glob\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\nfrom tensorflow.python.keras import backend as K\nimport plotly.graph_objects as go\nimport plotly.offline as py\nautosize =False\n\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\n\n%matplotlib inline\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_dir='/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\n#test_dir='/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\n#train=pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\n#test=pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\ntrain = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntest = pd.read_csv(\"../input/siim-isic-melanoma-classification/test.csv\")\ntrain.head()\ntest.head()\ntrain['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# remove duplicates\ndup=pd.read_csv(\"../input/siim-list-of-duplicates/2020_Challenge_duplicates.csv\")\n\ndrop_index_list =[]\nfor dup_image in dup.ISIC_id_paired:\n  for idx,image in enumerate(train.image_name):\n    if image == dup_image:\n      drop_index_list.append(idx)\n\nprint(\"Number of duplicates in dataset is:\", len(drop_index_list))\n\ntrain.drop(drop_index_list, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.countplot(train.benign_malignant, palette=\"Set2\")\nplt.title(\"Malignant vs. benign\")\nplt.show()\n\n# sex\nfig, ax = plt.subplots(1,2,figsize=(20,5))\n\nsns.countplot(train.sex, palette=\"Reds_r\", ax=ax[0])\nax[0].set_xlabel(\"\")\nax[0].set_title(\"Gender counts in training set\")\n\nsns.countplot(test.sex, palette=\"Blues_r\", ax=ax[1])\nax[1].set_xlabel(\"\")\nax[1].set_title(\"Gender counts in test set\")\nplt.show()\n\n# age\nfig, ax = plt.subplots(1,2,figsize=(20,5))\n\nsns.countplot(train.age_approx, color=\"tomato\", ax=ax[0])\nlabels = ax[0].get_xticklabels()\nax[0].set_xticklabels(labels, rotation=90)\nax[0].set_xlabel(\"Age\")\nax[0].set_title(\"Age distribution in training set\")\n\nsns.countplot(test.age_approx, color=\"skyblue\", ax=ax[1])\nlabels = ax[1].get_xticklabels()\nax[1].set_xticklabels(labels, rotation=90)\nax[1].set_xlabel(\"Age\")\nax[1].set_title(\"Age distribution in test set\")\nplt.show()\n\n# age average\nprint(train.age_approx.mean())\nprint(test.age_approx.mean())\n\n# affected areas\nfig, ax = plt.subplots(1,2,figsize=(20,5))\n\nsns.countplot(train.anatom_site_general_challenge, color=\"tomato\", ax=ax[0])\nlabels = ax[0].get_xticklabels()\nax[0].set_xticklabels(labels, rotation=90)\nax[0].set_xlabel(\"\")\nax[0].set_title(\"Affected areas in training set\")\n\nsns.countplot(test.anatom_site_general_challenge, color=\"skyblue\", ax=ax[1])\nlabels = ax[1].get_xticklabels()\nax[1].set_xticklabels(labels, rotation=90)\nax[1].set_xlabel(\"\")\nax[1].set_title(\"Affected areas in test set\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show images\n# https://www.kaggle.com/jagdmir/siim-melanoma-classification-modelling/data?fbclid=IwAR18v3usRMR7jyhZEGhANPbMqGuMaG9Lg8DANpygC3fmNkAvq_UnJxncXHI#Exploratory-Data-Analysis\n\n# benign\ndf_imlist= train[train['target']==0].sample(100)\nimgs = train.sample(30).image_name.values\n\nplt.figure(figsize=(8,8))\nimlist = []\ndf_imlist = df_imlist.head(9)\ndf_imlist = df_imlist.reset_index()\nfor i,k in enumerate(imgs):\n    im = cv2.imread(str('../input/siim-isic-melanoma-classification/jpeg/train/'+k+'.jpg'))\n    im = cv2.resize(im, (224,224))\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = im.astype(np.float32)/255\n    imlist.append(im)\nf,ax = plt.subplots(4,4, figsize =(8,8))\nfor i, im in enumerate(imlist):\n    ax[i//4, i%4].imshow(im)\n    ax[i//4, i%4].axis('off')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show malignant\ndf_imlist= train[train['target']==1].sample(100)\nimgs = train.sample(30).image_name.values\n\nplt.figure(figsize=(8,8))\nimlist = []\ndf_imlist = df_imlist.head(9)\ndf_imlist = df_imlist.reset_index()\nfor i,k in enumerate(imgs):\n    im = cv2.imread(str('../input/siim-isic-melanoma-classification/jpeg/train/'+k+'.jpg'))\n    im = cv2.resize(im, (224,224))\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    im = im.astype(np.float32)/255\n    imlist.append(im)\nf,ax = plt.subplots(2,8, figsize =(8,8))\nfor i, im in enumerate(imlist):\n    ax[i//4, i%4].imshow(im)\n    ax[i//4, i%4].axis('off')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# RGB distribution\n# https://www.kaggle.com/parulpandey/melanoma-classification-eda-starter\n\ndf_benign=train[train['target']==0].sample(1)\ndf_malignant=train[train['target']==1].sample(1)\nprint(df_benign)\n\nimg_dir = str(\"../input/siim-isic-melanoma-classification/jpeg/train\")\n\nf = plt.figure(figsize=(16,8))\nf.add_subplot(1,2, 1)\n\nsample_img = str(df_benign['image_name']+'.jpg')\nsample_img = sample_img.split(\" \")[4]\nsample_img = sample_img[:16]\nraw_image = plt.imread(os.path.join(img_dir, str(sample_img)))\nplt.imshow(raw_image, cmap='gray')\nplt.colorbar()\nplt.title('Benign Image')\nprint(f\"Image dimensions:  {raw_image.shape[0],raw_image.shape[1]}\")\nprint(f\"Maximum pixel value : {raw_image.max():.1f} ; Minimum pixel value:{raw_image.min():.1f}\")\nprint(f\"Mean value of the pixels : {raw_image.mean():.1f} ; Standard deviation : {raw_image.std():.1f}\")\n\nf.add_subplot(1,2, 2)\n\n#_ = plt.hist(raw_image.ravel(),bins = 256, color = 'orange',)\n_ = plt.hist(raw_image[:, :, 0].ravel(), bins = 256, color = 'red', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 1].ravel(), bins = 256, color = 'Green', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 2].ravel(), bins = 256, color = 'Blue', alpha = 0.5)\n_ = plt.xlabel('Intensity Value')\n_ = plt.ylabel('Count')\n_ = plt.legend(['Red_Channel', 'Green_Channel', 'Blue_Channel'])\nplt.show()\n\n# malignant\nf = plt.figure(figsize=(16,8))\nf.add_subplot(1,2, 1)\n\nsample_img = str(df_malignant['image_name']+'.jpg')\nsample_img = sample_img.split(\" \")[4]\nsample_img = sample_img[:16]\nraw_image = plt.imread(os.path.join(img_dir, str(sample_img)))\nplt.imshow(raw_image, cmap='gray')\nplt.colorbar()\nplt.title('Malignant Image')\nprint(f\"Image dimensions:  {raw_image.shape[0],raw_image.shape[1]}\")\nprint(f\"Maximum pixel value : {raw_image.max():.1f} ; Minimum pixel value:{raw_image.min():.1f}\")\nprint(f\"Mean value of the pixels : {raw_image.mean():.1f} ; Standard deviation : {raw_image.std():.1f}\")\n\nf.add_subplot(1,2, 2)\n\n#_ = plt.hist(raw_image.ravel(),bins = 256, color = 'orange',)\n_ = plt.hist(raw_image[:, :, 0].ravel(), bins = 256, color = 'red', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 1].ravel(), bins = 256, color = 'Green', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 2].ravel(), bins = 256, color = 'Blue', alpha = 0.5)\n_ = plt.xlabel('Intensity Value')\n_ = plt.ylabel('Count')\n_ = plt.legend(['Red_Channel', 'Green_Channel', 'Blue_Channel'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# RGB of all images\n# https://www.kaggle.com/datafan07/analysis-of-melanoma-metadata-and-effnet-ensemble?fbclid=IwAR2y49DfaraG5jAqt1sRs6-CYLIhqnuH1kJzTLEaWX7jZhDyhbGyqUKsRWg\n\nimg_stats_path = '/kaggle/input/melanoma2020imgtabular'\n\ntrain_attr = pd.read_csv(\n    os.path.join(img_stats_path, 'train_mean_colorres.csv'))\ntest_attr = pd.read_csv(os.path.join(img_stats_path, 'test_mean_colorres.csv'))\n\ntrain_attr.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([train, train_attr], axis=1)\ntest = pd.concat([test, test_attr], axis=1)\n\ntrain['res'] = train['width'].astype(str) + 'x' + train['height'].astype(str)\ntest['res'] = test['width'].astype(str) + 'x' + test['height'].astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a customized chart and giving in figsize etc.\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nplt.style.use('ggplot')\n\nfig = plt.figure(constrained_layout=True, figsize=(20, 12))\n\n# Creating a grid with 3 columns and 3 rows\n\ngrid = gridspec.GridSpec(ncols=3, nrows=3, figure=fig)\n\n# Customizing the first grid.\n\nax1 = fig.add_subplot(grid[0, :2])\n\n# Setting the title.\n\nax1.axes.set_title(\"RGB Channels of Benign Images\",fontsize=30)\n\n# Setting the axis font size \nax1.tick_params(labelsize=20)\n\n# Setting the range of the y-axis \nax1.set(ylim=(0, 0.02))\n\n# Plot.\n\nsns.distplot(train[train['target'] == 0].reds, #Pick out the reds from benign training images (target variable equals 0)\n             hist_kws={\n                 'rwidth': 0.75,\n                 'edgecolor': 'black',\n                 'alpha': 0.3\n             },\n             color='red',\n             kde=True,\n             ax=ax1,\n             label='Reds')\nsns.distplot(train[train['target'] == 0].greens, #Pick out the greens \n             hist_kws={\n                 'rwidth': 0.75,\n                 'edgecolor': 'black',\n                 'alpha': 0.3\n             },\n             color='green',\n             kde=True,\n             ax=ax1,\n             label='Greens')\nsns.distplot(train[train['target'] == 0].blues,#Pick out the blues \n             hist_kws={\n                 'rwidth': 0.75,\n                 'edgecolor': 'black',\n                 'alpha': 0.3\n             },\n             color='blue',\n             kde=True,\n             ax=ax1,\n             label='Blues')\n\nax1.legend(fontsize='x-large', title_fontsize='60',loc='upper left')\nax1.set_xlabel(\"Intensity value\",fontsize=25)\nax1.set_ylabel(\"Density\",fontsize=25)\n\n\n# Customizing the second grid.\n\nax2 = fig.add_subplot(grid[1, :2])\n\n\n# Set the title.\n\nax2.axes.set_title(\"RGB Channels of Malignant Images\",fontsize=30)\n\n# Setting the axis font size \nax2.tick_params(labelsize=20)\n\n# Setting the range of the y-axis \nax2.set(ylim=(0, 0.02))\n\n\n# Plot\n\nsns.distplot(train[train['target'] == 1].reds,#Pick out the reds from malignant training images (target variable equals 1)\n             hist_kws={\n                 'rwidth': 0.75,\n                 'edgecolor': 'black',\n                 'alpha': 0.3\n             },\n             color='red',\n             kde=True,\n             ax=ax2,\n             label='Reds')\nsns.distplot(train[train['target'] == 1].greens,#Pick out the greens \n             hist_kws={\n                 'rwidth': 0.75,\n                 'edgecolor': 'black',\n                 'alpha': 0.3\n             },\n             color='green',\n             kde=True,\n             ax=ax2,\n             label='Greens')\nsns.distplot(train[train['target'] == 1].blues,#Pick out the blues \n             hist_kws={\n                 'rwidth': 0.75,\n                 'edgecolor': 'black',\n                 'alpha': 0.3\n             },\n             color='blue',\n             kde=True,\n             ax=ax2,\n             label='Blues')\nax2.legend(fontsize='x-large', title_fontsize='60',loc='upper left')\n\nax2.set_xlabel(\"Intensity value\",fontsize=25)\nax2.set_ylabel(\"Density\",fontsize=25)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}