{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Melanoma Dataset EDA Light","metadata":{}},{"cell_type":"markdown","source":"Sources:\n    \n- https://www.kaggle.com/code/andradaolteanu/siim-melanoma-competition-eda-augmentations","metadata":{}},{"cell_type":"code","source":"# running interactively in kaggle or as background job in kaggle\nimport os\nif (get_ipython().config.IPKernelApp.connection_file.startswith('/root/.local/share')\n    or 'SHLVL' in os.environ):\n    BASE_PATH = '/kaggle/input/'\n    \nelse:\n    BASE_PATH = '../data/'","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:33.497374Z","iopub.execute_input":"2023-12-01T10:20:33.498348Z","iopub.status.idle":"2023-12-01T10:20:33.526819Z","shell.execute_reply.started":"2023-12-01T10:20:33.498311Z","shell.execute_reply":"2023-12-01T10:20:33.525814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f'Running on {DEVICE}')\n\nimport pathlib\nimport math\nfrom typing import Union\n\nimport torch.nn as nn\nimport missingno as msno\nimport pandas as pd\nfrom scipy.stats import pointbiserialr\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as animation\nimport pydicom\nfrom IPython.display import HTML\nimport numpy as np\nfrom skimage import measure\nimport copy\nimport torchvision\nimport torch.optim as optim","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-12-01T10:20:33.529066Z","iopub.execute_input":"2023-12-01T10:20:33.529783Z","iopub.status.idle":"2023-12-01T10:20:39.101059Z","shell.execute_reply.started":"2023-12-01T10:20:33.529752Z","shell.execute_reply":"2023-12-01T10:20:39.099847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(BASE_PATH+'siim-isic-melanoma-classification/train.csv')\ndf_test = pd.read_csv(BASE_PATH+'siim-isic-melanoma-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:39.102718Z","iopub.execute_input":"2023-12-01T10:20:39.103365Z","iopub.status.idle":"2023-12-01T10:20:39.233001Z","shell.execute_reply.started":"2023-12-01T10:20:39.103323Z","shell.execute_reply":"2023-12-01T10:20:39.231934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:39.234377Z","iopub.execute_input":"2023-12-01T10:20:39.234713Z","iopub.status.idle":"2023-12-01T10:20:39.267674Z","shell.execute_reply.started":"2023-12-01T10:20:39.234677Z","shell.execute_reply":"2023-12-01T10:20:39.266615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:39.269903Z","iopub.execute_input":"2023-12-01T10:20:39.270200Z","iopub.status.idle":"2023-12-01T10:20:39.298798Z","shell.execute_reply.started":"2023-12-01T10:20:39.270175Z","shell.execute_reply":"2023-12-01T10:20:39.297782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing Data","metadata":{}},{"cell_type":"code","source":"ser_perc = (df_train.isnull().sum() / len(df_train)).round(decimals=3)\npd.concat((df_train.isnull().sum(), \n           ser_perc), \n          axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:39.300530Z","iopub.execute_input":"2023-12-01T10:20:39.300877Z","iopub.status.idle":"2023-12-01T10:20:39.331593Z","shell.execute_reply.started":"2023-12-01T10:20:39.300849Z","shell.execute_reply":"2023-12-01T10:20:39.330540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(nrows=1, \n                               ncols=2, \n                               figsize = (16, 6))\n\nmsno.matrix(df_train, ax=ax1, color=(207/255, 196/255, 171/255), fontsize=10)\nmsno.matrix(df_test, ax=ax2, color=(218/255, 136/255, 130/255), fontsize=10)\n\nax1.set_title('Train Missing Values Map', fontsize = 16)\nax2.set_title('Test Missing Values Map', fontsize = 16);\n","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:39.332947Z","iopub.execute_input":"2023-12-01T10:20:39.333241Z","iopub.status.idle":"2023-12-01T10:20:40.061793Z","shell.execute_reply.started":"2023-12-01T10:20:39.333216Z","shell.execute_reply":"2023-12-01T10:20:40.060764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Labels","metadata":{}},{"cell_type":"code","source":"def make_autopct(values):\n    \"\"\"have percentage and absolute values both displayed in pie plot\"\"\"\n    def my_autopct(pct):\n        total = sum(values)\n        val = int(round(pct*total/100.0))\n        return '{p:.2f}%  ({v:d})'.format(p=pct,v=val)\n    return my_autopct\n\n\ndf_train['benign_malignant'].value_counts().plot(kind='pie',\n                                                 autopct=make_autopct(df_train['benign_malignant'].value_counts()))","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:21:38.015422Z","iopub.execute_input":"2023-12-01T10:21:38.016310Z","iopub.status.idle":"2023-12-01T10:21:38.153734Z","shell.execute_reply.started":"2023-12-01T10:21:38.016267Z","shell.execute_reply":"2023-12-01T10:21:38.152579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features","metadata":{}},{"cell_type":"code","source":"columns = df_train.columns[2:]\nprint(len(columns))\n\nfig, axes = plt.subplots(nrows=math.ceil(len(columns)/3), \n                         ncols=3,\n                         figsize=(15, 8))\naxes = axes.flatten()\n\nfor idx, column in enumerate(columns):\n    ax = axes[idx]\n    sizes = df_train[column].value_counts().to_dict()\n    ax.pie(sizes.values(), \n           labels=sizes.keys(),\n           # colors=['g', 'orange', 'r'])\n          )\n    ax.set_title(column)\n    \nfor ax in axes:\n    ax.set_axis_off()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:40.209269Z","iopub.execute_input":"2023-12-01T10:20:40.209680Z","iopub.status.idle":"2023-12-01T10:20:40.854634Z","shell.execute_reply.started":"2023-12-01T10:20:40.209644Z","shell.execute_reply":"2023-12-01T10:20:40.853162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.catplot(\n    data=df_train, x=\"sex\", y=\"age_approx\", hue=\"benign_malignant\",\n    kind=\"violin\", split=True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:40.856004Z","iopub.execute_input":"2023-12-01T10:20:40.857050Z","iopub.status.idle":"2023-12-01T10:20:41.475486Z","shell.execute_reply.started":"2023-12-01T10:20:40.857008Z","shell.execute_reply":"2023-12-01T10:20:41.474546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsns.set_style(\"whitegrid\")\nfig, ax = plt.subplots(figsize=(11, 7))\n\nsns.violinplot(\n    data=df_train, x=\"anatom_site_general_challenge\", y=\"age_approx\", hue=\"benign_malignant\",\n    kind=\"violin\", split=True, ax=ax\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:41.476851Z","iopub.execute_input":"2023-12-01T10:20:41.477348Z","iopub.status.idle":"2023-12-01T10:20:42.201299Z","shell.execute_reply.started":"2023-12-01T10:20:41.477321Z","shell.execute_reply":"2023-12-01T10:20:42.199549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 2))\nsns.histplot(df_train['age_approx'], \n             kde=True, \n             linewidth=0.1, \n             ax=plt.gca(), \n             alpha=0.3)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:42.203438Z","iopub.execute_input":"2023-12-01T10:20:42.203898Z","iopub.status.idle":"2023-12-01T10:20:42.865898Z","shell.execute_reply.started":"2023-12-01T10:20:42.203851Z","shell.execute_reply":"2023-12-01T10:20:42.864861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preview Images","metadata":{}},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:42.867391Z","iopub.execute_input":"2023-12-01T10:20:42.867702Z","iopub.status.idle":"2023-12-01T10:20:42.884377Z","shell.execute_reply.started":"2023-12-01T10:20:42.867676Z","shell.execute_reply":"2023-12-01T10:20:42.883208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_train_dicom = pathlib.Path(BASE_PATH+'siim-isic-melanoma-classification/train/')\npath_train_jpeg = pathlib.Path(BASE_PATH+'siim-isic-melanoma-classification/jpeg/train/')\n\nprint(len(list(path_train_dicom.iterdir())))\nprint(len(list(path_train_jpeg.iterdir())))\n\n\n# There's a .dcm dataset and a .jpg image for each train.csv entry","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:42.888833Z","iopub.execute_input":"2023-12-01T10:20:42.889150Z","iopub.status.idle":"2023-12-01T10:20:46.843343Z","shell.execute_reply.started":"2023-12-01T10:20:42.889123Z","shell.execute_reply":"2023-12-01T10:20:46.842264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(images: Union[np.array, list[np.array]], title='', cmap='gray'):  # (n, width, length)\n    \n    n_cols = 5\n    n_rows = math.ceil(len(images)/n_cols)\n    \n    fig, axes = plt.subplots(nrows=n_rows, \n                             ncols=n_cols,\n                             figsize=(n_cols*6, n_rows*6))\n    axes = axes.flatten()\n    \n    for i, image in enumerate(images):  # image: (width, length)\n        ax = axes[i]\n        ax.imshow(X=image,\n                  cmap=cmap)  # plt.cm.bone\n        ax.set_xticks([])\n        ax.set_yticks([])\n        \n    if title:\n        plt.suptitle(title, fontsize=30)\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:46.845062Z","iopub.execute_input":"2023-12-01T10:20:46.845452Z","iopub.status.idle":"2023-12-01T10:20:46.852264Z","shell.execute_reply.started":"2023-12-01T10:20:46.845416Z","shell.execute_reply":"2023-12-01T10:20:46.851558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot random DICOM vs JPEG","metadata":{}},{"cell_type":"code","source":"#image_names = df_train.sample(frac=1).iloc[:5]['image_name']\nimage_names=['ISIC_0015719',\n             'ISIC_0074311',\n             'ISIC_0078712',\n             'ISIC_0083035',\n             'ISIC_0084395']\n\nimage_paths_dicom = [f'{BASE_PATH}siim-isic-melanoma-classification/train/{image_name}.dcm' for image_name in image_names]\nimages_dicom = [pydicom.dcmread(image_path).pixel_array for image_path in image_paths_dicom]\n\nimage_paths_jpeg = [f'{BASE_PATH}siim-isic-melanoma-classification/jpeg/train/{image_name}.jpg' for image_name in image_names]\nimages_jpeg = [plt.imread(image_path) for image_path in image_paths_jpeg]\n\nplot_images(images_dicom + images_jpeg)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:20:46.853295Z","iopub.execute_input":"2023-12-01T10:20:46.853758Z","iopub.status.idle":"2023-12-01T10:21:13.515389Z","shell.execute_reply.started":"2023-12-01T10:20:46.853731Z","shell.execute_reply":"2023-12-01T10:21:13.514070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot Random Benign and Malignant Lesions","metadata":{"execution":{"iopub.execute_input":"2023-09-23T16:38:03.870423Z","iopub.status.busy":"2023-09-23T16:38:03.869951Z","iopub.status.idle":"2023-09-23T16:38:26.956289Z","shell.execute_reply":"2023-09-23T16:38:26.955004Z","shell.execute_reply.started":"2023-09-23T16:38:03.870389Z"}}},{"cell_type":"code","source":"image_names = df_train[df_train['benign_malignant'] == 'benign'].sample(frac=1).iloc[:5]['image_name']\n\nimage_paths = [f'/kaggle/input/siim-isic-melanoma-classification/jpeg/train/{image_name}.jpg' for image_name in image_names]\nimages = [plt.imread(image_path) for image_path in image_paths]\n\nplot_images(images, title='Random Benign Lesions')","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:21:13.516670Z","iopub.execute_input":"2023-12-01T10:21:13.517530Z","iopub.status.idle":"2023-12-01T10:21:27.141472Z","shell.execute_reply.started":"2023-12-01T10:21:13.517485Z","shell.execute_reply":"2023-12-01T10:21:27.139240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_names = df_train[df_train['benign_malignant'] == 'malignant'].sample(frac=1).iloc[:5]['image_name']\n\nimage_paths = [f'/kaggle/input/siim-isic-melanoma-classification/jpeg/train/{image_name}.jpg' for image_name in image_names]\nimages = [plt.imread(image_path) for image_path in image_paths]\n\nplot_images(images, title='Random Malignant Lesions')","metadata":{"execution":{"iopub.status.busy":"2023-12-01T10:21:27.142872Z","iopub.execute_input":"2023-12-01T10:21:27.143199Z","iopub.status.idle":"2023-12-01T10:21:38.012950Z","shell.execute_reply.started":"2023-12-01T10:21:27.143172Z","shell.execute_reply":"2023-12-01T10:21:38.011835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}