{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":997.377033,"end_time":"2025-10-10T13:39:16.038002","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-10T13:22:38.660969","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"5db3822e","cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2025-10-10T13:22:42.272437Z","iopub.status.busy":"2025-10-10T13:22:42.271638Z","iopub.status.idle":"2025-10-10T13:24:47.175319Z","shell.execute_reply":"2025-10-10T13:24:47.174425Z"},"papermill":{"duration":124.913609,"end_time":"2025-10-10T13:24:47.176962","exception":false,"start_time":"2025-10-10T13:22:42.263353","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c1e448d1","cell_type":"code","source":"import os\nfrom tqdm import tqdm\nfrom os import listdir\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n#plotly\n!pip install chart_studio\nimport plotly.express as px\nimport chart_studio.plotly as py\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\n\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\n\n\n#pydicom\nimport pydicom\n\n# Suppress warnings \nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# Settings for pretty nice plots\nplt.style.use('fivethirtyeight')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:47.191628Z","iopub.status.busy":"2025-10-10T13:24:47.191280Z","iopub.status.idle":"2025-10-10T13:24:56.753265Z","shell.execute_reply":"2025-10-10T13:24:56.752594Z"},"papermill":{"duration":9.570714,"end_time":"2025-10-10T13:24:56.754766","exception":false,"start_time":"2025-10-10T13:24:47.184052","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a28ebd22","cell_type":"code","source":"print(os.listdir(\"../input/siim-isic-melanoma-classification\"))","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:56.770244Z","iopub.status.busy":"2025-10-10T13:24:56.769572Z","iopub.status.idle":"2025-10-10T13:24:56.775846Z","shell.execute_reply":"2025-10-10T13:24:56.775070Z"},"papermill":{"duration":0.014773,"end_time":"2025-10-10T13:24:56.776936","exception":false,"start_time":"2025-10-10T13:24:56.762163","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1f815288","cell_type":"code","source":"IMAGE_PATH = \"../input/siim-isic-melanoma-classification/\"\n\ntrain_df = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_df = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\n\n\n#Training data\nprint('Training data shape: ', train_df.shape)\ntrain_df.head(5)","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:56.791202Z","iopub.status.busy":"2025-10-10T13:24:56.790990Z","iopub.status.idle":"2025-10-10T13:24:56.910409Z","shell.execute_reply":"2025-10-10T13:24:56.909626Z"},"papermill":{"duration":0.127886,"end_time":"2025-10-10T13:24:56.911593","exception":false,"start_time":"2025-10-10T13:24:56.783707","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1190cc76","cell_type":"code","source":"train_df.groupby(['benign_malignant']).count()['sex'].to_frame()","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:56.926863Z","iopub.status.busy":"2025-10-10T13:24:56.926629Z","iopub.status.idle":"2025-10-10T13:24:56.949863Z","shell.execute_reply":"2025-10-10T13:24:56.949044Z"},"papermill":{"duration":0.032101,"end_time":"2025-10-10T13:24:56.951047","exception":false,"start_time":"2025-10-10T13:24:56.918946","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"28cf57b8","cell_type":"markdown","source":"## 3. Data Exploration","metadata":{"papermill":{"duration":0.006941,"end_time":"2025-10-10T13:24:56.965186","exception":false,"start_time":"2025-10-10T13:24:56.958245","status":"completed"},"tags":[]}},{"id":"f58f1724","cell_type":"markdown","source":"### Missing Values","metadata":{"papermill":{"duration":0.006675,"end_time":"2025-10-10T13:24:56.978743","exception":false,"start_time":"2025-10-10T13:24:56.972068","status":"completed"},"tags":[]}},{"id":"f0aee868","cell_type":"code","source":"print('Train Set')\nprint(train_df.info())\nprint('-------------')\nprint('Test Set')\nprint(test_df.info())","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:56.993121Z","iopub.status.busy":"2025-10-10T13:24:56.992919Z","iopub.status.idle":"2025-10-10T13:24:57.019978Z","shell.execute_reply":"2025-10-10T13:24:57.019096Z"},"papermill":{"duration":0.035505,"end_time":"2025-10-10T13:24:57.021057","exception":false,"start_time":"2025-10-10T13:24:56.985552","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7d64888a","cell_type":"markdown","source":"### Total Number of images","metadata":{"papermill":{"duration":0.006972,"end_time":"2025-10-10T13:24:57.035069","exception":false,"start_time":"2025-10-10T13:24:57.028097","status":"completed"},"tags":[]}},{"id":"bea8b913","cell_type":"code","source":"print(\"Total images in Train set: \",train_df['image_name'].count())\nprint(\"Total images in Test set: \",test_df['image_name'].count())","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:57.049991Z","iopub.status.busy":"2025-10-10T13:24:57.049523Z","iopub.status.idle":"2025-10-10T13:24:57.055512Z","shell.execute_reply":"2025-10-10T13:24:57.054808Z"},"papermill":{"duration":0.014535,"end_time":"2025-10-10T13:24:57.056587","exception":false,"start_time":"2025-10-10T13:24:57.042052","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"49068817","cell_type":"markdown","source":"### Unique Ids","metadata":{"papermill":{"duration":0.006938,"end_time":"2025-10-10T13:24:57.070634","exception":false,"start_time":"2025-10-10T13:24:57.063696","status":"completed"},"tags":[]}},{"id":"ffd2ff54","cell_type":"code","source":"print(f\"The total patient ids are {train_df['patient_id'].count()}, from those the unique ids are {train_df['patient_id'].value_counts().shape[0]} \")","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:57.085876Z","iopub.status.busy":"2025-10-10T13:24:57.085673Z","iopub.status.idle":"2025-10-10T13:24:57.093920Z","shell.execute_reply":"2025-10-10T13:24:57.093113Z"},"papermill":{"duration":0.016986,"end_time":"2025-10-10T13:24:57.094975","exception":false,"start_time":"2025-10-10T13:24:57.077989","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2641de83","cell_type":"code","source":"columns = train_df.keys()\ncolumns = list(columns)\nprint(columns)","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:57.109739Z","iopub.status.busy":"2025-10-10T13:24:57.109544Z","iopub.status.idle":"2025-10-10T13:24:57.113319Z","shell.execute_reply":"2025-10-10T13:24:57.112611Z"},"papermill":{"duration":0.012369,"end_time":"2025-10-10T13:24:57.114368","exception":false,"start_time":"2025-10-10T13:24:57.101999","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"86ef2f02","cell_type":"markdown","source":"### Exploring the Target Column","metadata":{"papermill":{"duration":0.006938,"end_time":"2025-10-10T13:24:57.128327","exception":false,"start_time":"2025-10-10T13:24:57.121389","status":"completed"},"tags":[]}},{"id":"e9350802","cell_type":"code","source":"train_df['target'].value_counts()\n","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:57.143539Z","iopub.status.busy":"2025-10-10T13:24:57.143335Z","iopub.status.idle":"2025-10-10T13:24:57.150185Z","shell.execute_reply":"2025-10-10T13:24:57.149543Z"},"papermill":{"duration":0.015923,"end_time":"2025-10-10T13:24:57.151291","exception":false,"start_time":"2025-10-10T13:24:57.135368","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"5dbb8fe0","cell_type":"code","source":"train_df['target'].value_counts(normalize=True).iplot(kind='bar',\n                                                      yTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='red',\n                                                      theme='pearl',\n                                                      bargap=0.8,\n                                                      gridcolor='white',\n                                                     \n                                                      title='Distribution of the Target column in the training set')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:57.166630Z","iopub.status.busy":"2025-10-10T13:24:57.166031Z","iopub.status.idle":"2025-10-10T13:24:58.073942Z","shell.execute_reply":"2025-10-10T13:24:58.073295Z"},"papermill":{"duration":0.916995,"end_time":"2025-10-10T13:24:58.075405","exception":false,"start_time":"2025-10-10T13:24:57.158410","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e9ba7ada","cell_type":"markdown","source":"### Gender wise distribution","metadata":{"papermill":{"duration":0.01008,"end_time":"2025-10-10T13:24:58.096471","exception":false,"start_time":"2025-10-10T13:24:58.086391","status":"completed"},"tags":[]}},{"id":"e4821854","cell_type":"code","source":"train_df['sex'].value_counts(normalize=True)","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:58.117455Z","iopub.status.busy":"2025-10-10T13:24:58.116896Z","iopub.status.idle":"2025-10-10T13:24:58.124483Z","shell.execute_reply":"2025-10-10T13:24:58.123909Z"},"papermill":{"duration":0.019109,"end_time":"2025-10-10T13:24:58.125501","exception":false,"start_time":"2025-10-10T13:24:58.106392","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"f963e8d3","cell_type":"code","source":"train_df['sex'].value_counts(normalize=True).iplot(kind='bar',\n                                                      yTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='green',\n                                                      theme='pearl',\n                                                      bargap=0.8,\n                                                      gridcolor='white',\n                                                     \n                                                      title='Distribution of the Sex column in the training set')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:58.146586Z","iopub.status.busy":"2025-10-10T13:24:58.146118Z","iopub.status.idle":"2025-10-10T13:24:58.174951Z","shell.execute_reply":"2025-10-10T13:24:58.174325Z"},"papermill":{"duration":0.040499,"end_time":"2025-10-10T13:24:58.176109","exception":false,"start_time":"2025-10-10T13:24:58.135610","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ef63cb28","cell_type":"markdown","source":"### Gender V/s Target","metadata":{"papermill":{"duration":0.013185,"end_time":"2025-10-10T13:24:58.203078","exception":false,"start_time":"2025-10-10T13:24:58.189893","status":"completed"},"tags":[]}},{"id":"e8d3996f","cell_type":"code","source":"z=train_df.groupby(['target','sex'])['benign_malignant'].count().to_frame().reset_index()\nz.style.background_gradient(cmap='Reds')  ","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:58.229780Z","iopub.status.busy":"2025-10-10T13:24:58.229600Z","iopub.status.idle":"2025-10-10T13:24:58.398249Z","shell.execute_reply":"2025-10-10T13:24:58.397460Z"},"papermill":{"duration":0.183424,"end_time":"2025-10-10T13:24:58.399405","exception":false,"start_time":"2025-10-10T13:24:58.215981","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4b8ff58c","cell_type":"code","source":"sns.catplot(x='target',y='benign_malignant', hue='sex',data=z,kind='bar')\nplt.ylabel('Count')\nplt.xlabel('benign:0 vs malignant:1')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:58.427013Z","iopub.status.busy":"2025-10-10T13:24:58.426278Z","iopub.status.idle":"2025-10-10T13:24:58.812181Z","shell.execute_reply":"2025-10-10T13:24:58.811398Z"},"papermill":{"duration":0.400849,"end_time":"2025-10-10T13:24:58.813556","exception":false,"start_time":"2025-10-10T13:24:58.412707","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"6ccd99cd","cell_type":"markdown","source":"### Localisation of image site","metadata":{"papermill":{"duration":0.013871,"end_time":"2025-10-10T13:24:58.842060","exception":false,"start_time":"2025-10-10T13:24:58.828189","status":"completed"},"tags":[]}},{"id":"6840986e","cell_type":"code","source":"train_df['anatom_site_general_challenge'].value_counts(normalize=True).sort_values()","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:58.871959Z","iopub.status.busy":"2025-10-10T13:24:58.871295Z","iopub.status.idle":"2025-10-10T13:24:58.879270Z","shell.execute_reply":"2025-10-10T13:24:58.878648Z"},"papermill":{"duration":0.02405,"end_time":"2025-10-10T13:24:58.880235","exception":false,"start_time":"2025-10-10T13:24:58.856185","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"fb59b9f2","cell_type":"code","source":"train_df['anatom_site_general_challenge'].value_counts(normalize=True).sort_values().iplot(kind='barh',\n                                                      xTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='#FB8072',\n                                                      theme='pearl',\n                                                      bargap=0.2,\n                                                      gridcolor='white',\n                                                      title='Distribution of the imaged site in the training set')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:58.908037Z","iopub.status.busy":"2025-10-10T13:24:58.907824Z","iopub.status.idle":"2025-10-10T13:24:58.938500Z","shell.execute_reply":"2025-10-10T13:24:58.937758Z"},"papermill":{"duration":0.045863,"end_time":"2025-10-10T13:24:58.939559","exception":false,"start_time":"2025-10-10T13:24:58.893696","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7a8f3e0a","cell_type":"markdown","source":"### Location of imaged site w.r.t gender","metadata":{"papermill":{"duration":0.016073,"end_time":"2025-10-10T13:24:58.972331","exception":false,"start_time":"2025-10-10T13:24:58.956258","status":"completed"},"tags":[]}},{"id":"57fb01b4","cell_type":"code","source":"z1=train_df.groupby(['sex','anatom_site_general_challenge'])['benign_malignant'].count().to_frame().reset_index()\nz1.style.background_gradient(cmap='Reds')\nsns.catplot(x='anatom_site_general_challenge',y='benign_malignant', hue='sex',data=z1,kind='bar')\nplt.gcf().set_size_inches(10,8)\nplt.xlabel('location of imaged site')\nplt.xticks(rotation=45,fontsize='10', horizontalalignment='right')\nplt.ylabel('count of melanoma cases')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:59.006080Z","iopub.status.busy":"2025-10-10T13:24:59.005457Z","iopub.status.idle":"2025-10-10T13:24:59.402326Z","shell.execute_reply":"2025-10-10T13:24:59.401543Z"},"papermill":{"duration":0.415,"end_time":"2025-10-10T13:24:59.403530","exception":false,"start_time":"2025-10-10T13:24:58.988530","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"343b4f35","cell_type":"markdown","source":"### Age Distribution of patients","metadata":{"papermill":{"duration":0.017079,"end_time":"2025-10-10T13:24:59.438802","exception":false,"start_time":"2025-10-10T13:24:59.421723","status":"completed"},"tags":[]}},{"id":"5d769971","cell_type":"code","source":"train_df['age_approx'].iplot(kind='hist',bins=30,color='orange',xTitle='Age distribution',yTitle='Count')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:24:59.474227Z","iopub.status.busy":"2025-10-10T13:24:59.473885Z","iopub.status.idle":"2025-10-10T13:24:59.659047Z","shell.execute_reply":"2025-10-10T13:24:59.658458Z"},"papermill":{"duration":0.216898,"end_time":"2025-10-10T13:24:59.672692","exception":false,"start_time":"2025-10-10T13:24:59.455794","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2c3db8c8","cell_type":"markdown","source":"## Visualising Age KDEs","metadata":{"papermill":{"duration":0.058005,"end_time":"2025-10-10T13:24:59.788891","exception":false,"start_time":"2025-10-10T13:24:59.730886","status":"completed"},"tags":[]}},{"id":"c66d1f0b","cell_type":"markdown","source":"### Distribution of Ages w.r.t Target¶","metadata":{"papermill":{"duration":0.054344,"end_time":"2025-10-10T13:24:59.898318","exception":false,"start_time":"2025-10-10T13:24:59.843974","status":"completed"},"tags":[]}},{"id":"d4be7b9d","cell_type":"code","source":"# KDE plot of age that were diagnosed as benign\nsns.kdeplot(train_df.loc[train_df['target'] == 0, 'age_approx'], label = 'Benign',shade=True)\n\n# KDE plot of age that were diagnosed as malignant\nsns.kdeplot(train_df.loc[train_df['target'] == 1, 'age_approx'], label = 'Malignant',shade=True)\n\n# Labeling of plot\nplt.xlabel('Age (years)'); plt.ylabel('Density'); plt.title('Distribution of Ages');","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:00.009100Z","iopub.status.busy":"2025-10-10T13:25:00.008366Z","iopub.status.idle":"2025-10-10T13:25:00.469454Z","shell.execute_reply":"2025-10-10T13:25:00.468732Z"},"papermill":{"duration":0.51854,"end_time":"2025-10-10T13:25:00.471273","exception":false,"start_time":"2025-10-10T13:24:59.952733","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"119417b6","cell_type":"markdown","source":"### Distribution of Ages w.r.t gender","metadata":{"papermill":{"duration":0.056283,"end_time":"2025-10-10T13:25:00.586338","exception":false,"start_time":"2025-10-10T13:25:00.530055","status":"completed"},"tags":[]}},{"id":"81dc68e3","cell_type":"code","source":"# KDE plot of age that were diagnosed as benign\nsns.kdeplot(train_df.loc[train_df['sex'] == 'male', 'age_approx'], label = 'Male',shade=True)\n\n# KDE plot of age that were diagnosed as malignant\nsns.kdeplot(train_df.loc[train_df['sex'] == 'female', 'age_approx'], label = 'Female',shade=True)\n\n# Labeling of plot\nplt.xlabel('Age (years)'); plt.ylabel('Density'); plt.title('Distribution of Ages');","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:00.700832Z","iopub.status.busy":"2025-10-10T13:25:00.700568Z","iopub.status.idle":"2025-10-10T13:25:01.144003Z","shell.execute_reply":"2025-10-10T13:25:01.143322Z"},"papermill":{"duration":0.50249,"end_time":"2025-10-10T13:25:01.145398","exception":false,"start_time":"2025-10-10T13:25:00.642908","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"bf688fbd","cell_type":"markdown","source":"### Distribution of Diagnosis","metadata":{"papermill":{"duration":0.056045,"end_time":"2025-10-10T13:25:01.258741","exception":false,"start_time":"2025-10-10T13:25:01.202696","status":"completed"},"tags":[]}},{"id":"16f94641","cell_type":"code","source":"train_df['diagnosis'].value_counts()\n","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:01.371863Z","iopub.status.busy":"2025-10-10T13:25:01.371401Z","iopub.status.idle":"2025-10-10T13:25:01.379084Z","shell.execute_reply":"2025-10-10T13:25:01.378373Z"},"papermill":{"duration":0.06543,"end_time":"2025-10-10T13:25:01.380233","exception":false,"start_time":"2025-10-10T13:25:01.314803","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"beb4fa88","cell_type":"code","source":"train_df['diagnosis'].value_counts(normalize=True).sort_values().iplot(kind='barh',\n                                                      xTitle='Percentage', \n                                                      linecolor='black', \n                                                      opacity=0.7,\n                                                      color='blue',\n                                                      theme='pearl',\n                                                      bargap=0.2,\n                                                      gridcolor='white',\n                                                      title='Distribution in the training set')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:01.493528Z","iopub.status.busy":"2025-10-10T13:25:01.493288Z","iopub.status.idle":"2025-10-10T13:25:01.532330Z","shell.execute_reply":"2025-10-10T13:25:01.531571Z"},"papermill":{"duration":0.097263,"end_time":"2025-10-10T13:25:01.533484","exception":false,"start_time":"2025-10-10T13:25:01.436221","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"dbefa111","cell_type":"markdown","source":"### Patient Overlap","metadata":{"papermill":{"duration":0.059169,"end_time":"2025-10-10T13:25:01.654640","exception":false,"start_time":"2025-10-10T13:25:01.595471","status":"completed"},"tags":[]}},{"id":"729106d6","cell_type":"code","source":"# Extract patient id's for the training set\nids_train = train_df.patient_id.values\n# Extract patient id's for the validation set\nids_test = test_df.patient_id.values\n\n# Create a \"set\" datastructure of the training set id's to identify unique id's\nids_train_set = set(ids_train)\nprint(f'There are {len(ids_train_set)} unique Patient IDs in the training set')\n# Create a \"set\" datastructure of the validation set id's to identify unique id's\nids_test_set = set(ids_test)\nprint(f'There are {len(ids_test_set)} unique Patient IDs in the training set')\n\n# Identify patient overlap by looking at the intersection between the sets\npatient_overlap = list(ids_train_set.intersection(ids_test_set))\nn_overlap = len(patient_overlap)\nprint(f'There are {n_overlap} Patient IDs in both the training and test sets')\nprint('')\nprint(f'These patients are in both the training and test datasets:')\nprint(f'{patient_overlap}')","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:01.776515Z","iopub.status.busy":"2025-10-10T13:25:01.776223Z","iopub.status.idle":"2025-10-10T13:25:01.783445Z","shell.execute_reply":"2025-10-10T13:25:01.782693Z"},"papermill":{"duration":0.070993,"end_time":"2025-10-10T13:25:01.784421","exception":false,"start_time":"2025-10-10T13:25:01.713428","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"4fac723a","cell_type":"markdown","source":"## 4. Visualising Images : JPEG","metadata":{"papermill":{"duration":0.058816,"end_time":"2025-10-10T13:25:01.908098","exception":false,"start_time":"2025-10-10T13:25:01.849282","status":"completed"},"tags":[]}},{"id":"bf5372a3","cell_type":"markdown","source":"### Visualizing a random selection of images","metadata":{"papermill":{"duration":0.059795,"end_time":"2025-10-10T13:25:02.100813","exception":false,"start_time":"2025-10-10T13:25:02.041018","status":"completed"},"tags":[]}},{"id":"756c1ef5","cell_type":"code","source":"images = train_df['image_name'].values\n\n# Extract 9 random images from it\nrandom_images = [np.random.choice(images+'.jpg') for i in range(9)]\n\n# Location of the image dir\nimg_dir = IMAGE_PATH+'/jpeg/train'\n\nprint('Display Random Images')\n\n# Adjust the size of your images\nplt.figure(figsize=(10,8))\n\n# Iterate and plot random images\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    img = plt.imread(os.path.join(img_dir, random_images[i]))\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    \n# Adjust subplot parameters to give specified padding\nplt.tight_layout() ","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:02.220249Z","iopub.status.busy":"2025-10-10T13:25:02.219227Z","iopub.status.idle":"2025-10-10T13:25:28.002103Z","shell.execute_reply":"2025-10-10T13:25:28.001319Z"},"papermill":{"duration":25.855876,"end_time":"2025-10-10T13:25:28.015576","exception":false,"start_time":"2025-10-10T13:25:02.159700","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"00ff556e","cell_type":"markdown","source":"### Visualizing Images with benign lesions","metadata":{"papermill":{"duration":0.07493,"end_time":"2025-10-10T13:25:28.168757","exception":false,"start_time":"2025-10-10T13:25:28.093827","status":"completed"},"tags":[]}},{"id":"d58edeb6","cell_type":"code","source":"benign = train_df[train_df['benign_malignant']=='benign']\nmalignant = train_df[train_df['benign_malignant']=='malignant']","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:28.313499Z","iopub.status.busy":"2025-10-10T13:25:28.312943Z","iopub.status.idle":"2025-10-10T13:25:28.323966Z","shell.execute_reply":"2025-10-10T13:25:28.323413Z"},"papermill":{"duration":0.08466,"end_time":"2025-10-10T13:25:28.325124","exception":false,"start_time":"2025-10-10T13:25:28.240464","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"518c53ab","cell_type":"code","source":"images = benign['image_name'].values\n\n# Extract 9 random images from it\nrandom_images = [np.random.choice(images+'.jpg') for i in range(9)]\n\n# Location of the image dir\nimg_dir = IMAGE_PATH+'/jpeg/train'\n\nprint('Display benign Images')\n\n# Adjust the size of your images\nplt.figure(figsize=(10,8))\n\n# Iterate and plot random images\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    img = plt.imread(os.path.join(img_dir, random_images[i]))\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    \n# Adjust subplot parameters to give specified padding\nplt.tight_layout()   ","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:28.472639Z","iopub.status.busy":"2025-10-10T13:25:28.471842Z","iopub.status.idle":"2025-10-10T13:25:44.214099Z","shell.execute_reply":"2025-10-10T13:25:44.213419Z"},"papermill":{"duration":15.829306,"end_time":"2025-10-10T13:25:44.228415","exception":false,"start_time":"2025-10-10T13:25:28.399109","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"191041cc","cell_type":"markdown","source":"### Visualizing Images with Malignant lesions","metadata":{"papermill":{"duration":0.089073,"end_time":"2025-10-10T13:25:44.410108","exception":false,"start_time":"2025-10-10T13:25:44.321035","status":"completed"},"tags":[]}},{"id":"6f17a68e","cell_type":"code","source":"images = malignant['image_name'].values\n\n# Extract 9 random images from it\nrandom_images = [np.random.choice(images+'.jpg') for i in range(9)]\n\n# Location of the image dir\nimg_dir = IMAGE_PATH+'/jpeg/train'\n\nprint('Display malignant Images')\n\n# Adjust the size of your images\nplt.figure(figsize=(10,8))\n\n# Iterate and plot random images\nfor i in range(9):\n    plt.subplot(3, 3, i + 1)\n    img = plt.imread(os.path.join(img_dir, random_images[i]))\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    \n# Adjust subplot parameters to give specified padding\nplt.tight_layout()   ","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:44.584385Z","iopub.status.busy":"2025-10-10T13:25:44.583704Z","iopub.status.idle":"2025-10-10T13:25:52.828887Z","shell.execute_reply":"2025-10-10T13:25:52.828072Z"},"papermill":{"duration":8.346535,"end_time":"2025-10-10T13:25:52.842495","exception":false,"start_time":"2025-10-10T13:25:44.495960","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"68866aad","cell_type":"markdown","source":"### Histograms","metadata":{"papermill":{"duration":0.101651,"end_time":"2025-10-10T13:25:53.058021","exception":false,"start_time":"2025-10-10T13:25:52.956370","status":"completed"},"tags":[]}},{"id":"6a3c0853","cell_type":"markdown","source":"#### Benign category","metadata":{"papermill":{"duration":0.102045,"end_time":"2025-10-10T13:25:53.260552","exception":false,"start_time":"2025-10-10T13:25:53.158507","status":"completed"},"tags":[]}},{"id":"646faff1","cell_type":"code","source":"f = plt.figure(figsize=(16,8))\nf.add_subplot(1,2, 1)\n\nsample_img = benign['image_name'][0]+'.jpg'\nraw_image = plt.imread(os.path.join(img_dir, sample_img))\nplt.imshow(raw_image, cmap='gray')\nplt.colorbar()\nplt.title('Benign Image')\nprint(f\"Image dimensions:  {raw_image.shape[0],raw_image.shape[1]}\")\nprint(f\"Maximum pixel value : {raw_image.max():.1f} ; Minimum pixel value:{raw_image.min():.1f}\")\nprint(f\"Mean value of the pixels : {raw_image.mean():.1f} ; Standard deviation : {raw_image.std():.1f}\")\n\nf.add_subplot(1,2, 2)\n\n#_ = plt.hist(raw_image.ravel(),bins = 256, color = 'orange',)\n_ = plt.hist(raw_image[:, :, 0].ravel(), bins = 256, color = 'red', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 1].ravel(), bins = 256, color = 'Green', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 2].ravel(), bins = 256, color = 'Blue', alpha = 0.5)\n_ = plt.xlabel('Intensity Value')\n_ = plt.ylabel('Count')\n_ = plt.legend(['Red_Channel', 'Green_Channel', 'Blue_Channel'])\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:53.468741Z","iopub.status.busy":"2025-10-10T13:25:53.468467Z","iopub.status.idle":"2025-10-10T13:25:58.806236Z","shell.execute_reply":"2025-10-10T13:25:58.805418Z"},"papermill":{"duration":5.45205,"end_time":"2025-10-10T13:25:58.812676","exception":false,"start_time":"2025-10-10T13:25:53.360626","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"505612c8","cell_type":"markdown","source":"#### Malignant category","metadata":{"papermill":{"duration":0.11045,"end_time":"2025-10-10T13:25:59.043551","exception":false,"start_time":"2025-10-10T13:25:58.933101","status":"completed"},"tags":[]}},{"id":"7ad9878b","cell_type":"code","source":"f = plt.figure(figsize=(16,8))\nf.add_subplot(1,2, 1)\n\nsample_img = malignant['image_name'][235]+'.jpg'\nraw_image = plt.imread(os.path.join(img_dir, sample_img))\nplt.imshow(raw_image, cmap='gray')\nplt.colorbar()\nplt.title('Malignant Image')\nprint(f\"Image dimensions:  {raw_image.shape[0],raw_image.shape[1]}\")\nprint(f\"Maximum pixel value : {raw_image.max():.1f} ; Minimum pixel value:{raw_image.min():.1f}\")\nprint(f\"Mean value of the pixels : {raw_image.mean():.1f} ; Standard deviation : {raw_image.std():.1f}\")\n\nf.add_subplot(1,2, 2)\n\n#_ = plt.hist(raw_image.ravel(),bins = 256, color = 'orange',)\n_ = plt.hist(raw_image[:, :, 0].ravel(), bins = 256, color = 'red', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 1].ravel(), bins = 256, color = 'Green', alpha = 0.5)\n_ = plt.hist(raw_image[:, :, 2].ravel(), bins = 256, color = 'Blue', alpha = 0.5)\n_ = plt.xlabel('Intensity Value')\n_ = plt.ylabel('Count')\n_ = plt.legend(['Red_Channel', 'Green_Channel', 'Blue_Channel'])\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:25:59.256599Z","iopub.status.busy":"2025-10-10T13:25:59.255828Z","iopub.status.idle":"2025-10-10T13:26:02.074242Z","shell.execute_reply":"2025-10-10T13:26:02.073564Z"},"papermill":{"duration":2.930486,"end_time":"2025-10-10T13:26:02.081087","exception":false,"start_time":"2025-10-10T13:25:59.150601","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3517de45","cell_type":"markdown","source":"## 5 Preprocessing DIOCOM files","metadata":{"papermill":{"duration":0.113915,"end_time":"2025-10-10T13:26:02.314979","exception":false,"start_time":"2025-10-10T13:26:02.201064","status":"completed"},"tags":[]}},{"id":"bfc0e196","cell_type":"code","source":"print (pydicom.__version__)\n","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:26:02.539641Z","iopub.status.busy":"2025-10-10T13:26:02.538986Z","iopub.status.idle":"2025-10-10T13:26:02.543293Z","shell.execute_reply":"2025-10-10T13:26:02.542457Z"},"papermill":{"duration":0.118247,"end_time":"2025-10-10T13:26:02.544324","exception":false,"start_time":"2025-10-10T13:26:02.426077","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0a48a48a","cell_type":"code","source":"# https://www.kaggle.com/schlerp/getting-to-know-dicom-and-the-data\ndef show_dcm_info(dataset):\n    print(\"Filename.........:\", file_path)\n    print(\"Storage type.....:\", dataset.SOPClassUID)\n    print()\n\n    pat_name = dataset.PatientName\n    display_name = pat_name.family_name + \", \" + pat_name.given_name\n    print(\"Patient's name......:\", display_name)\n    print(\"Patient id..........:\", dataset.PatientID)\n    print(\"Patient's Age.......:\", dataset.PatientAge)\n    print(\"Patient's Sex.......:\", dataset.PatientSex)\n    print(\"Modality............:\", dataset.Modality)\n    print(\"Body Part Examined..:\", dataset.BodyPartExamined)\n   \n    \n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print(\"Pixel spacing....:\", dataset.PixelSpacing)","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:26:02.768315Z","iopub.status.busy":"2025-10-10T13:26:02.767853Z","iopub.status.idle":"2025-10-10T13:26:02.773573Z","shell.execute_reply":"2025-10-10T13:26:02.772794Z"},"papermill":{"duration":0.118892,"end_time":"2025-10-10T13:26:02.774713","exception":false,"start_time":"2025-10-10T13:26:02.655821","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"dcbb046a","cell_type":"code","source":"def plot_pixel_array(dataset, figsize=(5,5)):\n    plt.figure(figsize=figsize)\n    plt.grid(False)\n    plt.imshow(dataset.pixel_array)\n    plt.show()\n    \ni = 1\nnum_to_plot = 5\nfor file_name in os.listdir('../input/siim-isic-melanoma-classification/train/'):\n        file_path = os.path.join('../input/siim-isic-melanoma-classification/train/',file_name)\n        dataset = pydicom.dcmread(file_path)\n        show_dcm_info(dataset)\n        plot_pixel_array(dataset)\n    \n        if i >= num_to_plot:\n            break\n    \n        i += 1","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:26:03.000094Z","iopub.status.busy":"2025-10-10T13:26:02.999239Z","iopub.status.idle":"2025-10-10T13:26:11.626497Z","shell.execute_reply":"2025-10-10T13:26:11.625715Z"},"papermill":{"duration":8.744033,"end_time":"2025-10-10T13:26:11.630093","exception":false,"start_time":"2025-10-10T13:26:02.886060","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"56f90379","cell_type":"markdown","source":"### Extracting DIOCOM files information in a dataframe","metadata":{"papermill":{"duration":0.128769,"end_time":"2025-10-10T13:26:11.896462","exception":false,"start_time":"2025-10-10T13:26:11.767693","status":"completed"},"tags":[]}},{"id":"6c8511f3","cell_type":"code","source":"# source: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/154658\nfolder='train'\nPATH='../input/siim-isic-melanoma-classification/'\n\ndef extract_DICOM_attributes(folder):\n    images = list(os.listdir(os.path.join(PATH, folder)))\n    df = pd.DataFrame()\n    for image in tqdm(images):\n        image_name = image.split(\".\")[0]\n        dicom_file_path = os.path.join(PATH,folder,image)\n        dicom_file_dataset = pydicom.dcmread(dicom_file_path)\n        study_date = dicom_file_dataset.StudyDate\n        modality = dicom_file_dataset.Modality\n        age = dicom_file_dataset.PatientAge\n        sex = dicom_file_dataset.PatientSex\n        body_part_examined = dicom_file_dataset.BodyPartExamined\n        patient_orientation = dicom_file_dataset.PatientOrientation\n        photometric_interpretation = dicom_file_dataset.PhotometricInterpretation\n        rows = dicom_file_dataset.Rows\n        columns = dicom_file_dataset.Columns\n\n        df = pd.concat([\n            df,pd.DataFrame(\n                {\n            'image_name': image_name, \n            'dcm_modality': modality,\n            'dcm_study_date':study_date, \n            'dcm_age': age, \n            'dcm_sex': sex,            \n            'dcm_body_part_examined': body_part_examined,\n            'dcm_patient_orientation': patient_orientation,            \n            'dcm_photometric_interpretation': photometric_interpretation,           \n            'dcm_rows': rows, \n            'dcm_columns': columns\n        }, \n                                    \n                index=[0],\n                )\n                       ])\n    return df","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:26:12.150856Z","iopub.status.busy":"2025-10-10T13:26:12.150098Z","iopub.status.idle":"2025-10-10T13:26:12.157167Z","shell.execute_reply":"2025-10-10T13:26:12.156542Z"},"papermill":{"duration":0.134736,"end_time":"2025-10-10T13:26:12.158299","exception":false,"start_time":"2025-10-10T13:26:12.023563","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"29bd5427","cell_type":"code","source":"extract_DICOM_attributes('train')\n","metadata":{"execution":{"iopub.execute_input":"2025-10-10T13:26:12.412244Z","iopub.status.busy":"2025-10-10T13:26:12.411413Z","iopub.status.idle":"2025-10-10T13:39:14.625185Z","shell.execute_reply":"2025-10-10T13:39:14.624288Z"},"papermill":{"duration":782.342582,"end_time":"2025-10-10T13:39:14.628024","exception":false,"start_time":"2025-10-10T13:26:12.285442","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}