{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20270,"databundleVersionId":1222630,"sourceType":"competition"},{"sourceId":63056,"databundleVersionId":8940774,"sourceType":"competition"},{"sourceId":1193409,"sourceType":"datasetVersion","datasetId":679322}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport shutil\nimport warnings\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nsns.set_style(\"whitegrid\")\nwarnings.filterwarnings('ignore')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-06-30T17:20:29.084764Z","iopub.execute_input":"2024-06-30T17:20:29.085720Z","iopub.status.idle":"2024-06-30T17:20:29.092427Z","shell.execute_reply.started":"2024-06-30T17:20:29.085686Z","shell.execute_reply":"2024-06-30T17:20:29.091198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SRC_DIR19 = '/kaggle/input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input'\nSRC_DIR20 = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train'\nSRC_DIR24 = '/kaggle/input/isic-2024-challenge/train-image/image'\nRESULT_DIR = '/kaggle/working/isic'\nos.makedirs(RESULT_DIR, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:20:29.258733Z","iopub.execute_input":"2024-06-30T17:20:29.259163Z","iopub.status.idle":"2024-06-30T17:20:29.268326Z","shell.execute_reply.started":"2024-06-30T17:20:29.259130Z","shell.execute_reply":"2024-06-30T17:20:29.267147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load and Merge metadata","metadata":{}},{"cell_type":"markdown","source":"## Preprocess ISIC 2024","metadata":{}},{"cell_type":"code","source":"df_train24 = pd.read_csv('/kaggle/input/isic-2024-challenge/train-metadata.csv', index_col='isic_id')\ndf_train24['source'] = 'ISIC24'\ndf_train24","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:21:33.630962Z","iopub.execute_input":"2024-06-30T17:21:33.631359Z","iopub.status.idle":"2024-06-30T17:21:39.137761Z","shell.execute_reply.started":"2024-06-30T17:21:33.631329Z","shell.execute_reply":"2024-06-30T17:21:39.136543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = cv2.imread(f'/kaggle/input/isic-2024-challenge/train-image/image/ISIC_0052109.jpg', cv2.IMREAD_COLOR)\nplt.imshow(image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:21:39.139778Z","iopub.execute_input":"2024-06-30T17:21:39.140174Z","iopub.status.idle":"2024-06-30T17:21:39.424547Z","shell.execute_reply.started":"2024-06-30T17:21:39.140141Z","shell.execute_reply":"2024-06-30T17:21:39.423345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess ISIC 2020","metadata":{}},{"cell_type":"code","source":"df_train20 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv', index_col='image_name')\ndf_train20['source'] = 'ISIC20'\ndf_train20","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:21:39.426622Z","iopub.execute_input":"2024-06-30T17:21:39.427295Z","iopub.status.idle":"2024-06-30T17:21:39.502106Z","shell.execute_reply.started":"2024-06-30T17:21:39.427254Z","shell.execute_reply":"2024-06-30T17:21:39.500957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = cv2.imread(f'/kaggle/input/siim-isic-melanoma-classification/jpeg/train/ISIC_0082543.jpg', cv2.IMREAD_COLOR)\nplt.imshow(image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:21:39.504634Z","iopub.execute_input":"2024-06-30T17:21:39.505062Z","iopub.status.idle":"2024-06-30T17:21:43.275041Z","shell.execute_reply.started":"2024-06-30T17:21:39.505021Z","shell.execute_reply":"2024-06-30T17:21:43.273807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess ISIC 2019\n**2019 versions contains datasets from 2019, 2018 and 2017.**","metadata":{}},{"cell_type":"code","source":"df_train19 = pd.read_csv('/kaggle/input/isic-2019/ISIC_2019_Training_GroundTruth.csv', index_col='image')\ndf_meta19 = pd.read_csv('/kaggle/input/isic-2019/ISIC_2019_Training_Metadata.csv', index_col='image')\ndf_meta19['target'] = df_train19['MEL'].apply(lambda x: int(x))\ndf_meta19['source'] = 'ISIC19'\ndf_meta19","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:22:23.518111Z","iopub.execute_input":"2024-06-30T17:22:23.518933Z","iopub.status.idle":"2024-06-30T17:22:23.628191Z","shell.execute_reply.started":"2024-06-30T17:22:23.518897Z","shell.execute_reply":"2024-06-30T17:22:23.627092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = cv2.imread(f'/kaggle/input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/ISIC_0000016.jpg', cv2.IMREAD_COLOR)\nplt.imshow(image)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:22:27.958824Z","iopub.execute_input":"2024-06-30T17:22:27.959873Z","iopub.status.idle":"2024-06-30T17:22:28.405221Z","shell.execute_reply.started":"2024-06-30T17:22:27.959826Z","shell.execute_reply":"2024-06-30T17:22:28.404094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merge metadata","metadata":{}},{"cell_type":"code","source":"dataset = pd.concat([df_meta19, df_train20, df_train24])\ndataset","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:22:29.959097Z","iopub.execute_input":"2024-06-30T17:22:29.959511Z","iopub.status.idle":"2024-06-30T17:22:31.347675Z","shell.execute_reply.started":"2024-06-30T17:22:29.959481Z","shell.execute_reply":"2024-06-30T17:22:31.346554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset['target'] = dataset['target'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:22:31.349335Z","iopub.execute_input":"2024-06-30T17:22:31.349648Z","iopub.status.idle":"2024-06-30T17:22:31.360099Z","shell.execute_reply.started":"2024-06-30T17:22:31.349611Z","shell.execute_reply":"2024-06-30T17:22:31.359026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.to_csv(os.path.join(RESULT_DIR, 'merged_metadata.csv'))","metadata":{"execution":{"iopub.status.busy":"2024-06-30T17:22:43.230184Z","iopub.execute_input":"2024-06-30T17:22:43.230573Z","iopub.status.idle":"2024-06-30T17:23:06.055877Z","shell.execute_reply.started":"2024-06-30T17:22:43.230541Z","shell.execute_reply":"2024-06-30T17:23:06.054575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Collect image files","metadata":{}},{"cell_type":"code","source":"# KAGGLE KERNEL DOESN'T HAVE ENOUGH MEMORY SPACE\n\n# DST_DIR_ISIC19 = os.path.join(RESULT_DIR, 'train/isic2019')\n# DST_DIR_ISIC20 = os.path.join(RESULT_DIR, 'train/isic2020')\n# DST_DIR_ISIC24 = os.path.join(RESULT_DIR, 'train/isic2024')\n\n# for dst_dir, src_dir in [(DST_DIR_ISIC19, SRC_DIR19), (DST_DIR_ISIC20, SRC_DIR20), (DST_DIR_ISIC24, SRC_DIR24)]:\n#     shutil.copytree(src_dir, dst_dir)   \n#     print(dst_dir, 'moved')\n\n# shutil.make_archive(\"skin_cancer_detection_dataset\", \"zip\", \"/kaggle/working/\")","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:45.454870Z","iopub.execute_input":"2024-06-30T16:47:45.455196Z","iopub.status.idle":"2024-06-30T16:47:45.459480Z","shell.execute_reply.started":"2024-06-30T16:47:45.455168Z","shell.execute_reply":"2024-06-30T16:47:45.458318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Simple EDA","metadata":{}},{"cell_type":"code","source":"sns.histplot(data=dataset, x='target', hue='source', multiple='stack', kde=False)\n\nplt.title('Histogram of Target Values Grouped by Source')\nplt.xlabel('Target')\nplt.ylabel('Count')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:45.461188Z","iopub.execute_input":"2024-06-30T16:47:45.461533Z","iopub.status.idle":"2024-06-30T16:47:47.048432Z","shell.execute_reply.started":"2024-06-30T16:47:45.461505Z","shell.execute_reply":"2024-06-30T16:47:47.047347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.groupby(['source', 'target']).size()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:47.052382Z","iopub.execute_input":"2024-06-30T16:47:47.052690Z","iopub.status.idle":"2024-06-30T16:47:47.105654Z","shell.execute_reply.started":"2024-06-30T16:47:47.052664Z","shell.execute_reply":"2024-06-30T16:47:47.104552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset['age_approx'].hist(bins=10);","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:47.106923Z","iopub.execute_input":"2024-06-30T16:47:47.107223Z","iopub.status.idle":"2024-06-30T16:47:47.474928Z","shell.execute_reply.started":"2024-06-30T16:47:47.107197Z","shell.execute_reply":"2024-06-30T16:47:47.473823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=dataset.dropna(subset=['age_approx']), x='age_approx', hue='source', multiple='stack', bins=10, kde=False)\n\nplt.title('Histogram of Target Values Grouped by Source')\nplt.xlabel('Target')\nplt.ylabel('Count')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:47.476445Z","iopub.execute_input":"2024-06-30T16:47:47.476853Z","iopub.status.idle":"2024-06-30T16:47:49.453763Z","shell.execute_reply.started":"2024-06-30T16:47:47.476816Z","shell.execute_reply":"2024-06-30T16:47:49.452665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.groupby(['source', 'age_approx']).size()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:49.454888Z","iopub.execute_input":"2024-06-30T16:47:49.455207Z","iopub.status.idle":"2024-06-30T16:47:49.513880Z","shell.execute_reply.started":"2024-06-30T16:47:49.455172Z","shell.execute_reply":"2024-06-30T16:47:49.512789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=dataset.dropna(subset=['age_approx']), x='age_approx', hue='source', multiple='stack', bins=10, kde=False)\n\nplt.title('Histogram of Target Values Grouped by Source')\nplt.xlabel('Target')\nplt.ylabel('Count')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:49.515086Z","iopub.execute_input":"2024-06-30T16:47:49.515402Z","iopub.status.idle":"2024-06-30T16:47:51.409369Z","shell.execute_reply.started":"2024-06-30T16:47:49.515374Z","shell.execute_reply":"2024-06-30T16:47:51.408218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset['sex'].fillna('unknown').hist();","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:51.410995Z","iopub.execute_input":"2024-06-30T16:47:51.411421Z","iopub.status.idle":"2024-06-30T16:47:51.832867Z","shell.execute_reply.started":"2024-06-30T16:47:51.411385Z","shell.execute_reply":"2024-06-30T16:47:51.831471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data=dataset.dropna(subset=['sex']), x='sex', hue='source', multiple='stack', kde=False)\n\nplt.title('Histogram of Target Values Grouped by Source')\nplt.xlabel('Target')\nplt.ylabel('Count')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-30T16:47:51.834245Z","iopub.execute_input":"2024-06-30T16:47:51.834563Z","iopub.status.idle":"2024-06-30T16:47:53.982267Z","shell.execute_reply.started":"2024-06-30T16:47:51.834535Z","shell.execute_reply":"2024-06-30T16:47:53.980490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Warning!\n**Don't forget about possible sample overlaps in datasets.** Checking and removing duplicates should be the next step.","metadata":{}}]}