{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Main Idea\n\nLets merge these datasets from [topic](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/154296#864656) by [@andrewmvd](https://www.kaggle.com/andrewmvd):\n\n---\n- [Melanoma Detection Dataset](https://www.kaggle.com/wanderdust/skin-lesion-analysis-toward-melanoma-detection)\n- [Skin Lesion Images for Melanoma Classification](https://www.kaggle.com/andrewmvd/isic-2019)\n- [Skin Cancer MNIST: HAM10000](https://www.kaggle.com/kmader/skin-cancer-mnist-ham10000)\n---\n\n- [SIIM-ISIC Melanoma Classification](https://www.kaggle.com/c/siim-isic-melanoma-classification/data)\n","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nimport cv2\nfrom skimage import io\nfrom tqdm import tqdm\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!mkdir '1024x1024-dataset-melanoma'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# isic 2019","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_gt = pd.read_csv('../input/isic-2019/ISIC_2019_Training_GroundTruth.csv')\nimage_id = df_gt.iloc[25]['image']\nimage = cv2.imread(f'../input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/{image_id}.jpg', cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## [@ipateam](https://www.kaggle.com/ipateam) Thanks a lot for [finding dublicated images](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/155859#878163)! ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_downsampled = df_gt[df_gt['image'].str.contains('downsampled')]\ndf_downsampled.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print('[ALL]:', df_gt.shape[0])\nprint('[∩ isic2020]:', len(set(df_train['image_name'].values).intersection(df_gt['image'].values)))\nprint('[downsampled isic2019 ∩ isic2020]:', len(set(df_train['image_name'].values).intersection([\n    image_id[:-12] for image_id in df_downsampled['image'].values\n])))\nprint('[downsampled isic2019 ∩ isic2019]:', len(set(df_gt['image'].values).intersection([\n    image_id[:-12] for image_id in df_downsampled['image'].values\n])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SLATMD [Almost completely repeated] [Removed]","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = glob('../input/skin-lesion-analysis-toward-melanoma-detection/skin-lesions/*/*/*.jpg')\nimage = cv2.imread(paths[777], cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"image_ids = [path.split('/')[-1][:-4] for path in paths]\nprint('[ALL]:', len(image_ids))\nprint('[∩ isic2020]:', len(set(image_ids).intersection(df_train['image_name'].values)))\nprint('[∩ isic2019]:', len(set(image_ids).intersection(df_gt['image'].values)))\nprint('[∩ isic2019 downsampled]:', len(set(image_ids).intersection([image_id[:-12] for image_id in df_gt[df_gt['image'].str.contains('downsampled')]['image'].values])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Skin Cancer MNIST: HAM10000 [Repeated]","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_meta = pd.read_csv('../input/skin-cancer-mnist-ham10000/HAM10000_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = df_meta.iloc[777]['image_id']\nimage = cv2.imread(f'../input/skin-cancer-mnist-ham10000/HAM10000_images_part_1/{image_id}.jpg', cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print('[ALL]:', df_meta.shape[0])\nprint('[∩ isic2020]:', len(set(df_meta['image_id'].values).intersection(df_train['image_name'].values)))\nprint('[∩ isic2019]:', len(set(df_meta['image_id'].values).intersection(df_gt['image'].values)))\nprint('[∩ slatmd]:', len(set(df_meta['image_id'].values).intersection(image_ids)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge datasets & metadata","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"NEED_IMAGE_SAVE = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = {\n    'patient_id' : [],\n    'image_id': [],\n    'target': [],\n    'source': [],\n    'sex': [],\n    'age_approx': [],\n    'anatom_site_general_challenge': [],\n}\n\n# isic2020\ndf_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv', index_col='image_name')\nfor image_id, row in tqdm(df_train.iterrows(), total=df_train.shape[0]):\n    if image_id in dataset['image_id']:\n        continue\n    dataset['patient_id'].append(row['patient_id'])\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(row['target'])\n    dataset['source'].append('ISIC20')\n    dataset['sex'].append(row['sex'])\n    dataset['age_approx'].append(row['age_approx'])\n    dataset['anatom_site_general_challenge'].append(row['anatom_site_general_challenge'])\n\n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/siim-isic-melanoma-classification/jpeg/train/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (1024, 1024), cv2.INTER_AREA)\n        cv2.imwrite(f'./1024x1024-dataset-melanoma/{image_id}.jpg', image)\n\n# isic2019\ndf_gt = pd.read_csv('../input/isic-2019/ISIC_2019_Training_GroundTruth.csv', index_col='image')\ndf_meta = pd.read_csv('../input/isic-2019/ISIC_2019_Training_Metadata.csv', index_col='image')\nfor image_id, row in tqdm(df_meta.iterrows(), total=df_meta.shape[0]):\n    if image_id in dataset['image_id']:\n        continue\n\n    dataset['patient_id'].append(row['lesion_id'])\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(int(df_gt.loc[image_id]['MEL']))\n    dataset['source'].append('ISIC19')\n    dataset['sex'].append(row['sex'])\n    dataset['age_approx'].append(row['age_approx'])\n    dataset['anatom_site_general_challenge'].append(\n        {'anterior torso': 'torso', 'posterior torso': 'torso'}.get(row['anatom_site_general'], row['anatom_site_general'])\n    )\n    \n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.resize(image, (1024, 1024), cv2.INTER_AREA)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        cv2.imwrite(f'./1024x1024-dataset-melanoma/{image_id}.jpg', image)\n\n    \ndataset = pd.DataFrame(dataset)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.to_csv('marking.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you all for reading my kernel!\n\n","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}