{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Main Idea\n\nLets merge these datasets from [topic](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/154296#864656) by [@andrewmvd](https://www.kaggle.com/andrewmvd):\n\n---\n- [Melanoma Detection Dataset](https://www.kaggle.com/wanderdust/skin-lesion-analysis-toward-melanoma-detection)\n- [Skin Lesion Images for Melanoma Classification](https://www.kaggle.com/andrewmvd/isic-2019)\n- [Skin Cancer MNIST: HAM10000](https://www.kaggle.com/kmader/skin-cancer-mnist-ham10000)\n---\n\n- [SIIM-ISIC Melanoma Classification](https://www.kaggle.com/c/siim-isic-melanoma-classification/data)\n","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nimport cv2\nfrom skimage import io\nfrom tqdm import tqdm\nimport seaborn as sns\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!mkdir '224x224-dataset-melanoma'\n!mkdir '224x224-test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# isic 2019","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_gt = pd.read_csv('../input/isic-2019/ISIC_2019_Training_GroundTruth.csv')\nimage_id = df_gt.iloc[25]['image']\nimage = cv2.imread(f'../input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/{image_id}.jpg', cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print('[ALL]:', df_gt.shape[0])\nprint('[∩ isic2020]:', len(set(df_train['image_name'].values).intersection(df_gt['image'].values)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SLATMD","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = glob('../input/skin-lesion-analysis-toward-melanoma-detection/skin-lesions/*/*/*.jpg')\nprint(len(paths))\nimage = cv2.imread(paths[2], cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"image_ids = [path.split('/')[-1][:-4] for path in paths]\nprint('[ALL]:', len(image_ids))\nprint('[∩ isic2020]:', len(set(image_ids).intersection(df_train['image_name'].values)))\nprint('[∩ isic2019]:', len(set(image_ids).intersection(df_gt['image'].values)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Skin Cancer MNIST: HAM10000 [Repeated]","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_meta = pd.read_csv('../input/skin-cancer-mnist-ham10000/HAM10000_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = df_meta.iloc[777]['image_id']\nimage = cv2.imread(f'../input/skin-cancer-mnist-ham10000/HAM10000_images_part_1/{image_id}.jpg', cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print('[ALL]:', df_meta.shape[0])\nprint('[∩ isic2020]:', len(set(df_meta['image_id'].values).intersection(df_train['image_name'].values)))\nprint('[∩ isic2019]:', len(set(df_meta['image_id'].values).intersection(df_gt['image'].values)))\nprint('[∩ slatmd]:', len(set(df_meta['image_id'].values).intersection(image_ids)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge datasets & metadata","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"NEED_IMAGE_SAVE = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists('./224x224-dataset-melanoma/melanoma'):\n    os.makedirs('./224x224-dataset-melanoma/melanoma')\n    \nif not os.path.exists('./224x224-dataset-melanoma/other'):\n    os.makedirs('./224x224-dataset-melanoma/other')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = {\n    'patient_id' : [],\n    'image_id': [],\n    'target': [],\n    'source': [],\n    'sex': [],\n    'age_approx': [],\n    'anatom_site_general_challenge': [],\n}\n\n\n# isic2020\ndf_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv', index_col='image_name')\nfor image_id, row in tqdm(df_train.iterrows(), total=df_train.shape[0]):\n    if image_id in dataset['image_id']:\n        continue\n    dataset['patient_id'].append(row['patient_id'])\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(row['target'])\n    dataset['source'].append('ISIC20')\n    dataset['sex'].append(row['sex'])\n    dataset['age_approx'].append(row['age_approx'])\n    dataset['anatom_site_general_challenge'].append(row['anatom_site_general_challenge'])\n\n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/siim-isic-melanoma-classification/jpeg/train/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (224,224), cv2.INTER_AREA)\n        dest = f'./224x224-dataset-melanoma/other/{image_id}.jpg'\n        if row['target']==1:\n            dest = f'./224x224-dataset-melanoma/melanoma/{image_id}.jpg'\n        cv2.imwrite(dest, image)\n\n# isic2019\ndf_gt = pd.read_csv('../input/isic-2019/ISIC_2019_Training_GroundTruth.csv', index_col='image')\ndf_meta = pd.read_csv('../input/isic-2019/ISIC_2019_Training_Metadata.csv', index_col='image')\nfor image_id, row in tqdm(df_meta.iterrows(), total=df_meta.shape[0]):\n    if image_id in dataset['image_id']:\n        continue\n    dataset['patient_id'].append(row['lesion_id'])\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(int(df_gt.loc[image_id]['MEL']))\n    dataset['source'].append('ISIC19')\n    dataset['sex'].append(row['sex'])\n    dataset['age_approx'].append(row['age_approx'])\n    dataset['anatom_site_general_challenge'].append(\n        {'anterior torso': 'torso', 'posterior torso': 'torso'}.get(row['anatom_site_general'], row['anatom_site_general'])\n    )\n    \n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.resize(image, (224,224), cv2.INTER_AREA)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        dest = f'./224x224-dataset-melanoma/other/{image_id}.jpg'\n        if int(df_gt.loc[image_id]['MEL'])==1:\n            dest = f'./224x224-dataset-melanoma/melanoma/{image_id}.jpg'\n        cv2.imwrite(dest, image)\n\n\n# skin-lesion-analysis-toward-melanoma-detection\npaths = glob('../input/skin-lesion-analysis-toward-melanoma-detection/skin-lesions/*/*/*.jpg')\nfor path in tqdm(paths, total=len(paths)):\n    diagnosis, image_id = path.split('/')[-2:]\n    image_id = image_id[:-4]\n    \n    if image_id in dataset['image_id']:\n        continue\n    \n    target = int(diagnosis == 'melanoma')\n    dataset['patient_id'].append(np.nan)\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(target)\n    dataset['source'].append('SLATMD')\n    dataset['sex'].append(np.nan)\n    dataset['age_approx'].append(np.nan)\n    dataset['anatom_site_general_challenge'].append(np.nan)\n    \n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(path, cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (224,224), cv2.INTER_AREA)\n        dest = f'./224x224-dataset-melanoma/other/{image_id}.jpg'\n        if target==1:\n            dest = f'./224x224-dataset-melanoma/melanoma/{image_id}.jpg'\n        cv2.imwrite(dest, image)\n    \ndataset = pd.DataFrame(dataset)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.to_csv('merged_data.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stratify GroupKFold Splitting\n\nhttps://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import numpy as np\nimport random\nimport pandas as pd\nfrom collections import Counter, defaultdict\n\ndef stratified_group_k_fold(X, y, groups, k, seed=None):\n    \"\"\" https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation \"\"\"\n    labels_num = np.max(y) + 1\n    y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))\n    y_distr = Counter()\n    for label, g in zip(y, groups):\n        y_counts_per_group[g][label] += 1\n        y_distr[label] += 1\n\n    y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))\n    groups_per_fold = defaultdict(set)\n\n    def eval_y_counts_per_fold(y_counts, fold):\n        y_counts_per_fold[fold] += y_counts\n        std_per_label = []\n        for label in range(labels_num):\n            label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])\n            std_per_label.append(label_std)\n        y_counts_per_fold[fold] -= y_counts\n        return np.mean(std_per_label)\n    \n    groups_and_y_counts = list(y_counts_per_group.items())\n    random.Random(seed).shuffle(groups_and_y_counts)\n\n    for g, y_counts in tqdm(sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])), total=len(groups_and_y_counts)):\n        best_fold = None\n        min_eval = None\n        for i in range(k):\n            fold_eval = eval_y_counts_per_fold(y_counts, i)\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = i\n        y_counts_per_fold[best_fold] += y_counts\n        groups_per_fold[best_fold].add(g)\n\n    all_groups = set(groups)\n    for i in range(k):\n        train_groups = all_groups - groups_per_fold[i]\n        test_groups = groups_per_fold[i]\n\n        train_indices = [i for i, g in enumerate(groups) if g in train_groups]\n        test_indices = [i for i, g in enumerate(groups) if g in test_groups]\n\n        yield train_indices, test_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_folds = pd.read_csv(f'merged_data.csv')\ndf_folds['patient_id'] = df_folds['patient_id'].fillna(df_folds['image_id'])\ndf_folds['sex'] = df_folds['sex'].fillna('unknown')\ndf_folds['anatom_site_general_challenge'] = df_folds['anatom_site_general_challenge'].fillna('unknown')\ndf_folds['age_approx'] = df_folds['age_approx'].fillna(round(df_folds['age_approx'].mean()))\ndf_folds = df_folds.set_index('image_id')\n\ndef get_stratify_group(row):\n    stratify_group = row['sex']\n    stratify_group += f'_{row[\"anatom_site_general_challenge\"]}'\n    stratify_group += f'_{row[\"source\"]}'\n    stratify_group += f'_{row[\"target\"]}'\n    return stratify_group\n\ndf_folds['stratify_group'] = df_folds.apply(get_stratify_group, axis=1)\ndf_folds['stratify_group'] = df_folds['stratify_group'].astype('category').cat.codes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_folds.loc[:, 'fold'] = 0\n\nskf = stratified_group_k_fold(X=df_folds.index, y=df_folds['stratify_group'], groups=df_folds['patient_id'], k=5, seed=42)\n\nfor fold_number, (train_index, val_index) in enumerate(skf):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(df_folds[df_folds['fold'] == 0]['patient_id'].values).intersection(df_folds[df_folds['fold'] == 1]['patient_id'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds[df_folds['fold'] == 0]['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds[df_folds['fold'] == 1]['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds.to_csv('folds.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# test isic2020\ndf_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv', index_col='image_name')\n\nfor image_id, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):   \n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/siim-isic-melanoma-classification/jpeg/test/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (224,224), cv2.INTER_AREA)\n        cv2.imwrite(f'kaggle/working/224x224-test/{image_id}.jpg', image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def zipdir(path, ziph):\n    # ziph is zipfile handle\n    for root, dirs, files in os.walk(path):\n        for file in files:\n            if not file == 'all_data.zip':\n                ziph.write(os.path.join(root, file))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"zipf = zipfile.ZipFile('all_data.zip', 'w', zipfile.ZIP_DEFLATED)\nzipdir('/kaggle/working', zipf)\nzipf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}