{"cells":[{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nimport cv2\nfrom skimage import io\nfrom tqdm import tqdm\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!mkdir '384x288-dataset-melanoma'\n!mkdir '384x288-test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# isic 2019","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_gt = pd.read_csv('../input/isic-2019/ISIC_2019_Training_GroundTruth.csv')\nimage_id = df_gt.iloc[25]['image']\nimage = cv2.imread(f'../input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/{image_id}.jpg', cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## [@ipateam](https://www.kaggle.com/ipateam) Thanks a lot for [finding dublicated images](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/155859#878163)! ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_downsampled = df_gt[df_gt['image'].str.contains('downsampled')]\ndf_downsampled.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print('[ALL]:', df_gt.shape[0])\nprint('[∩ isic2020]:', len(set(df_train['image_name'].values).intersection(df_gt['image'].values)))\nprint('[downsampled isic2019 ∩ isic2020]:', len(set(df_train['image_name'].values).intersection([\n    image_id[:-12] for image_id in df_downsampled['image'].values\n])))\nprint('[downsampled isic2019 ∩ isic2019]:', len(set(df_gt['image'].values).intersection([\n    image_id[:-12] for image_id in df_downsampled['image'].values\n])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SLATMD [Almost completely repeated] [Removed]","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"paths = glob('../input/skin-lesion-analysis-toward-melanoma-detection/skin-lesions/*/*/*.jpg')\nimage = cv2.imread(paths[777], cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"image_ids = [path.split('/')[-1][:-4] for path in paths]\nprint('[ALL]:', len(image_ids))\nprint('[∩ isic2020]:', len(set(image_ids).intersection(df_train['image_name'].values)))\nprint('[∩ isic2019]:', len(set(image_ids).intersection(df_gt['image'].values)))\nprint('[∩ isic2019 downsampled]:', len(set(image_ids).intersection([image_id[:-12] for image_id in df_gt[df_gt['image'].str.contains('downsampled')]['image'].values])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Skin Cancer MNIST: HAM10000 [Repeated]","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_meta = pd.read_csv('../input/skin-cancer-mnist-ham10000/HAM10000_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = df_meta.iloc[777]['image_id']\nimage = cv2.imread(f'../input/skin-cancer-mnist-ham10000/HAM10000_images_part_1/{image_id}.jpg', cv2.IMREAD_COLOR)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image.shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print('[ALL]:', df_meta.shape[0])\nprint('[∩ isic2020]:', len(set(df_meta['image_id'].values).intersection(df_train['image_name'].values)))\nprint('[∩ isic2019]:', len(set(df_meta['image_id'].values).intersection(df_gt['image'].values)))\nprint('[∩ slatmd]:', len(set(df_meta['image_id'].values).intersection(image_ids)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge datasets & metadata","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"NEED_IMAGE_SAVE = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = {\n    'patient_id' : [],\n    'image_id': [],\n    'target': [],\n    'source': [],\n    'sex': [],\n    'age_approx': [],\n    'anatom_site_general_challenge': [],\n}\n\n# isic2020\ndf_train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv', index_col='image_name')\nfor image_id, row in tqdm(df_train.iterrows(), total=df_train.shape[0]):\n    if image_id in dataset['image_id']:\n        continue\n    dataset['patient_id'].append(row['patient_id'])\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(row['target'])\n    dataset['source'].append('ISIC20')\n    dataset['sex'].append(row['sex'])\n    dataset['age_approx'].append(row['age_approx'])\n    dataset['anatom_site_general_challenge'].append(row['anatom_site_general_challenge'])\n\n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/siim-isic-melanoma-classification/jpeg/train/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (384, 288), cv2.INTER_AREA)\n        cv2.imwrite(f'./384x288-dataset-melanoma/{image_id}.jpg', image)\n\n# isic2019\ndf_gt = pd.read_csv('../input/isic-2019/ISIC_2019_Training_GroundTruth.csv', index_col='image')\ndf_meta = pd.read_csv('../input/isic-2019/ISIC_2019_Training_Metadata.csv', index_col='image')\nfor image_id, row in tqdm(df_meta.iterrows(), total=df_meta.shape[0]):\n    if image_id in dataset['image_id']:\n        continue\n\n    dataset['patient_id'].append(row['lesion_id'])\n    dataset['image_id'].append(image_id)\n    dataset['target'].append(int(df_gt.loc[image_id]['MEL']))\n    dataset['source'].append('ISIC19')\n    dataset['sex'].append(row['sex'])\n    dataset['age_approx'].append(row['age_approx'])\n    dataset['anatom_site_general_challenge'].append(\n        {'anterior torso': 'torso', 'posterior torso': 'torso'}.get(row['anatom_site_general'], row['anatom_site_general'])\n    )\n    \n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/isic-2019/ISIC_2019_Training_Input/ISIC_2019_Training_Input/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.resize(image, (384, 288), cv2.INTER_AREA)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        cv2.imwrite(f'./384x288-dataset-melanoma/{image_id}.jpg', image)\n\n    \ndataset = pd.DataFrame(dataset).set_index('image_id')    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Excluding duplicates (with [clustering approach](https://www.kaggle.com/shonenkov/dbscan-clustering-check-marking))","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_duplicates = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/duplicates_13062020.csv', index_col='image_ids')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_value(duplicate_data, row, field):\n    if row[field] == 1 or duplicate_data.shape[0] <= 2:\n        return duplicate_data.iloc[0][field]\n    if 'ISIC20' in duplicate_data.source.values:\n        duplicate_data = duplicate_data[duplicate_data.source == 'ISIC20']\n    return sorted(duplicate_data[field].value_counts().items(), key=lambda x: -x[1])[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cleaned_duplicates = {\n    'image_id': [],\n    'patient_id': [],\n    'target': [],\n    'source': [],\n    'sex': [],\n    'age_approx': [],\n    'anatom_site_general_challenge': [],\n}\ndrop_image_ids = []\nfor image_ids, row in df_duplicates.iterrows():\n    image_ids = image_ids.split('.')\n    drop_image_ids.extend(image_ids)\n    duplicate_data = dataset.loc[image_ids].sort_values('source', ascending=False)\n    for field in [    \n        'patient_id',\n        'target',\n        'source',\n        'sex',\n        'age_approx',\n        'anatom_site_general_challenge',\n    ]:\n        cleaned_duplicates[field].append(get_value(duplicate_data, row, field))\n    cleaned_duplicates['image_id'].append(duplicate_data.index.values[0])\n\ncleaned_duplicates = pd.DataFrame(cleaned_duplicates).set_index('image_id')\ncleaned_duplicates.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = dataset.drop(drop_image_ids)\ndataset = dataset.append(cleaned_duplicates)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset.to_csv('marking.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Simple EDA:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset['source'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dataset['target'].value_counts())\ndataset['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset['sex'].fillna('unknown').hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset['age_approx'].hist(bins=50);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset['anatom_site_general_challenge'].fillna('unknown').value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stratify GroupKFold Splitting\n\nhttps://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import numpy as np\nimport random\nimport pandas as pd\nfrom collections import Counter, defaultdict\n\ndef stratified_group_k_fold(X, y, groups, k, seed=None):\n    \"\"\" https://www.kaggle.com/jakubwasikowski/stratified-group-k-fold-cross-validation \"\"\"\n    labels_num = np.max(y) + 1\n    y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))\n    y_distr = Counter()\n    for label, g in zip(y, groups):\n        y_counts_per_group[g][label] += 1\n        y_distr[label] += 1\n\n    y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))\n    groups_per_fold = defaultdict(set)\n\n    def eval_y_counts_per_fold(y_counts, fold):\n        y_counts_per_fold[fold] += y_counts\n        std_per_label = []\n        for label in range(labels_num):\n            label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])\n            std_per_label.append(label_std)\n        y_counts_per_fold[fold] -= y_counts\n        return np.mean(std_per_label)\n    \n    groups_and_y_counts = list(y_counts_per_group.items())\n    random.Random(seed).shuffle(groups_and_y_counts)\n\n    for g, y_counts in tqdm(sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])), total=len(groups_and_y_counts)):\n        best_fold = None\n        min_eval = None\n        for i in range(k):\n            fold_eval = eval_y_counts_per_fold(y_counts, i)\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = i\n        y_counts_per_fold[best_fold] += y_counts\n        groups_per_fold[best_fold].add(g)\n\n    all_groups = set(groups)\n    for i in range(k):\n        train_groups = all_groups - groups_per_fold[i]\n        test_groups = groups_per_fold[i]\n\n        train_indices = [i for i, g in enumerate(groups) if g in train_groups]\n        test_indices = [i for i, g in enumerate(groups) if g in test_groups]\n\n        yield train_indices, test_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_folds = pd.read_csv('marking.csv')\ndf_folds['patient_id'] = df_folds['patient_id'].fillna(df_folds['image_id'])\ndf_folds['sex'] = df_folds['sex'].fillna('unknown')\ndf_folds['anatom_site_general_challenge'] = df_folds['anatom_site_general_challenge'].fillna('unknown')\ndf_folds['age_approx'] = df_folds['age_approx'].fillna(round(df_folds['age_approx'].mean()))\n\npatient_id_2_count = df_folds[['patient_id', 'image_id']].groupby('patient_id').count()['image_id'].to_dict()\n\ndf_folds = df_folds.set_index('image_id')\n\ndef get_stratify_group(row):\n    stratify_group = row['sex']\n#     stratify_group += f'_{row[\"anatom_site_general_challenge\"]}'\n    stratify_group += f'_{row[\"source\"]}'\n    stratify_group += f'_{row[\"target\"]}'\n    patient_id_count = patient_id_2_count[row[\"patient_id\"]]\n    if patient_id_count > 80:\n        stratify_group += f'_80'\n    elif patient_id_count > 60:\n        stratify_group += f'_60'\n    elif patient_id_count > 50:\n        stratify_group += f'_50'\n    elif patient_id_count > 30:\n        stratify_group += f'_30'\n    elif patient_id_count > 20:\n        stratify_group += f'_20'\n    elif patient_id_count > 10:\n        stratify_group += f'_10'\n    else:\n        stratify_group += f'_0'\n    return stratify_group\n\ndf_folds['stratify_group'] = df_folds.apply(get_stratify_group, axis=1)\ndf_folds['stratify_group'] = df_folds['stratify_group'].astype('category').cat.codes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_folds.loc[:, 'fold'] = 0\n\nskf = stratified_group_k_fold(X=df_folds.index, y=df_folds['stratify_group'], groups=df_folds['patient_id'], k=5, seed=42)\n\nfor fold_number, (train_index, val_index) in enumerate(skf):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(df_folds[df_folds['fold'] == 0]['patient_id'].values).intersection(df_folds[df_folds['fold'] == 1]['patient_id'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds[df_folds['fold'] == 0]['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds[df_folds['fold'] == 1]['target'].hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds.to_csv('folds_13062020.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# test isic2020\ndf_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv', index_col='image_name')\nfor image_id, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):   \n    if NEED_IMAGE_SAVE:\n        image = cv2.imread(f'../input/siim-isic-melanoma-classification/jpeg/test/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = cv2.resize(image, (384, 288), cv2.INTER_AREA)\n        cv2.imwrite(f'./384x288-test/{image_id}.jpg', image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final Dataset\n\nFinal dataset jpeg 512x512 you can find [here](https://www.kaggle.com/shonenkov/melanoma-merged-external-data-512x512-jpeg)\n\nexamples for usage:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_folds = pd.read_csv('../input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\ndf_folds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = cv2.imread(f'../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/ISIC_0074268.jpg', cv2.IMREAD_COLOR)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = cv2.imread(f'../input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/ISIC_0089356.jpg', cv2.IMREAD_COLOR)\nio.imshow(image);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you all for reading my kernel!\n\n","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}