{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom os import listdir \nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\nimport pylab \nimport math\nimport re\nimport random\nfrom tqdm import tqdm\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.pipeline import Pipeline\n\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.multiclass import type_of_target\nfrom sklearn.metrics import accuracy_score\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Model,Sequential\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Conv2D, MaxPooling2D, AveragePooling2D\nfrom keras.layers.core import Dense, Flatten, Dropout, Lambda\n\nfrom kaggle_datasets import KaggleDatasets\n\nimport warnings\nwarnings.filterwarnings('ignore') \n\nsns.set(rc={'figure.figsize': (20, 5)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Tensorflow version \" + tf.__version__)\n\nAUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tpu\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu) \n    tf.tpu.experimental.initialize_tpu_system(tpu) \n    strategy = tf.distribute.experimental.TPUStrategy(tpu) \nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n\nDATASET = '512x512-melanoma-tfrecords-70k-images'\nGCS_PATH = KaggleDatasets().get_gcs_path(DATASET) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nSIZE = [512,512]\nEPOCHS = 3\nLABEL_SMOOTHING = 0.05\nN_SPLITS = 5\nI = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"basepath = \"../input/siim-isic-melanoma-classification/\"\nimagesRGBpath = \"../input/melanoma-classification-rgb-image/\" ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(listdir(basepath))\nprint(listdir(imagesRGBpath))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_info = pd.read_csv(basepath + \"train.csv\")\ntest_info = pd.read_csv(basepath + \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_info.isnull().sum())\nprint(test_info.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## patient_id","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_id_count_train = train_info.groupby('patient_id').aggregate({'patient_id': 'count'})\npatient_id_count_test = test_info.groupby('patient_id').aggregate({'patient_id': 'count'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(221)\nsns.distplot(patient_id_count_train, color=\"green\", kde_kws = {'color': 'gray', 'lw':1, 'label': 'patient id count' })\nplt.subplot(222)\nsns.distplot(patient_id_count_test, color=\"green\", kde_kws = {'color': 'gray', 'lw':1, 'label': 'patient id count' })\nplt.subplot(223)\nsns.boxplot(patient_id_count_train)\nplt.subplot(224)\nsns.boxplot(patient_id_count_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## sex","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sex_count_train = train_info.groupby('sex').aggregate({'sex': 'count'}).rename(columns={'sex': 'sex_count'}).reset_index()\nsex_count_test = test_info.groupby('sex').aggregate({'sex': 'count'}).rename(columns={'sex': 'sex_count'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(121)\nsns.barplot('sex', 'sex_count', data = sex_count_train)\nplt.subplot(122)\nsns.barplot('sex', 'sex_count', data = sex_count_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Age","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"age_count_train = train_info.groupby('age_approx').aggregate({'age_approx': 'count'})\nage_count_test = test_info.groupby('age_approx').aggregate({'age_approx': 'count'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize': (20, 12)})\nplt.subplot(221)\nsns.distplot(train_info.age_approx, color=\"green\", kde_kws = {'color': 'gray', 'lw':1, 'label': 'patient age count' })\nplt.subplot(222)\nsns.distplot(test_info.age_approx, color=\"green\", kde_kws = {'color': 'gray', 'lw':1, 'label': 'patient age count' })\nplt.subplot(223)\nstats.probplot(age_count_train.age_approx, dist=\"norm\", plot=pylab)\nplt.subplot (224)\nstats.probplot(age_count_test.age_approx, dist=\"norm\", plot=pylab)\npylab.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## anatom site","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"anatom_count_train = train_info.groupby('anatom_site_general_challenge').aggregate(\n    {'anatom_site_general_challenge': 'count'}).rename(columns={'anatom_site_general_challenge': 'anatom_site_count'}).reset_index()\nanatom_count_test = test_info.groupby('anatom_site_general_challenge').aggregate(\n    {'anatom_site_general_challenge': 'count'}).rename(columns={'anatom_site_general_challenge': 'anatom_site_count'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize': (20, 5)})\n\nplt.subplot(121)\nanatom_count_train = sns.barplot('anatom_site_general_challenge', 'anatom_site_count', data = anatom_count_train)\nfor item in anatom_count_train.get_xticklabels():\n    item.set_rotation(45)  \n    \nplt.subplot(122)\nanatom_count_test = sns.barplot('anatom_site_general_challenge', 'anatom_site_count', data = anatom_count_test)\nfor item in anatom_count_test.get_xticklabels():\n    item.set_rotation(45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most of all the place of occurrence of the formation on the torso","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## diagnosis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"diagnosis_count_train = train_info.groupby('diagnosis').aggregate({'diagnosis': 'count'}).rename(\n    columns={'diagnosis': 'count_diagnosis'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"diagnosis_count = sns.barplot('diagnosis', 'count_diagnosis', data = diagnosis_count_train)\nfor item in diagnosis_count.get_xticklabels():\n    item.set_rotation(45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## benign_malignant","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"benign_malignant_count_train = train_info.groupby('benign_malignant').aggregate({'benign_malignant': 'count'}).rename(\n    columns={'benign_malignant': 'count_benign_malignant'}).reset_index()\n\ntarget_count_train = train_info.groupby('target').aggregate({'target': 'count'}).rename(\n    columns={'target': 'count_target'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(121)\nsns.barplot('benign_malignant', 'count_benign_malignant', data = benign_malignant_count_train)\nplt.subplot(122)\nsns.barplot('target', 'count_target', data = target_count_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"These two indicators duplicate each other","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## missing values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Confidence interval calculation function: \ndef derf(sample, mean, std):\n    age_shape = sample['age_approx'].shape[0] \n    standard_error_ofthe_mean = std / math.sqrt(age_shape)\n    random_mean = random.uniform(mean-(1.96*standard_error_ofthe_mean), mean+(1.96*standard_error_ofthe_mean))\n    return round(random_mean, 2) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Select only those values that have gaps","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"T_index = []\nfor i, t in enumerate(train_info['sex'].isnull()):\n    if t == True:\n        T_index.append(i)\n        \n# select only those values in which there are gaps\ntrain_info_NanSEX = train_info.loc[T_index] \ntrain_info_NanSEX.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Missing gender and age values are observed only for one group, some of the patients have been observed for several years, we select the values without gaps from train_info (patient index indicators, gender and age) and see if patient indices are observed in train_info_NanSEX, if yes, we can restore the gender value and age","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_info_SeAgId = train_info[['patient_id', 'sex', 'age_approx']].dropna() \n\nCount_env = 0\nfor u in train_info_NanSEX['patient_id']:\n    if u in list(train_info_SeAgId['patient_id']):\n        Count_env+=1\n        \nprint(Count_env)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No, there are no intersections","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"fill in missing gender values with median value","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_info[['sex']] = train_info['sex'].fillna('male')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's divide the data into 4 groups for age recovery.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# target 0 girls:\nSex_female_target0 = train_info.loc[(train_info.sex == 'female') & (train_info.target == 0)]\n# target 1 girls:\nSex_female_target1 = train_info.loc[(train_info.sex == 'female') & (train_info.target == 1)]\n#  target 0 guys:\nSex_male_target0 = train_info.loc[(train_info.sex == 'male') & (train_info.target == 0)] \n#  target 1 guys:\nSex_male_target1 = train_info.loc[(train_info.sex == 'male') & (train_info.target == 1)] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(Sex_female_target0.isnull().sum())\nprint(Sex_male_target0.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The age will be restored for the groups Sex_female_target0 and Sex_male_target0","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize': (20, 10)})\n\nplt.subplot (221)\nsns.distplot(Sex_female_target0['age_approx'], color=\"green\", kde_kws = {'color': 'g', 'lw':1, 'label': 'distribution Age female_target 0' })\nplt.subplot (222)\nsns.distplot(Sex_female_target1['age_approx'], color=\"r\", kde_kws = {'color': 'r', 'lw':1, 'label': 'distribution Age female_target 1' })\nplt.subplot (223)\nsns.distplot(Sex_male_target0['age_approx'], color=\"blue\", kde_kws = {'color': 'blue', 'lw':1, 'label': 'distribution Age male_target 0' })\nplt.subplot (224)\nsns.distplot(Sex_male_target1['age_approx'], color=\"gray\", kde_kws = {'color': 'gray', 'lw':1, 'label': 'distribution Age male_target 1' })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"female_target0_mean, female_target1_mean = Sex_female_target0['age_approx'].mean(), Sex_female_target1['age_approx'].mean()\nmale_target0_mean, male_target1_mean = Sex_male_target0['age_approx'].mean(), Sex_male_target1['age_approx'].mean()\n\nfemale_target0_std, female_target1_std = Sex_female_target0['age_approx'].std(), Sex_female_target1['age_approx'].std()\nmale_target0_std, male_target1_std = Sex_male_target0['age_approx'].std(), Sex_male_target1['age_approx'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in train_info.loc[(train_info['sex']=='female') & (train_info['target']==0) & \n                        (train_info['age_approx'].isnull())].index:\n    train_info.at[i, 'age_approx'] = derf(Sex_female_target0, female_target0_mean, female_target0_std)\n    \nfor i in train_info.loc[(train_info['sex']=='male') & (train_info['target']==0) & \n                        (train_info['age_approx'].isnull())].index:\n    train_info.at[i, 'age_approx'] = derf(Sex_male_target0, male_target0_mean, male_target0_std)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anatom_Sex_female_target0 = Sex_female_target0.groupby('anatom_site_general_challenge').aggregate({\n    'anatom_site_general_challenge': 'count'}).rename(columns={\n    'anatom_site_general_challenge': 'count_anatom'}).reset_index()\n\nanatom_Sex_female_target1 = Sex_female_target1.groupby('anatom_site_general_challenge').aggregate({\n    'anatom_site_general_challenge': 'count'}).rename(columns={\n    'anatom_site_general_challenge': 'count_anatom'}).reset_index()\n\n\nanatom_Sex_male_target0 = Sex_male_target0.groupby('anatom_site_general_challenge').aggregate({\n    'anatom_site_general_challenge': 'count'}).rename(columns={\n    'anatom_site_general_challenge': 'count_anatom'}).reset_index()\n\nanatom_Sex_male_target1 = Sex_male_target1.groupby('anatom_site_general_challenge').aggregate({\n    'anatom_site_general_challenge': 'count'}).rename(columns={\n    'anatom_site_general_challenge': 'count_anatom'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize': (22, 7)})\n\nplt.subplot(221)\nanatom_count_Sex_female_target0 = sns.barplot('anatom_site_general_challenge', 'count_anatom', data = anatom_Sex_female_target0)\nfor item in anatom_count_Sex_female_target0.get_xticklabels():\n    item.set_rotation(45)\n    \nplt.subplot(222)       \nanatom_count_Sex_female_target1 = sns.barplot('anatom_site_general_challenge', 'count_anatom', data = anatom_Sex_female_target1)\nfor item in anatom_count_Sex_female_target1.get_xticklabels():\n    item.set_rotation(45)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(221)\nanatom_count_Sex_male_target0 = sns.barplot('anatom_site_general_challenge', 'count_anatom', data = anatom_Sex_male_target0)\nfor item in anatom_count_Sex_male_target0.get_xticklabels():\n    item.set_rotation(45)\n    \nplt.subplot(222)       \nanatom_count_Sex_male_target1 = sns.barplot('anatom_site_general_challenge', 'count_anatom', data = anatom_Sex_male_target1)\nfor item in anatom_count_Sex_male_target1.get_xticklabels():\n    item.set_rotation(45)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Even at the level of groups of men with benign formations and malignant formations and the same groups, women most often have a form on the torso, that is, benign formations (on the left of both histograms) are observed on the torso with about the same frequency as malignant ones.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_info[['anatom_site_general_challenge']] = train_info['anatom_site_general_challenge'].fillna('torso')\ntrain_info.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_info[['anatom_site_general_challenge']] = test_info['anatom_site_general_challenge'].fillna('torso')\ntest_info.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Individual patient information**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_gender_train = train_info.groupby(\"patient_id\").sex.unique().apply(lambda l: l[0])\npatient_gender_test = test_info.groupby(\"patient_id\").sex.unique().apply(lambda l: l[0])\n\ntrain_patients = pd.DataFrame(index=patient_gender_train.index.values, data=patient_gender_train.values, columns=[\"sex\"])\ntest_patients = pd.DataFrame(index=patient_gender_test.index.values, data=patient_gender_test.values, columns=[\"sex\"])\n\ntrain_patients.loc[:, \"num_images\"] = train_info.groupby(\"patient_id\").size() \ntest_patients.loc[:, \"num_images\"] = test_info.groupby(\"patient_id\").size() \n\ntrain_patients.loc[:, \"min_age\"] = train_info.groupby(\"patient_id\").age_approx.min()\ntrain_patients.loc[:, \"max_age\"] = train_info.groupby(\"patient_id\").age_approx.max()\ntest_patients.loc[:, \"min_age\"] = test_info.groupby(\"patient_id\").age_approx.min()\ntest_patients.loc[:, \"max_age\"] = test_info.groupby(\"patient_id\").age_approx.max()\n\ntrain_patients.loc[:, \"age_span\"] = train_patients[\"max_age\"] - train_patients[\"min_age\"] \ntest_patients.loc[:, \"age_span\"] = test_patients[\"max_age\"] - test_patients[\"min_age\"]\n\ntrain_patients.loc[:, \"benign_cases\"] = train_info.groupby([\"patient_id\", \"benign_malignant\"]).size().loc[:, \"benign\"]\ntrain_patients.loc[:, \"malignant_cases\"] = train_info.groupby([\"patient_id\", \"benign_malignant\"]).size().loc[:, \"malignant\"]\n\ntrain_patients[\"min_age_malignant\"] = train_info.groupby([\"patient_id\", \"benign_malignant\"]).age_approx.min().loc[:, \"malignant\"]\ntrain_patients[\"max_age_malignant\"] = train_info.groupby([\"patient_id\", \"benign_malignant\"]).age_approx.max().loc[:, \"malignant\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_patients.sort_values(by=\"malignant_cases\", ascending=False).head()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_patients_age_span = train_patients.groupby('age_span').aggregate({'age_span': 'count'}).rename(columns={\n    'age_span': 'count_age_span'}).reset_index()\n\ntest_patients_age_span = test_patients.groupby('age_span').aggregate({'age_span': 'count'}).rename(columns={\n    'age_span': 'count_age_span'}).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,2,figsize=(20,12))\nsns.countplot(train_patients.sex, ax=ax[0,0], palette=\"Reds\")\nax[0,0].set_title(\"Gender counts with unique patient ids in train\")\nsns.countplot(test_patients.sex, ax=ax[0,1], palette=\"Blues\");\nax[0,1].set_title(\"Gender counts with unique patient ids in test\");\n\ntrain_patients_age_span_ax = sns.barplot('age_span', 'count_age_span', data = train_patients_age_span, ax=ax[1,0]);\nfor item in train_patients_age_span_ax.get_xticklabels():\n    item.set_rotation(45)\n    \ntrain_patients_age_span_ay = sns.barplot('age_span', 'count_age_span', data = test_patients_age_span, ax=ax[1,1]);\nfor item in train_patients_age_span_ay.get_xticklabels():\n    item.set_rotation(45)\n\nax[1,0].set_title(\"Patients age span in train\")\nax[1,1].set_title(\"Patients age span in test\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RGB Table","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def creation_ofstatistical_tables(tables, statistical_tables):\n    for i, rgb_id in tqdm(enumerate(tables.index)):\n        x = tables.loc['{}'.format(rgb_id)].values\n    \n        statistical_tables.loc[i, 'mean'] = x.mean()\n        statistical_tables.loc[i, 'des'] = np.var(x)\n        statistical_tables.loc[i, 'std'] = x.std()\n        statistical_tables.loc[i, 'max'] = x.max()\n        statistical_tables.loc[i, 'min'] = x.min()\n\n        statistical_tables.loc[i, 'quan0.25'] = np.quantile(x, 0.25)\n        statistical_tables.loc[i, 'quan0.5'] = np.quantile(x, 0.5)\n        statistical_tables.loc[i, 'quan0.75'] = np.quantile(x, 0.75)\n        \n    return statistical_tables","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DFrame_ISIC_in_RGB1 = pd.read_csv(imagesRGBpath + 'DFrame_ISIC_in_RGB1.csv')\nDFrame_ISIC_in_RGB2 = pd.read_csv(imagesRGBpath + 'DFrame_ISIC_in_RGB2.csv')\nDFrame_ISIC_in_RGB3 = pd.read_csv(imagesRGBpath + 'DFrame_ISIC_in_RGB3.csv') \nDFrame_ISIC_in_TEST = pd.read_csv(imagesRGBpath + 'DFrame_ISIC_in_TEST_RGB.csv') \n\nDTrain_patient_Static = pd.read_csv(imagesRGBpath + 'Train_RGB_Static.csv') \nDTest_patient_Static = pd.read_csv(imagesRGBpath + 'Test_RGB_Static.csv') \n\nDTrain_patient_Static = DTrain_patient_Static.rename(columns={'image_name': 'ST_image_name'})\nDTest_patient_Static = DTest_patient_Static.rename(columns={'image_name': 'ST_image_name'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RGB_Table = np.concatenate((DFrame_ISIC_in_RGB1, DFrame_ISIC_in_RGB2, DFrame_ISIC_in_RGB3), axis=1) \ncolumns_RGM = list(DFrame_ISIC_in_RGB1.columns) + list(DFrame_ISIC_in_RGB2.columns) + list(DFrame_ISIC_in_RGB3.columns)\nRGB_Table = pd.DataFrame(data=RGB_Table, columns=columns_RGM) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RGB_Table = pd.DataFrame.transpose(RGB_Table)\nRGB_Table_Test = pd.DataFrame.transpose(DFrame_ISIC_in_TEST)\n\nRGB_Table  = RGB_Table.rename(columns={0: 'R', 1: 'G', 2: 'B'})\nRGB_Table_Test  = RGB_Table_Test.rename(columns={0: 'R', 1: 'G', 2: 'B'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RGB_Table_stat = pd.DataFrame(index=range(RGB_Table.shape[0]), dtype=np.float64,\n                       columns=['mean', 'des', 'std', 'max', 'min', 'quan0.25', 'quan0.5', 'quan0.75'])\n\nRGB_Table_stat_TEST = pd.DataFrame(index=range(RGB_Table_Test.shape[0]), dtype=np.float64,\n                       columns=['mean', 'des', 'std', 'max', 'min', 'quan0.25', 'quan0.5', 'quan0.75'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RGB_Table_stat = creation_ofstatistical_tables(RGB_Table, RGB_Table_stat)\nRGB_Table_stat_TEST = creation_ofstatistical_tables(RGB_Table_Test, RGB_Table_stat_TEST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RGB_Table_commonEnd = np.concatenate((RGB_Table, RGB_Table_stat), axis=1) \nRGB_Table_commonEnd_test = np.concatenate((RGB_Table_Test, RGB_Table_stat_TEST), axis=1) \n\nRGB_Table_commonEnd = pd.DataFrame(data=RGB_Table_commonEnd, columns=list(RGB_Table.columns) + list(RGB_Table_stat.columns),\n                                  index=RGB_Table.index) \n\nRGB_Table_commonEnd_test = pd.DataFrame(data=RGB_Table_commonEnd_test, columns=list(RGB_Table_Test.columns) + \n                                        list(RGB_Table_stat_TEST.columns), index=RGB_Table_Test.index) \n\nRGB_Table_commonEnd['l_image_name'] = RGB_Table_commonEnd.index.map(lambda x: str(x)[:-4]) \nRGB_Table_commonEnd_test['l_image_name'] = RGB_Table_commonEnd_test.index.map(lambda x: str(x)[:-4]) \n\ntrain_F = train_info.merge(RGB_Table_commonEnd, left_on='image_name', right_on='l_image_name')\ntest_F = test_info.merge(RGB_Table_commonEnd_test, left_on='image_name', right_on='l_image_name')\n\nDTrain = np.concatenate((train_F, DTrain_patient_Static), axis=1) \nDTest = np.concatenate((test_F, DTest_patient_Static), axis=1) \n\nDTrain_F = pd.DataFrame(data=DTrain, columns=list(train_F.columns) + list(DTrain_patient_Static.columns))                             \nDTest_F = pd.DataFrame(data=DTest, columns=list(test_F.columns) + list(DTest_patient_Static.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = DTrain_F.target\nDTrain_F = DTrain_F.drop(['image_name', 'patient_id', 'diagnosis', 'benign_malignant', 'target', 'l_image_name', \n                        'ST_image_name'], axis=1) \nDTest_F = DTest_F.drop(['image_name', 'patient_id', 'l_image_name', 'ST_image_name'], axis=1) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = Y.astype(float)\ntype_of_target(Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DTrain_F_encoder = DTrain_F[['sex', 'anatom_site_general_challenge']]\nDTest_F_encoder = DTest_F[['sex', 'anatom_site_general_challenge']]\nDTrain_F_encoder = pd.get_dummies(DTrain_F_encoder) \nDTest_F_encoder = pd.get_dummies(DTest_F_encoder) \n\nDTrain_F = pd.merge(DTrain_F.reset_index(), DTrain_F_encoder.reset_index())\nDTest_F = pd.merge(DTest_F.reset_index(), DTest_F_encoder.reset_index())\n\nDTrain_F = DTrain_F.drop(['index', 'sex', 'anatom_site_general_challenge'], axis=1) \nDTest_F = DTest_F.drop(['index', 'sex', 'anatom_site_general_challenge'], axis=1) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(DTrain_F.shape)\nprint(Y.shape)\nprint(DTest_F.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Scaling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler_imput = Pipeline([\n        (\"scaler\", MinMaxScaler())\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DTrain_F = pd.DataFrame(scaler_imput.fit_transform(DTrain_F), columns=DTrain_F.columns).astype(float)\nDTest_F = pd.DataFrame(scaler_imput.transform(DTest_F), columns=DTest_F.columns).astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DTrain_F.head(5)  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## XGBClassifier","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# distribution of values in the target variable\nprint('Y==0', '{}: {}'.format(sum(Y==0), sum(Y==0)/len(Y)))\nprint('Y==1', '{}: {}'.format(sum(Y==1), sum(Y==1)/len(Y)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(DTrain_F, Y, test_size=0.20, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('{}: {}'.format(sum(y_train==0), sum(y_train==0)/len(Y)))\nprint('{}: {}'.format(sum(y_train==1), sum(y_train==1)/len(Y)))\n\nprint('{}: {}'.format(sum(y_test==0), sum(y_test==0)/len(Y)))\nprint('{}: {}'.format(sum(y_test==1), sum(y_test==1)/len(Y)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {'colsample_bytree': 0.7887514489701739, \n         'learning_rate': 0.03952688683476441, \n         'max_depth': 5, \n         'min_child_weight': 3, \n         'n_estimators': 185, \n         'num_class': 2, \n         'objective': 'multi:softprob', \n         'subsample': 0.928924055966708,\n         'seed': 42 }\n\nclf_xgb = XGBClassifier(**param)  \nclf_xgb.fit(X_train, y_train, verbose = True, early_stopping_rounds=10, eval_metric='merror', eval_set=[(X_test, y_test)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import plot_confusion_matrix\nplot_confusion_matrix(clf_xgb, X_test, y_test, values_format='d', display_labels=['0', '1'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Due to the imbalance of our data towards objects of class 0, we have learned very well (perhaps even too much) to determine class 0 when we really have class 0 (TN)\n\nAt the same moment, we almost did not find objects with class 1 (lower right corner - TP), that is, we did not find any such object correctly, and at the same moment we made quite a lot of mistakes (lower left corner) when we attribute the object to the class 0, but in fact it is 1c, that is, we admit a large FN (error of the second kind)\n\nThe problem is that the data is not balanced, we use the parameter scale_pos_weight - Balancing of positive and negative weights.\nwhich adds a penalty for misclassifying the minority class\n\nAnd also reg_lambda which will add regularization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"param_dist = {'n_estimators': [385], #185\n              'learning_rate': [0.03952688683476441],\n              'subsample': [0.928924055966708],\n              'max_depth': [6], \n              'colsample_bytree': [0.7887514489701739],\n              'min_child_weight': [2],\n              'num_class': [2],\n              'objective': ['multi:softprob'],\n              'scale_pos_weight': [50],\n              'reg_lambda': [0.09893832910164219],\n             }\n\n\nmy_model = XGBClassifier()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"skfolds = StratifiedKFold(n_splits=N_SPLITS, random_state=SEED)\n    \nfor train_index, test_index in skfolds.split(X_train, y_train):\n    print('\\n{} of kfold {}'.format(I, skfolds.n_splits)) \n    \n    X_train_folds = DTrain_F.T[train_index]\n    y_train_folds = Y[train_index]\n    \n    #print(X_train_folds.shape, y_train_folds.shape)\n       \n    X_test_fold = DTrain_F.T[test_index]\n    y_test_fold = Y[test_index]\n    \n    print('{}: {}'.format(sum(y_train_folds==0), sum(y_train_folds==0)/len(Y)))\n    print('{}: {}'.format(sum(y_train_folds==1), sum(y_train_folds==1)/len(Y)))\n\n    print('{}: {}'.format(sum(y_test_fold==0), sum(y_test_fold==0)/len(Y)))\n    print('{}: {}'.format(sum(y_test_fold==1), sum(y_test_fold==1)/len(Y)))\n    \n    XGB_model = GridSearchCV(my_model, param_dist, cv=2,  scoring = 'roc_auc')\n    \n    XGB_model.fit(X_train_folds.T, y_train_folds)\n    print (XGB_model.best_params_)\n    \n    y_pred = XGB_model.predict(X_test_fold.T)\n    \n    print('accuracy_score', accuracy_score(y_test_fold, y_pred))\n    print('correct', sum(y_pred==y_test_fold)/len(y_pred))\n    I += 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = XGB_model.best_estimator_\nbest_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now let's build the error matrix again","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_matrix(best_model, X_test, y_test, values_format='d', display_labels=['0', '1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"best_model.fit(DTrain_F, Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred2 = best_model.predict_proba(DTest_F)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub2 = pd.DataFrame({'image_name': test_info['image_name'],\n                    'target': ypred2})\n#sub2.to_csv('submission2.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub2.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED) \n\nseed_everything(SEED)\ntrain_filenames = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\ntest_filenames = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames[0:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames,valid_filenames = train_test_split(train_filenames,test_size = 0.2,random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3) \n    image = tf.cast(image, tf.float32)/255.0 \n    image = tf.reshape(image, [*SIZE, 3]) \n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames): \n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    \n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"target\": tf.io.FixedLenFeature([], tf.int64),  } \n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32) \n    return image, label \n\ndef read_unlabeled_tfrecord(example):\n    \n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string), }\n    \n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT) \n    image = decode_image(example['image'])\n    image_name = example['image_name']\n    return image, image_name\n\ndef load_dataset(filenames, labeled=True, ordered=False): \n    \n    ignore_order = tf.data.Options() \n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    \n    dataset = (tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n              .with_options(ignore_order) \n              .map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO))\n            \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (load_dataset(train_filenames, labeled=True)\n    .shuffle(SEED)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .repeat()\n    .prefetch(AUTO))\n\nvalid_dataset = (load_dataset(valid_filenames, labeled=True)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset)\nprint(valid_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ret(a):\n    return  a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    \n    model= Sequential()\n\n    model.add(Lambda(ret, input_shape = (*SIZE, 3)))\n\n    model.add(Conv2D(64, (3,3), padding= 'same', activation = 'relu'))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(64, (3,3), padding= 'same', activation = 'relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(64, (3,3), padding= 'same', activation = 'relu'))\n    model.add(BatchNormalization())\n\n    model.add(Conv2D(32, (3,3), padding= 'same', activation = 'relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding=\"same\"))\n    model.add(BatchNormalization())\n    model.add(Flatten())\n\n    model.add(Dense(400, activation = 'relu'))\n    model.add(BatchNormalization())\n    #model.add(Dropout(0.4))\n    \n    model.add(Dense(300, activation = 'relu'))\n    model.add(BatchNormalization())\n    #model.add(Dropout(0.4))\n\n    model.add(Dense(100, activation = 'softmax'))\n    model.add(BatchNormalization())\n    #model.add(Dropout(0.4))\n\n    model.add(Dense(1, activation='sigmoid'))\n\n    model.compile(loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING), \n                  metrics = ['accuracy',tf.keras.metrics.AUC(name='auc')], optimizer = 'adam')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = count_data_items(train_filenames) // BATCH_SIZE\nprint(STEPS_PER_EPOCH)\nmodel_fit = model.fit(train_dataset, epochs=EPOCHS, steps_per_epoch=STEPS_PER_EPOCH, validation_data=valid_dataset) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = count_data_items(test_filenames)\ntest_dataset = (load_dataset(test_filenames, labeled=False,ordered=True)\n    .batch(BATCH_SIZE))\n\ntest_dataset_images = test_dataset.map(lambda image, image_name: image)\ntest_dataset_image_name = test_dataset.map(lambda image, image_name: image_name).unbatch()\ntest_ids = next(iter(test_dataset_image_name.batch(num_test_images))).numpy().astype('U')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = model.predict(test_dataset_images, verbose=1) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred_df.to_csv('pred_df.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_mean = sub2.merge(pred_df, left_on='image_name', right_on='image_name')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df_mean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred_mean = pd.DataFrame.from_dict({'image_name': list(pred_df_mean.image_name), \n#                                    'target': pred_df_mean[['target_x', 'target_y']].mean(axis=1)})\n\npred_mean3 = pd.DataFrame.from_dict({'image_name': list(pred_df_mean.image_name), \n                                    'target': 0.3 * pred_df_mean['target_x'] + pred_df_mean['target_y']})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred_mean.to_csv('pred_mean.csv',index = False)\n\npred_mean3.to_csv('pred_mean3.csv',index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}