{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport random\nimport numpy as np\nimport pandas as pd\nfrom scipy.stats import rankdata\nimport efficientnet.tfkeras as efn\nfrom matplotlib import pyplot as plt\nfrom sklearn.utils import class_weight\nfrom sklearn.metrics import roc_auc_score\nfrom kaggle_datasets import KaggleDatasets\nimport efficientnet.tfkeras as effn\nfrom tqdm import tqdm\nimport PIL\nimport re","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(\"running on tpu\",tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\nprint(\"Replicas\",strategy.num_replicas_in_sync)\n\n                                         ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gcs_paths = [x.split('/')[2:][0] for x in tf.io.gfile.glob('../input/*')]\ngcs_paths.sort()\ngcs_paths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_dict = {}\n\nfor path in gcs_paths:\n    path_dict[path] = KaggleDatasets().get_gcs_path(path)\n    print(f'{path}\\t| {KaggleDatasets().get_gcs_path(path)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 8*strategy.num_replicas_in_sync\nimage_size=[384,384]\ndim = image_size[0]\nautotune = tf.data.experimental.AUTOTUNE\nk = image_size[0]\n#gcs_path = KaggleDatasets().get_gcs_path(\"siim-isic-melanoma-classification\")\ntrain_file_path_2019 = tf.io.gfile.glob(path_dict[\"isic2019-1024x1024\"]+\"/train*.tfrec\")\ntrain_file_path_2020= tf.io.gfile.glob(path_dict[\"melanoma-1024x1024\"]+\"/train*.tfrec\")\n#sorted file list\ntest_file_path = tf.io.gfile.glob(path_dict[\"siim-isic-melanoma-classification\"]+\"/tfrecords/test*.tfrec\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"number of train files in 2020 data :\",len(train_file_path_2020))\nprint(\"number of train files in 2019 data :\",len(train_file_path_2019))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(42)\nnp.random.seed(42)\ntf.random.set_seed(42)\nautotune = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimage_size =[1024,1024]\nbatch_size = 8 * strategy.num_replicas_in_sync\ndim = image_size[0]\n\nval_skip = len(train_file_path_2019) // 5 #number of folds\nval_fname ={}\n\nindex = 0\nfor n in range(1,6):\n    val_fname[f\"fold_{n}\"] = train_file_path_2019[index:index+val_skip]\n    index+=val_skip\ntotal_data = train_file_path_2020 + train_file_path_2019\ntrain_fname = [list(set(total_data) - set(val_fname[f'fold_{n}'])) for n in range(1,6)]\ntrain_data_fname = [i for j in train_fname for i in j]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_fname[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format ={\n      \"image\" : tf.io.FixedLenFeature([],tf.string),\n      \"image_name\" : tf.io.FixedLenFeature([],tf.string),\n      \"patient_id\" : tf.io.FixedLenFeature([],tf.int64),\n      \"sex\"        : tf.io.FixedLenFeature([],tf.int64),\n      \"age_approx\" : tf.io.FixedLenFeature([],tf.int64),\n      \"anatom_site_general_challenge\" : tf.io.FixedLenFeature([],tf.int64),\n      \"diagnosis\"  : tf.io.FixedLenFeature([],tf.int64),\n      \"target\"     : tf.io.FixedLenFeature([],tf.int64),\n      #\"tfrecord\"   : tf.io.FixedLenFeature([],tf.int64,default_value=None),\n      #\"width\"      : tf.io.FixedLenFeature([],tf.int64,default_value=None)\n      \n  }    \n    example = tf.io.parse_single_example(example,tfrec_format)\n    tabular_data = [example[\"sex\"],example[\"age_approx\"],example[\"anatom_site_general_challenge\"]]\n\n    return example[\"image\"],example[\"target\"],tabular_data\ndef read_unlabeled_tfrecord(example):\n    tfrec_format ={\n      \"image\" : tf.io.FixedLenFeature([],tf.string),\n      \"image_name\" : tf.io.FixedLenFeature([],tf.string),\n      \"patient_id\" : tf.io.FixedLenFeature([],tf.int64),\n      \"sex\"        : tf.io.FixedLenFeature([],tf.string),\n      \"age_approx\" : tf.io.FixedLenFeature([],tf.int64),\n      \"anatom_site_general_challenge\" : tf.io.FixedLenFeature([],tf.int64),\n  }\n    example = tf.io.parse_single_example(example,tfrec_format)\n    tabular_data = [example[\"sex\"],example[\"age_approx\"],example[\"anatom_site_general_challenge\"]]\n    tabular_data = [tf.cast(tabular_data[feat],dtype=tf.float32) for feat in tabular_data]\n    tabular_data = tf.stack(tabular_data)\n    return example[\"image\"],example[\"image_name\"],tabular_data\n\ndef data_augment(img,label,tabular,train=True):\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.random_saturation(img,0.7,1.3)\n    img = tf.image.random_contrast(img,0.8,1.2)\n    img = tf.image.random_brightness(img,0.1)\n    return img,label,tabular\n\ndef prepare_image(img,label,tabular):\n    img = tf.image.decode_jpeg(img,channels=3)\n    img = tf.cast(img,tf.float32)/255.0\n    return img,label,tabular    \n\ndef get_dataset(files,labeled=True):\n    ds = tf.data.TFRecordDataset(files,num_parallel_reads=autotune)\n    ds =ds.cache()\n    if labeled:\n        ds = ds.map(read_labeled_tfrecord,num_parallel_calls = autotune)\n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        ds = ds.map(lambda img,label,tabular : prepare_image(img,label,tabular),num_parallel_calls=autotune)\n        ds = ds.map(lambda img,label,tabular : data_augment(img,label,tabular,train=True),num_parallel_calls=autotune)\n    else:\n        ds = ds.map(read_unlabeled_tfrecord,num_parallel_calls = autotune)\n        ds = ds.map(lambda img,img_name,tabular:prepare_image(image,label,tabular),num_parallel_calls=autotune)\n\n    ds = ds.batch(64)\n    ds = ds.prefetch(autotune)\n    return ds\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=pd.read_csv(\"../input/isic2019-1024x1024/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_dataset(thumb_size, cols, rows, ds):\n    mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols + (cols-1), \n                                             thumb_size*rows + (rows-1)))\n   \n    for idx, data in enumerate(iter(ds)):\n        img, target,tabular = data\n        ix  = idx % cols\n        iy  = idx // cols\n        img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n        img = PIL.Image.fromarray(img)\n        img = img.resize((thumb_size, thumb_size), resample=PIL.Image.BILINEAR)\n        mosaic.paste(img, (ix*thumb_size + ix, \n                           iy*thumb_size + iy))\n\n    display(mosaic)\n    \nds = get_dataset(train_data_fname).unbatch().take(12*5)   \nshow_dataset(64, 12, 5, ds)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callbacks\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 0.000005\n    lr_max     = 0.000020 * 8\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\nclass save_best_n(tf.keras.callbacks.Callback):\n    def __init__(self, fn, model):\n        self.fn = fn\n        self.model = model\n\n    def on_epoch_end(self, epoch, logs=None):\n        \n        if (epoch>0):\n            score=logs.get(\"val_auc\")\n        else:\n            score=-1\n      \n        if (score > best_score[fold_num].min()):\n          \n            idx_min=np.argmin(best_score[fold_num])\n\n            best_score[fold_num][idx_min]=score\n            best_epoch[fold_num][idx_min]=epoch+1\n\n            path_best_model=f'best_model_fold_{self.fn}_{idx_min}.hdf5'\n            self.model.save(\"/kaggle/working\"/path_best_model)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cnn():\n    with strategy.scope():\n        model= effn.EfficientNetB7(input_shape=(1024,1024,3),\n                                  weights=\"imagenet\",\n                                  include_top=False)\n        model.trainable= True\n        x=tf.keras.layers.GlobalAveragePooling2D()(model.output)\n        x = tf.keras.layers.Flatten()(x)\n        x = tf.keras.layers.Dense(512,kernel_regularizer = tf.keras.regularizers.l2(l=0.01),activation=\"relu\")(x)\n        x = tf.keras.layers.Dropout(0.2)(x)\n        x = tf.keras.layers.Dense(128,kernel_regularizer = tf.keras.regularizers.l2(l=0.01),activation = \"relu\")(x)\n        x = tf.keras.layers.Dropout(0.2)(x)\n        x = tf.keras.layers.Dense(64,kernel_regularizer=tf.keras.regularizers.l2(l=0.01),activation=\"relu\")(x)\n        x = tf.keras.layers.Dense(32,kernel_regularizer= tf.keras.regularizers.l2(l=0.01),activation=\"relu\")(x)\n        model = tf.keras.Model(model.input,x)\n        return model\ndef get_mlp():\n    with strategy.scope():\n        inp = tf.keras.layers.Input(shape=(3),name='inp2')\n        model = tf.keras.layers.Dense(16,activation=\"relu\",kernel_regularizer= tf.keras.regularizers.l2(l=0.01))(inp)\n        model = tf.keras.layers.Dense(32,activation=\"relu\",kernel_regularizer= tf.keras.regularizers.l2(l=0.01))(model)\n        moedl = tf.keras.layers.Dense(64,activation=\"relu\",kernel_regularizer= tf.keras.regularizers.l2(l=0.01))(model)\n        model = tf.keras.Model(inp,model)\n        return model\ndef get_model():\n    cnn = get_cnn()\n    mlp = get_mlp()\n    concat = tf.keras.layers.concatenate([cnn.output,mlp.output])\n    output = tf.keras.layers.Dense(1,activation=\"sigmoid\")(concat)\n    model = tf.keras.Model([cnn.input,mlp.input],output)\n    model.compile(optimizer=\"adam\",loss=\"binary_crossentropy\",metrics=[\"AUC\",\"accuracy\"])\n    return model\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(\n    model, to_file='model.png', show_shapes=False, show_layer_names=True,\n    rankdir='TB', expand_nested=False, dpi=96\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1))for filename in filenames]\n    return np.sum(n)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"step_size_train = count_data_items(train_data_fname)\nstep_size_validation= count_data_items(val_fname[\"fold_1\"])\nprint(step_size_train,step_size_validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def setup_input(image,label,tabular_data):\n    \n    return {\"inp1\":image,\"inp2\":tabular_data},label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"histories = []\nbest_epoch = {f:np.zeros(1) for f in range(1,6)}\nbest_score = {f:np.zeros(1) for f in range(1,6)}\nfor fold_num in range(1,6):\n    tf.keras.backend.clear_session()\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    print(\"-\"*50)\n    print(f\"strating fold {fold_num} out of 6\")\n    files_train = train_data_fname\n    files_val = val_fname[f\"fold_{fold_num}\"]\n    train_dataset = get_dataset(files_val)\n    train_dataset = train_dataset.map(setup_input,num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    \n    val_dataset = get_dataset(files_val)\n    val_dataset = val_dataset.map(setup_input,num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    \n    model = get_model()\n    history = model.fit(files_train,\n                       steps_per_epoch=step_size_train,\n                       epochs=15,\n                       validation_data=val_dataset,\n                       verbose=2,\n                       callbacks=[get_lr_callback(),save_best_n(fold_num,model)])\n    idx_sorted=np.argsort(best_score[fold_num])\n    best_score[fold_num]=np.array(best_score[fold_num])[idx_sorted]\n    best_epoch[fold_num]=np.array(best_epoch[fold_num])[idx_sorted]\n\n    print(f\"\\nFold {fold_num} is finished. The best epochs: {[int(best_epoch[fold_num][i]) for i in range(len(best_epoch[fold_num]))]}\")\n    print(f\"The corresponding scores: {[round(best_score[fold_num][i], 5) for i in range(len(best_epoch[fold_num]))]}\")\n\n    histories.append(history)         \n                          \n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}