{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":2198.01327,"end_time":"2023-09-03T09:52:21.933272","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-09-03T09:15:43.920002","version":"2.3.4"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ☀️ Imports and Setup","metadata":{"papermill":{"duration":0.017945,"end_time":"2023-09-03T09:15:46.526932","exception":false,"start_time":"2023-09-03T09:15:46.508987","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import re\nimport os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tempfile\nimport matplotlib as mlp\nimport matplotlib.pyplot as plt\nimport sklearn\nimport math\n\nfrom PIL import Image\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow import keras\nfrom keras import layers, callbacks\nfrom keras.losses import BinaryCrossentropy\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom keras import backend as K\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n    \nprint(tf.__version__)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":51.160223,"end_time":"2023-09-03T09:16:37.703032","exception":false,"start_time":"2023-09-03T09:15:46.542809","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:53:59.994024Z","iopub.execute_input":"2023-09-04T13:53:59.995122Z","iopub.status.idle":"2023-09-04T13:54:09.728391Z","shell.execute_reply.started":"2023-09-04T13:53:59.995082Z","shell.execute_reply":"2023-09-04T13:54:09.727542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mlp.rcParams['figure.figsize'] = (12, 10)\ncolors = plt.rcParams['axes.prop_cycle'].by_key()['color']","metadata":{"papermill":{"duration":0.025047,"end_time":"2023-09-03T09:16:37.745567","exception":false,"start_time":"2023-09-03T09:16:37.720520","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:09.730163Z","iopub.execute_input":"2023-09-04T13:54:09.731511Z","iopub.status.idle":"2023-09-04T13:54:09.737386Z","shell.execute_reply.started":"2023-09-04T13:54:09.731476Z","shell.execute_reply":"2023-09-04T13:54:09.736118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🦆 Hyperparameters","metadata":{"papermill":{"duration":0.017105,"end_time":"2023-09-03T09:16:37.781033","exception":false,"start_time":"2023-09-03T09:16:37.763928","status":"completed"},"tags":[]}},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH = KaggleDatasets().get_gcs_path()\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [1024, 1024]\nIMAGE_RESIZE = [380, 380]\nEPOCHS = 50","metadata":{"papermill":{"duration":0.026473,"end_time":"2023-09-03T09:16:37.824359","exception":false,"start_time":"2023-09-03T09:16:37.797886","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:09.738683Z","iopub.execute_input":"2023-09-04T13:54:09.738969Z","iopub.status.idle":"2023-09-04T13:54:10.270829Z","shell.execute_reply.started":"2023-09-04T13:54:09.738943Z","shell.execute_reply":"2023-09-04T13:54:10.269867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔨 Prepare Dataset","metadata":{"papermill":{"duration":0.01702,"end_time":"2023-09-03T09:16:37.858778","exception":false,"start_time":"2023-09-03T09:16:37.841758","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec'),\n    test_size=0.1, random_state=5\n)\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')\nprint('Train TFRecord Files:', len(TRAINING_FILENAMES))\nprint('Validation TFRecord Files:', len(VALID_FILENAMES))\nprint('Test TFRecord Files:', len(TEST_FILENAMES))","metadata":{"papermill":{"duration":0.051882,"end_time":"2023-09-03T09:16:37.927520","exception":false,"start_time":"2023-09-03T09:16:37.875638","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:10.273241Z","iopub.execute_input":"2023-09-04T13:54:10.273663Z","iopub.status.idle":"2023-09-04T13:54:11.709188Z","shell.execute_reply.started":"2023-09-04T13:54:10.273635Z","shell.execute_reply":"2023-09-04T13:54:11.708306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","metadata":{"papermill":{"duration":0.025386,"end_time":"2023-09-03T09:16:38.013564","exception":false,"start_time":"2023-09-03T09:16:37.988178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.716991Z","iopub.execute_input":"2023-09-04T13:54:11.717417Z","iopub.status.idle":"2023-09-04T13:54:11.731659Z","shell.execute_reply.started":"2023-09-04T13:54:11.717381Z","shell.execute_reply":"2023-09-04T13:54:11.730831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum","metadata":{"papermill":{"duration":0.027718,"end_time":"2023-09-03T09:16:38.058593","exception":false,"start_time":"2023-09-03T09:16:38.030875","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.732921Z","iopub.execute_input":"2023-09-04T13:54:11.733810Z","iopub.status.idle":"2023-09-04T13:54:11.742178Z","shell.execute_reply.started":"2023-09-04T13:54:11.733763Z","shell.execute_reply":"2023-09-04T13:54:11.741457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset","metadata":{"papermill":{"duration":0.027203,"end_time":"2023-09-03T09:16:38.102708","exception":false,"start_time":"2023-09-03T09:16:38.075505","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.743793Z","iopub.execute_input":"2023-09-04T13:54:11.744611Z","iopub.status.idle":"2023-09-04T13:54:11.752602Z","shell.execute_reply.started":"2023-09-04T13:54:11.744577Z","shell.execute_reply":"2023-09-04T13:54:11.751801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data augmentation","metadata":{"papermill":{"duration":0.016759,"end_time":"2023-09-03T09:16:38.136606","exception":false,"start_time":"2023-09-03T09:16:38.119847","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))","metadata":{"execution":{"iopub.status.busy":"2023-09-04T13:54:11.753776Z","iopub.execute_input":"2023-09-04T13:54:11.754563Z","iopub.status.idle":"2023-09-04T13:54:11.765743Z","shell.execute_reply.started":"2023-09-04T13:54:11.754532Z","shell.execute_reply":"2023-09-04T13:54:11.764818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 180. * tf.random.normal([1],dtype='float32')\n    shr = 2. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/8.\n    h_shift = 8. * tf.random.normal([1],dtype='float32') \n    w_shift = 8. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]),label","metadata":{"papermill":{"duration":0.032937,"end_time":"2023-09-03T09:16:38.235684","exception":false,"start_time":"2023-09-03T09:16:38.202747","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.770044Z","iopub.execute_input":"2023-09-04T13:54:11.770688Z","iopub.status.idle":"2023-09-04T13:54:11.783518Z","shell.execute_reply.started":"2023-09-04T13:54:11.770644Z","shell.execute_reply":"2023-09-04T13:54:11.782376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation_pipeline(image, label):\n    image,_ = transform(image,label)\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.7, 1.3)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    image = tf.image.random_brightness(image, 0.1)\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    return image, label","metadata":{"papermill":{"duration":0.027526,"end_time":"2023-09-03T09:16:38.281288","exception":false,"start_time":"2023-09-03T09:16:38.253762","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.784843Z","iopub.execute_input":"2023-09-04T13:54:11.785186Z","iopub.status.idle":"2023-09-04T13:54:11.797868Z","shell.execute_reply.started":"2023-09-04T13:54:11.785144Z","shell.execute_reply":"2023-09-04T13:54:11.796928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_pipeline(image, label):\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    return image, label","metadata":{"papermill":{"duration":0.02469,"end_time":"2023-09-03T09:16:38.323542","exception":false,"start_time":"2023-09-03T09:16:38.298852","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.799296Z","iopub.execute_input":"2023-09-04T13:54:11.800376Z","iopub.status.idle":"2023-09-04T13:54:11.808253Z","shell.execute_reply.started":"2023-09-04T13:54:11.800321Z","shell.execute_reply":"2023-09-04T13:54:11.807415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define loading methods","metadata":{"papermill":{"duration":0.017048,"end_time":"2023-09-03T09:16:38.357896","exception":false,"start_time":"2023-09-03T09:16:38.340848","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"papermill":{"duration":0.026773,"end_time":"2023-09-03T09:16:38.401800","exception":false,"start_time":"2023-09-03T09:16:38.375027","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.809897Z","iopub.execute_input":"2023-09-04T13:54:11.810561Z","iopub.status.idle":"2023-09-04T13:54:11.819459Z","shell.execute_reply.started":"2023-09-04T13:54:11.810530Z","shell.execute_reply":"2023-09-04T13:54:11.818446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.map(reshape_pipeline, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"papermill":{"duration":0.025636,"end_time":"2023-09-03T09:16:38.444886","exception":false,"start_time":"2023-09-03T09:16:38.419250","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.821051Z","iopub.execute_input":"2023-09-04T13:54:11.821486Z","iopub.status.idle":"2023-09-04T13:54:11.842799Z","shell.execute_reply.started":"2023-09-04T13:54:11.821454Z","shell.execute_reply":"2023-09-04T13:54:11.841714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.map(reshape_pipeline, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"papermill":{"duration":0.025868,"end_time":"2023-09-03T09:16:38.488273","exception":false,"start_time":"2023-09-03T09:16:38.462405","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.843889Z","iopub.execute_input":"2023-09-04T13:54:11.844667Z","iopub.status.idle":"2023-09-04T13:54:11.855288Z","shell.execute_reply.started":"2023-09-04T13:54:11.844634Z","shell.execute_reply":"2023-09-04T13:54:11.854207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"papermill":{"duration":0.025975,"end_time":"2023-09-03T09:16:38.531687","exception":false,"start_time":"2023-09-03T09:16:38.505712","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.856677Z","iopub.execute_input":"2023-09-04T13:54:11.856987Z","iopub.status.idle":"2023-09-04T13:54:11.866086Z","shell.execute_reply.started":"2023-09-04T13:54:11.856959Z","shell.execute_reply":"2023-09-04T13:54:11.865275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint(\n    'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n        NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES\n    )\n)","metadata":{"papermill":{"duration":0.026079,"end_time":"2023-09-03T09:16:38.575104","exception":false,"start_time":"2023-09-03T09:16:38.549025","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.867463Z","iopub.execute_input":"2023-09-04T13:54:11.868032Z","iopub.status.idle":"2023-09-04T13:54:11.882087Z","shell.execute_reply.started":"2023-09-04T13:54:11.868001Z","shell.execute_reply":"2023-09-04T13:54:11.880987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest_csv = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","metadata":{"papermill":{"duration":0.139173,"end_time":"2023-09-03T09:16:38.731633","exception":false,"start_time":"2023-09-03T09:16:38.592460","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:11.883335Z","iopub.execute_input":"2023-09-04T13:54:11.883664Z","iopub.status.idle":"2023-09-04T13:54:12.031998Z","shell.execute_reply.started":"2023-09-04T13:54:11.883637Z","shell.execute_reply":"2023-09-04T13:54:12.030644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_img = train_csv['target'].size\n\nmalignant = np.count_nonzero(train_csv['target'])\nbenign = total_img - malignant\n\nprint('Examples:\\n    Total: {}\\n    Positive: {} ({:.2f}% of total)\\n'.format(\n    total_img, malignant, 100 * malignant / total_img))","metadata":{"papermill":{"duration":0.027142,"end_time":"2023-09-03T09:16:38.776500","exception":false,"start_time":"2023-09-03T09:16:38.749358","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:12.033653Z","iopub.execute_input":"2023-09-04T13:54:12.034573Z","iopub.status.idle":"2023-09-04T13:54:12.044612Z","shell.execute_reply.started":"2023-09-04T13:54:12.034539Z","shell.execute_reply":"2023-09-04T13:54:12.043514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()","metadata":{"papermill":{"duration":1.102135,"end_time":"2023-09-03T09:16:39.896329","exception":false,"start_time":"2023-09-03T09:16:38.794194","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:12.046025Z","iopub.execute_input":"2023-09-04T13:54:12.046936Z","iopub.status.idle":"2023-09-04T13:54:13.538213Z","shell.execute_reply.started":"2023-09-04T13:54:12.046894Z","shell.execute_reply":"2023-09-04T13:54:13.537171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch(ds):\n    plt.figure(figsize=(20,4))\n    for idx, data in enumerate(iter(ds)):\n        ax = plt.subplot(2,10,idx+1)\n        img, target = data\n        img = img.numpy()\n        plt.imshow(img)\n        if target.numpy():\n            plt.title(\"MALIGNANT\")\n        else:\n            plt.title(\"BENIGN\")\n        plt.axis(\"off\")","metadata":{"papermill":{"duration":0.027509,"end_time":"2023-09-03T09:16:39.941734","exception":false,"start_time":"2023-09-03T09:16:39.914225","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:13.539793Z","iopub.execute_input":"2023-09-04T13:54:13.540230Z","iopub.status.idle":"2023-09-04T13:54:13.547276Z","shell.execute_reply.started":"2023-09-04T13:54:13.540198Z","shell.execute_reply":"2023-09-04T13:54:13.546167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#show_batch(train_dataset.unbatch().take(20))","metadata":{"papermill":{"duration":14.639457,"end_time":"2023-09-03T09:16:54.598574","exception":false,"start_time":"2023-09-03T09:16:39.959117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:13.548689Z","iopub.execute_input":"2023-09-04T13:54:13.549022Z","iopub.status.idle":"2023-09-04T13:54:13.564543Z","shell.execute_reply.started":"2023-09-04T13:54:13.548992Z","shell.execute_reply":"2023-09-04T13:54:13.563174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Examples WITH Melanoma')\nimgs = train_csv.loc[train_csv.target==1].sample(10).image_name.values\nplt.figure(figsize=(20,8))\nfor i,k in enumerate(imgs):\n    plt.subplot(2,10,i+1); plt.axis('off')\n    img = Image.open('../input/siim-isic-melanoma-classification/jpeg/train/%s.jpg'%k)\n    img = img.resize(IMAGE_RESIZE)\n    plt.imshow(img)\nplt.show()\n\nprint('\\n\\nExamples WITHOUT Melanoma')\nplt.figure(figsize=(20,8))\nimgs = train_csv.loc[train_csv.target==0].sample(10).image_name.values\nfor i,k in enumerate(imgs):\n    plt.subplot(2,10,i+1); plt.axis('off')\n    img = Image.open('../input/siim-isic-melanoma-classification/jpeg/train/%s.jpg'%k)\n    img = img.resize(IMAGE_RESIZE)\n    plt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-04T13:54:54.615853Z","iopub.execute_input":"2023-09-04T13:54:54.616309Z","iopub.status.idle":"2023-09-04T13:55:00.638157Z","shell.execute_reply.started":"2023-09-04T13:54:54.616277Z","shell.execute_reply":"2023-09-04T13:55:00.636556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔥 Build the Model","metadata":{"papermill":{"duration":0.025241,"end_time":"2023-09-03T09:16:54.651545","exception":false,"start_time":"2023-09-03T09:16:54.626304","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# def exponential_decay(lr0, s):\n#     def exponential_decay_fn(epoch):\n#         return lr0 * 0.1 **(epoch / s)\n#     return exponential_decay_fn\n\n# exponential_decay_fn = exponential_decay(0.01, 20)\n\n# lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn)","metadata":{"papermill":{"duration":0.033303,"end_time":"2023-09-03T09:16:54.709833","exception":false,"start_time":"2023-09-03T09:16:54.676530","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:18.708716Z","iopub.execute_input":"2023-09-04T13:54:18.709255Z","iopub.status.idle":"2023-09-04T13:54:18.713664Z","shell.execute_reply.started":"2023-09-04T13:54:18.709224Z","shell.execute_reply":"2023-09-04T13:54:18.712271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"papermill":{"duration":0.303765,"end_time":"2023-09-03T09:16:55.039112","exception":false,"start_time":"2023-09-03T09:16:54.735347","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:18.715230Z","iopub.execute_input":"2023-09-04T13:54:18.715665Z","iopub.status.idle":"2023-09-04T13:54:19.089239Z","shell.execute_reply.started":"2023-09-04T13:54:18.715622Z","shell.execute_reply":"2023-09-04T13:54:19.088133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model(output_bias = None, metrics = None):    \n    if output_bias is not None:\n        output_bias = tf.keras.initializers.Constant(output_bias)\n        \n    base_model = tf.keras.applications.DenseNet201(input_shape=(*IMAGE_RESIZE, 3),\n                                                   include_top=False,\n                                                   weights='imagenet',\n                                                   pooling='avg')\n    \n    base_model.trainable = True\n    \n    model = keras.Sequential([\n        base_model,\n        layers.Dense(1, activation='sigmoid',\n                              bias_initializer=output_bias)\n    ])\n    \n    model.compile(optimizer='adam',\n                  loss=BinaryCrossentropy(label_smoothing=0.05),\n                  metrics=metrics)\n    \n    return model","metadata":{"papermill":{"duration":0.037811,"end_time":"2023-09-03T09:16:55.104323","exception":false,"start_time":"2023-09-03T09:16:55.066512","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:19.090536Z","iopub.execute_input":"2023-09-04T13:54:19.090830Z","iopub.status.idle":"2023-09-04T13:54:19.099438Z","shell.execute_reply.started":"2023-09-04T13:54:19.090803Z","shell.execute_reply":"2023-09-04T13:54:19.097994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE","metadata":{"papermill":{"duration":0.033704,"end_time":"2023-09-03T09:16:55.164400","exception":false,"start_time":"2023-09-03T09:16:55.130696","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:19.101070Z","iopub.execute_input":"2023-09-04T13:54:19.101511Z","iopub.status.idle":"2023-09-04T13:54:19.110651Z","shell.execute_reply.started":"2023-09-04T13:54:19.101474Z","shell.execute_reply":"2023-09-04T13:54:19.109417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"initial_bias = np.log([malignant/benign])\ninitial_bias","metadata":{"papermill":{"duration":0.036633,"end_time":"2023-09-03T09:16:55.227326","exception":false,"start_time":"2023-09-03T09:16:55.190693","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:19.117686Z","iopub.execute_input":"2023-09-04T13:54:19.118142Z","iopub.status.idle":"2023-09-04T13:54:19.126367Z","shell.execute_reply.started":"2023-09-04T13:54:19.118112Z","shell.execute_reply":"2023-09-04T13:54:19.125262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_for_0 = (1 / benign)*(total_img)/2.0 \nweight_for_1 = (1 / malignant)*(total_img)/2.0\n\nclass_weight = {0: weight_for_0, 1: weight_for_1}\n\nprint('Weight for class 0: {:.2f}'.format(weight_for_0))\nprint('Weight for class 1: {:.2f}'.format(weight_for_1))","metadata":{"papermill":{"duration":0.037316,"end_time":"2023-09-03T09:16:55.293940","exception":false,"start_time":"2023-09-03T09:16:55.256624","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:19.127785Z","iopub.execute_input":"2023-09-04T13:54:19.128283Z","iopub.status.idle":"2023-09-04T13:54:19.138097Z","shell.execute_reply.started":"2023-09-04T13:54:19.128253Z","shell.execute_reply":"2023-09-04T13:54:19.136814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = make_model(output_bias = initial_bias, metrics=tf.keras.metrics.AUC(name='auc'))","metadata":{"papermill":{"duration":67.53421,"end_time":"2023-09-03T09:18:02.855591","exception":false,"start_time":"2023-09-03T09:16:55.321381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:19.139845Z","iopub.execute_input":"2023-09-04T13:54:19.140337Z","iopub.status.idle":"2023-09-04T13:54:31.007789Z","shell.execute_reply.started":"2023-09-04T13:54:19.140308Z","shell.execute_reply":"2023-09-04T13:54:31.006916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_cb = ModelCheckpoint(\"melanoma_model.h5\",save_best_only=True)\nearly_stopping_cb = EarlyStopping(patience=10,restore_best_weights=True)","metadata":{"papermill":{"duration":0.036178,"end_time":"2023-09-03T09:18:02.922387","exception":false,"start_time":"2023-09-03T09:18:02.886209","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:31.009815Z","iopub.execute_input":"2023-09-04T13:54:31.010659Z","iopub.status.idle":"2023-09-04T13:54:31.016996Z","shell.execute_reply.started":"2023-09-04T13:54:31.010617Z","shell.execute_reply":"2023-09-04T13:54:31.015910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚅 Train","metadata":{"papermill":{"duration":0.027597,"end_time":"2023-09-03T09:18:02.977470","exception":false,"start_time":"2023-09-03T09:18:02.949873","status":"completed"},"tags":[]}},{"cell_type":"code","source":"history = model.fit(\n    train_dataset, epochs=EPOCHS,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    validation_data=valid_dataset,\n    validation_steps=VALID_STEPS,\n    callbacks=[checkpoint_cb, early_stopping_cb, lr_callback],\n    class_weight=class_weight\n)","metadata":{"papermill":{"duration":1973.140987,"end_time":"2023-09-03T09:50:56.146791","exception":false,"start_time":"2023-09-03T09:18:03.005804","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:31.018833Z","iopub.execute_input":"2023-09-04T13:54:31.019157Z","iopub.status.idle":"2023-09-04T13:54:46.052588Z","shell.execute_reply.started":"2023-09-04T13:54:31.019129Z","shell.execute_reply":"2023-09-04T13:54:46.051011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📈 Evaluation","metadata":{"papermill":{"duration":0.258588,"end_time":"2023-09-03T09:50:56.706469","exception":false,"start_time":"2023-09-03T09:50:56.447881","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_metrics(history):\n  metrics = ['loss', 'auc']\n  for n, metric in enumerate(metrics):\n    name = metric.replace(\"_\",\" \").capitalize()\n    plt.subplot(2,2,n+1)\n    plt.plot(history.epoch, history.history[metric], color=colors[0], label='Train')\n    plt.plot(history.epoch, history.history['val_'+metric],\n             color=colors[0], linestyle=\"--\", label='Val')\n    plt.xlabel('Epoch')\n    plt.ylabel(name)\n    plt.legend()","metadata":{"papermill":{"duration":0.270727,"end_time":"2023-09-03T09:50:57.235875","exception":false,"start_time":"2023-09-03T09:50:56.965148","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.053524Z","iopub.status.idle":"2023-09-04T13:54:46.054000Z","shell.execute_reply.started":"2023-09-04T13:54:46.053751Z","shell.execute_reply":"2023-09-04T13:54:46.053770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_metrics(history)","metadata":{"papermill":{"duration":2.419773,"end_time":"2023-09-03T09:50:59.910860","exception":false,"start_time":"2023-09-03T09:50:57.491087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.055484Z","iopub.status.idle":"2023-09-04T13:54:46.055904Z","shell.execute_reply.started":"2023-09-04T13:54:46.055705Z","shell.execute_reply":"2023-09-04T13:54:46.055726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📝 Prediction","metadata":{"papermill":{"duration":0.255947,"end_time":"2023-09-03T09:51:00.426448","exception":false,"start_time":"2023-09-03T09:51:00.170501","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)","metadata":{"papermill":{"duration":60.629856,"end_time":"2023-09-03T09:52:01.313378","exception":false,"start_time":"2023-09-03T09:51:00.683522","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.057732Z","iopub.status.idle":"2023-09-04T13:54:46.058144Z","shell.execute_reply.started":"2023-09-04T13:54:46.057954Z","shell.execute_reply":"2023-09-04T13:54:46.057973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📃 Submission file","metadata":{"papermill":{"duration":0.269223,"end_time":"2023-09-03T09:52:01.903090","exception":false,"start_time":"2023-09-03T09:52:01.633867","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nsub.head()","metadata":{"papermill":{"duration":0.304482,"end_time":"2023-09-03T09:52:02.474090","exception":false,"start_time":"2023-09-03T09:52:02.169608","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.059268Z","iopub.status.idle":"2023-09-04T13:54:46.059661Z","shell.execute_reply.started":"2023-09-04T13:54:46.059473Z","shell.execute_reply":"2023-09-04T13:54:46.059492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')","metadata":{"papermill":{"duration":8.071455,"end_time":"2023-09-03T09:52:10.814023","exception":false,"start_time":"2023-09-03T09:52:02.742568","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.060819Z","iopub.status.idle":"2023-09-04T13:54:46.061196Z","shell.execute_reply.started":"2023-09-04T13:54:46.061002Z","shell.execute_reply":"2023-09-04T13:54:46.061020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})\npred_df.head()","metadata":{"papermill":{"duration":0.29988,"end_time":"2023-09-03T09:52:11.387443","exception":false,"start_time":"2023-09-03T09:52:11.087563","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.062497Z","iopub.status.idle":"2023-09-04T13:54:46.062884Z","shell.execute_reply.started":"2023-09-04T13:54:46.062695Z","shell.execute_reply":"2023-09-04T13:54:46.062713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sub['target']\nsub = sub.merge(pred_df, on='image_name')\nsub.to_csv('submission.csv', index=False)\nsub.head()","metadata":{"papermill":{"duration":0.346059,"end_time":"2023-09-03T09:52:12.008654","exception":false,"start_time":"2023-09-03T09:52:11.662595","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-09-04T13:54:46.063997Z","iopub.status.idle":"2023-09-04T13:54:46.064394Z","shell.execute_reply.started":"2023-09-04T13:54:46.064186Z","shell.execute_reply":"2023-09-04T13:54:46.064203Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}