{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook I'll try to build a tf dataset that can be automatically balanced\n\nKey features\n* 512x512 -> 320x320 images\n* little data augmentation","metadata":{}},{"cell_type":"code","source":"!pip install efficientnet\nimport numpy as np\nimport pandas as pd\nimport os\nimport re\nimport random, math, time\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as effnet\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm import tqdm","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-09-05T08:05:52.863627Z","iopub.execute_input":"2023-09-05T08:05:52.863994Z","iopub.status.idle":"2023-09-05T08:06:02.398100Z","shell.execute_reply.started":"2023-09-05T08:05:52.863959Z","shell.execute_reply":"2023-09-05T08:06:02.397218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    print('No TPU')\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')\n\nprint(\"REPLICAS: %d\" % REPLICAS)\n\nfiles_train = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\nfiles_test = tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:02.400267Z","iopub.execute_input":"2023-09-05T08:06:02.400621Z","iopub.status.idle":"2023-09-05T08:06:03.304735Z","shell.execute_reply.started":"2023-09-05T08:06:02.400589Z","shell.execute_reply":"2023-09-05T08:06:03.303768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    img_read_size = 512,\n    img_crop_size = 400,\n    img_size      = 320,\n    \n    pos_ratio     = 0.2,\n    \n    batch_size    = 16 * REPLICAS,\n    epochs        = 40,\n    \n    initial_learning_rate = 1e-6,\n    min_learning_rate     = 1e-6,\n    max_learning_rate     = 5e-5 * REPLICAS,\n    rampup_epochs         = 5,\n    sustain_epochs        = 0,\n    exp_decay             = 0.8,\n    \n    test_batch_size       = 32 * REPLICAS,\n    aug_reps_in_test      = 20,\n    \n    rot               = 180.0,\n    shr               =   2.0,\n    hzoom             =   8.0,\n    wzoom             =   8.0,\n    hshift            =   8.0,\n    wshift            =   8.0\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:03.306299Z","iopub.execute_input":"2023-09-05T08:06:03.306668Z","iopub.status.idle":"2023-09-05T08:06:03.314602Z","shell.execute_reply.started":"2023-09-05T08:06:03.306630Z","shell.execute_reply":"2023-09-05T08:06:03.313638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3) \n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [config[\"img_read_size\"], config[\"img_read_size\"], 3])\n    return image\n\n\ndef _read_image_label(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label \n\n\ndef _read_image_meta_label(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"sex\": tf.io.FixedLenFeature([], tf.int64),\n        \"age_approx\": tf.io.FixedLenFeature([], tf.int64),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    meta = tf.stack([example[\"sex\"], example[\"age_approx\"]])\n    return image, meta, label \n\n\ndef _read_image_name(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    name = tf.cast(example['image_name'], tf.string)\n    return image, name\n\n\ndef _read_image_meta_name(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"sex\": tf.io.FixedLenFeature([], tf.int64),\n        \"age_approx\": tf.io.FixedLenFeature([], tf.int64),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    meta = tf.stack([example[\"sex\"], example[\"age_approx\"]])\n    name = tf.cast(example['image_name'], tf.string)\n    return image, meta, name \n\n\ndef load_dataset_image_label(filenames):\n    dataset = (\n        tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n        .map(_read_image_label, num_parallel_calls=AUTO)\n    )\n    return dataset\n\n\ndef load_dataset_image_meta_label(filenames):\n    dataset = (\n        tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n        .map(_read_image_meta_label, num_parallel_calls=AUTO)\n    )\n    return dataset\n\n\ndef load_dataset_image_name(filenames):\n    dataset = (\n        tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n        .map(_read_image_name, num_parallel_calls=AUTO)\n    )\n    return dataset\n\n\ndef load_dataset_image_meta_name(filenames):\n    dataset = (\n        tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) \n        .map(_read_image_meta_name, num_parallel_calls=AUTO)\n    )\n    return dataset\n\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:03.316167Z","iopub.execute_input":"2023-09-05T08:06:03.316813Z","iopub.status.idle":"2023-09-05T08:06:03.347111Z","shell.execute_reply.started":"2023-09-05T08:06:03.316774Z","shell.execute_reply":"2023-09-05T08:06:03.346291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = image.shape[1]\n    XDIM = DIM % 2 # fix for size 331\n    \n    rot = config['rot'] * tf.random.normal([1], dtype='float32')\n    shr = config['shr'] * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / config['hzoom']\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / config['wzoom']\n    h_shift = config['hshift'] * tf.random.normal([1], dtype='float32') \n    w_shift = config['wshift'] * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM // 2, -DIM // 2, -1), DIM)\n    y   = tf.tile(tf.range(-DIM // 2, DIM // 2), [DIM])\n    z   = tf.ones([DIM * DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM // 2 + XDIM + 1, DIM // 2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM // 2 - idx2[0,], DIM // 2 - 1 + idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d, [DIM, DIM, 3])","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:03.350550Z","iopub.execute_input":"2023-09-05T08:06:03.350941Z","iopub.status.idle":"2023-09-05T08:06:03.378770Z","shell.execute_reply.started":"2023-09-05T08:06:03.350903Z","shell.execute_reply":"2023-09-05T08:06:03.377796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aug_img(img, *args):\n    img = transform(img)\n    img = tf.image.random_crop(img, [config[\"img_crop_size\"], config[\"img_crop_size\"], 3])\n    img = tf.image.random_flip_left_right(img)\n    img = tf.image.random_flip_up_down(img)\n    img = tf.image.random_brightness(img, max_delta=0.5)\n    img = tf.image.random_hue(img, max_delta=0.2)\n    img = tf.image.random_saturation(img, 0.8, 1.2)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    return (img,) + tuple(args)\n\n\ndef finalize(img, *args):\n    img = tf.image.resize(img, [config[\"img_size\"], config[\"img_size\"]])\n    return (img,) + tuple(args)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:03.380467Z","iopub.execute_input":"2023-09-05T08:06:03.380879Z","iopub.status.idle":"2023-09-05T08:06:03.394231Z","shell.execute_reply.started":"2023-09-05T08:06:03.380826Z","shell.execute_reply":"2023-09-05T08:06:03.393352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nd0 = load_dataset_image_label(files_train[:-1]).filter(lambda i, l: (l <= 0)).repeat()\nd1 = load_dataset_image_label(files_train[:-1]).filter(lambda i, l: (l >= 1)).cache().repeat()\n\nchoice_dataset = tf.data.Dataset.range(2).repeat().map(lambda x: tf.cast(tf.random.uniform(shape=[]) < config['pos_ratio'], tf.int64))\n\ndata_train = (\n    tf.data.experimental.choose_from_datasets([d0, d1], choice_dataset)\n    .shuffle(1024)\n    .map(aug_img, num_parallel_calls=AUTO)\n    .map(finalize, num_parallel_calls=AUTO)\n    .batch(config[\"batch_size\"])\n    .prefetch(AUTO)\n)\n\ndata_val = (\n    load_dataset_image_label(files_train[-1:])\n    .map(finalize, num_parallel_calls=AUTO)\n    .batch(config[\"batch_size\"])\n    .prefetch(AUTO)\n)\n\ndata_test = (\n    load_dataset_image_name(files_test)\n    .map(aug_img, num_parallel_calls=AUTO)\n    .map(finalize, num_parallel_calls=AUTO)\n    .batch(config[\"test_batch_size\"])\n    .prefetch(AUTO)\n)\n\nprint(\"Train\", data_train)\nprint(\"Val  \", data_val)\nprint(\"Test \", data_test)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:03.395714Z","iopub.execute_input":"2023-09-05T08:06:03.396220Z","iopub.status.idle":"2023-09-05T08:06:07.389783Z","shell.execute_reply.started":"2023-09-05T08:06:03.396175Z","shell.execute_reply":"2023-09-05T08:06:07.389004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Estimating train dataset performance\")\nones = 0\nN = 200\nfor i, (imgs, labels) in tqdm(enumerate(data_train), total=N):\n    ones += labels.numpy().sum()\n    if i >= N:\n        break\nprint(\"Average number of positives:\\t%.4f\" % (ones / (N * REPLICAS * config['batch_size'])))","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:06:07.391167Z","iopub.execute_input":"2023-09-05T08:06:07.391534Z","iopub.status.idle":"2023-09-05T08:08:37.802803Z","shell.execute_reply.started":"2023-09-05T08:06:07.391495Z","shell.execute_reply":"2023-09-05T08:08:37.801783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K.clear_session()\nmodel = keras.Sequential([\n    effnet.EfficientNetB5(weights='imagenet', include_top=False),\n    keras.layers.GlobalAveragePooling2D(),\n    keras.layers.Dense(1024, activation='relu'),\n    keras.layers.Dropout(0.2),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.Dropout(0.2),\n    keras.layers.Dense(256, activation='relu'),\n    keras.layers.Dense(1, activation='sigmoid')\n])\nmodel.compile(\n    optimizer=keras.optimizers.Adam(1e-4), \n    loss=keras.losses.BinaryCrossentropy(), \n    metrics=[keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:08:37.806190Z","iopub.execute_input":"2023-09-05T08:08:37.806474Z","iopub.status.idle":"2023-09-05T08:08:51.970995Z","shell.execute_reply.started":"2023-09-05T08:08:37.806445Z","shell.execute_reply":"2023-09-05T08:08:51.970005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(epoch):\n    if epoch < config['rampup_epochs']:\n        return config['initial_learning_rate'] + (config['max_learning_rate'] - config['initial_learning_rate']) * epoch / config['rampup_epochs']\n    elif epoch < config['rampup_epochs'] + config['sustain_epochs']:\n        return max_lr\n    else:\n        return config['min_learning_rate'] + (config['max_learning_rate'] - config['min_learning_rate']) * config['exp_decay'] ** (epoch - config['rampup_epochs'] - config['sustain_epochs'])\n    pass\n\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n\nlrs = []\nfor e in range(config['epochs']):\n    lrs.append(lrfn(e))\nplt.plot(lrs)\nplt.ylim(0)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:08:51.972467Z","iopub.execute_input":"2023-09-05T08:08:51.972855Z","iopub.status.idle":"2023-09-05T08:08:52.140615Z","shell.execute_reply.started":"2023-09-05T08:08:51.972799Z","shell.execute_reply":"2023-09-05T08:08:52.139565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_POS_APPROX = 584\nSTEPS_PER_EPOCH = int(np.ceil(N_POS_APPROX / config['pos_ratio'] / config['batch_size']))\nhistory = model.fit(\n    data_train,\n    epochs          = config['epochs'],\n    steps_per_epoch = STEPS_PER_EPOCH,\n    validation_data = data_val,\n    callbacks       = [lr_callback]\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:08:52.142079Z","iopub.execute_input":"2023-09-05T08:08:52.142505Z","iopub.status.idle":"2023-09-05T09:48:30.265336Z","shell.execute_reply.started":"2023-09-05T08:08:52.142462Z","shell.execute_reply":"2023-09-05T09:48:30.264360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'], color='r',label= 'Training loss')\nplt.plot(history.history['val_loss'], color='g', label= 'validation loss')\nplt.title('Training and Validation loss')\nplt.figure()\nplt.plot(history.history['auc'], color='r', label= 'Training accuracy' )\nplt.plot(history.history['val_auc'], color='g', label= 'validation accuracy')\nplt.title('Training and Validation Accuracy')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-05T09:48:30.267672Z","iopub.execute_input":"2023-09-05T09:48:30.268119Z","iopub.status.idle":"2023-09-05T09:48:30.565234Z","shell.execute_reply.started":"2023-09-05T09:48:30.268084Z","shell.execute_reply":"2023-09-05T09:48:30.564410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names = list(data_test.map(lambda i, n: n).as_numpy_iterator())\nnames = np.vectorize(lambda x: x.decode('utf-8'))(np.concatenate(names, 0))\npreds = model.predict(data_test.map(lambda i, n: i), verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T09:48:30.566487Z","iopub.execute_input":"2023-09-05T09:48:30.566780Z","iopub.status.idle":"2023-09-05T09:53:30.534540Z","shell.execute_reply.started":"2023-09-05T09:48:30.566750Z","shell.execute_reply":"2023-09-05T09:53:30.531821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names = list(data_test.map(lambda i, n: n).as_numpy_iterator())\nnames = np.vectorize(lambda x: x.decode('utf-8'))(np.concatenate(names, 0))","metadata":{"execution":{"iopub.status.busy":"2023-09-05T09:53:30.541619Z","iopub.execute_input":"2023-09-05T09:53:30.544379Z","iopub.status.idle":"2023-09-05T09:55:42.227074Z","shell.execute_reply.started":"2023-09-05T09:53:30.544257Z","shell.execute_reply":"2023-09-05T09:55:42.225942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = [model.predict(data_test.map(lambda i, n: i), verbose=1) for rep in range(config['aug_reps_in_test'])]","metadata":{"execution":{"iopub.status.busy":"2023-09-05T09:55:42.228720Z","iopub.execute_input":"2023-09-05T09:55:42.229136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({'image_name': names, 'target': np.concatenate(preds, 1).mean(1)}).groupby('image_name').mean().reset_index().to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}