{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet > /dev/null","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Melanoma Detection \n\nBare Bone Modularized code written in tensorflow 2.x for melanoma detection It could be good starting point to implement your strategy on top of it.\nIn this kernel I tried to build a pipeline that is easy to maintain and provide greater extend to modification\n\n**Note:** This kernel works on any hardware out of the box without any extra configuration \n\n### Things that are Implemented\n[TFrec-loader] --> [tf.Data.Dataset()] --> [PreProcess functions] --> [tf sequential model with pretrained weights] --> [training instructions]\n\n### Things that can be strategically implemented\n* Cross Validation[Folds]\n* Augmentation\n* Model Tuning\n* etc,","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\n\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom kaggle_datasets import KaggleDatasets\n\nimport numpy as np\n\ntf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class config:\n    GCS_DS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\n    TRAIN_CSV = '../input/siim-isic-melanoma-classification/train.csv'\n    TEST_CSV = '../input/siim-isic-melanoma-classification/test.csv'\n    TRAIN_FILES = tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords/train*')\n    TEST_FILES = tf.io.gfile.glob(GCS_DS_PATH + '/tfrecords/test*')\n    VALIDATION_CSV = \"\"\n    \n    TOTAL_TRAIN_IMG = 0\n    TOTAL_TEST_IMG = 0\n    \n    IMG_SIZE = [1024, 1024]\n    IMG_RESHAPE = [512,512]\n    IMG_SHAPE = (512, 512, 3)\n    DO_FINETUNE = True\n    \n    BATCH_SIZE = 8\n    BUFFER_SIZE = 100\n    EPOCHES = 10 \n    \n    LOSS = tf.keras.losses.BinaryCrossentropy()\n    OPTIMIZER = tf.keras.optimizers.Adam(learning_rate=0.01)\n    ACCURACY = ['accuracy']\n    \n    STRATEGY = None\n    \n    LOG_DIR = './log'\n    CHECKPOINT_DIR = './log/checkpoint/cp.cpkt'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    config.STRATEGY = strategy\n    config.BATCH_SIZE = 8 * strategy.num_replicas_in_sync \nelse:\n    strategy = tf.distribute.get_strategy() \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nconfig.TOTAL_TRAIN_IMG = len(pd.read_csv(config.TRAIN_CSV).image_name)\nconfig.TOTAL_TEST_IMG = len(pd.read_csv(config.TEST_CSV).image_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    model = tf.keras.Sequential([\n                efn.EfficientNetB0(\n                    input_shape=config.IMG_SHAPE,\n                    weights='imagenet',\n                    include_top=False\n                ),\n                tf.keras.layers.GlobalAveragePooling2D(),\n                tf.keras.layers.Dense(1, activation='sigmoid')\n            ])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Helper Functions\ndef process_training_data(data_file):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"target\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    data = tf.io.parse_single_example(data_file, LABELED_TFREC_FORMAT)\n    img = tf.image.decode_jpeg(data['image'], channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [*config.IMG_SIZE, 3])\n    img = tf.image.resize(img, config.IMG_RESHAPE)\n\n    \n    label = tf.cast(data['target'], tf.int32)\n\n    return img, label\n\ndef process_test_data(data_file):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n    }\n    data = tf.io.parse_single_example(data_file, LABELED_TFREC_FORMAT)\n    img = tf.image.decode_jpeg(data['image'], channels=3)\n    img = tf.cast(img, tf.float32) / 255.0\n    img = tf.reshape(img, [*config.IMG_SIZE, 3])\n    img = tf.image.resize(img, config.IMG_RESHAPE)\n\n    \n    idnum = data['image_name']\n\n    return img, idnum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_engine(model,dataset):\n    model.compile(\n            optimizer=config.OPTIMIZER, \n            loss=config.LOSS, \n            metrics=config.ACCURACY\n        )\n    history = model.fit(\n        dataset, \n        epochs=config.EPOCHES, \n        steps_per_epoch=(config.TOTAL_TRAIN_IMG//config.BATCH_SIZE),\n    )\n\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run():\n    #Creating Dataset\n    dataset = (\n        tf.data.TFRecordDataset(\n            config.TRAIN_FILES,  \n            num_parallel_reads=tf.data.experimental.AUTOTUNE\n        ).map(\n            process_training_data,\n            num_parallel_calls=tf.data.experimental.AUTOTUNE\n        ).repeat(\n        ).shuffle(\n            buffer_size=config.BUFFER_SIZE\n        ).batch(\n            config.BATCH_SIZE\n        ).prefetch(\n            tf.data.experimental.AUTOTUNE\n        )\n    )\n    \n    #Setup model and train\n    if config.STRATEGY is not None:\n        with strategy.scope():\n            model = get_model()\n    else:\n        model = get_model()\n        \n    history = fit_engine(model, dataset)\n        \n    return model, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model, history = run()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = (\n    tf.data.TFRecordDataset(\n        config.TEST_FILES,  \n        num_parallel_reads=tf.data.experimental.AUTOTUNE\n    ).map(\n        process_test_data,\n        num_parallel_calls=tf.data.experimental.AUTOTUNE\n    ).batch(\n        config.BATCH_SIZE\n    ).prefetch(\n        tf.data.experimental.AUTOTUNE\n    )\n)\n\ntest_imgs = test_dataset.map(lambda images, ids: images)\nimg_ids_ds = test_dataset.map(lambda images, ids: ids).unbatch()\n\npredictions = model.predict(test_imgs).flatten()\n\nimg_ids = []\nfor coutner, ids in enumerate(img_ids_ds):\n    if coutner%500 == 0:\n        print(coutner)\n    img_ids.append(ids.numpy())\n\nimg_ids = np.array(img_ids).astype('U')\n\nnp.savetxt(\n    'sample_submission.csv', \n    np.rec.fromarrays([img_ids, predictions]), \n    fmt=['%s', '%f'], \n    delimiter=',', \n    header='image_name,target', \n    comments=''\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('sample_submission.csv').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}