{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import everything, set up seeds","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install efficientnet > /dev/null\n\nimport os\nimport random\nimport re\nimport math\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport PIL as pil\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as be\nfrom tensorflow.keras.models import *\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.metrics import *\nfrom tensorflow.keras.losses import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.regularizers import *\nfrom tensorflow.keras.callbacks import *\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\nfrom kaggle_datasets import KaggleDatasets\nfrom tqdm import tqdm\n\nimport efficientnet.tfkeras as efn\n\nSEED = 42\nrandom.seed(a=SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read data for all CSVs and Tfrecs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"local_path = '/kaggle/input/siim-isic-melanoma-classification'\ndf_train = pd.read_csv(os.path.join(local_path, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(local_path, 'test.csv'))\ndf_submission = pd.read_csv(os.path.join(local_path, 'sample_submission.csv'))\n\nprocessed_path = KaggleDatasets().get_gcs_path('melanoma-256x256')\ntraining = np.sort(np.array(tf.io.gfile.glob(processed_path + '/train*.tfrec')))\ntesting = np.sort(np.array(tf.io.gfile.glob(processed_path + '/test*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create folds for later","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 5\n\nX_train = [None] * FOLDS\nX_val = [None] * FOLDS\n\nkfold = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)\nfor fold, (id_train, id_val) in enumerate(kfold.split(np.arange(len(training)))):\n    X_train[fold] = tf.io.gfile.glob([processed_path + '/train%.2i*.tfrec' % x for x in id_train])\n    X_val[fold] = tf.io.gfile.glob([processed_path + '/train%.2i*.tfrec' % x for x in id_val])\n    \nX_test = testing","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Class to use TPU / GPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Processor():\n    def __init__(self, device='TPU'):\n        self.device = device\n        self.tpu_resolve()\n        self.tpu_initialize()\n        self.gpu()\n        self.replicas = 1\n        self.strategy = []\n\n    def tpu_resolve(self):\n        try:\n            self.tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n            print('Running on TPU', self.tpu.master())\n        except ValueError:\n            self.tpu = None\n            self.device = 'GPU'\n        \n    def tpu_initialize(self):\n        if self.tpu and self.device == 'TPU':\n            try:\n                tf.config.experimental_connect_to_cluster(self.tpu)\n                tf.tpu.experimental.initialize_tpu_system(self.tpu)\n                self.strategy = tf.distribute.experimental.TPUStrategy(self.tpu)\n                self.replicas = self.strategy.num_replicas_in_sync\n            except _:\n                print('Failed to initialize TPU Cluster')\n                self.device = 'GPU'\n                \n    def gpu(self):\n        if self.device == 'GPU':\n            gpu_devices = len(tf.config.experimental.list_physical_devices('GPU'))\n            if gpu_devices > 0:\n                self.strategy = tf.distribute.get_strategy()\n                self.replicas = self.strategy.num_replicas_in_sync\n                print('Connected to GPU with', gpu_devices, 'devices')\n            else:\n                self.device = 'CPU'\n                print('Connected to CPU')\n        \nproc = Processor()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parse the dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset():\n    def __init__(self, files, is_training=True, augment=False, batch_size=16):\n        self.is_training = is_training\n        self.files = files\n        \n        auto = tf.data.experimental.AUTOTUNE\n\n        ds = tf.data.TFRecordDataset(files, num_parallel_reads=auto)\n        ds = ds.cache()\n        ds = ds.map(self.read, num_parallel_calls=auto)\n        ds = ds.map( lambda img, img_or_target: (self.prepare_image(img, augment), img_or_target) , \n                    num_parallel_calls=auto)\n        ds = ds.batch(16)\n        ds = ds.prefetch(auto)\n        self.dataset = ds\n    \n    def stream(self):\n        return self.dataset\n        \n    def read(self, item):\n        if self.is_training:\n            item = tf.io.parse_single_example(item, {\n                'image': tf.io.FixedLenFeature([], tf.string),\n                'target': tf.io.FixedLenFeature([], tf.int64),\n            })\n            return item['image'], item['target']\n        else:\n            item = tf.io.parse_single_example(item, {\n                'image': tf.io.FixedLenFeature([], tf.string),\n                'image_name': tf.io.FixedLenFeature([], tf.string),        \n            })\n            return item['image'], item['image_name']\n    \n    def prepare_image(self, img, augment=False):\n        read = 256\n\n        img = tf.image.decode_jpeg(img, channels=3)\n        img = tf.cast(img, tf.float32) / 255.0\n        \n        if augment:\n            img = self.transform(img, read)\n            img = tf.image.random_flip_left_right(img)\n            img = tf.image.random_saturation(img, 0.7, 1.3)\n            img = tf.image.random_contrast(img, 0.8, 1.2)\n            img = tf.image.random_brightness(img, 0.1)\n    \n        img = tf.reshape(img, [read, read, 3])\n        return img\n\n    def count(self):\n        n = [ int(re.compile(r\"-([0-9]*)\\.\").search(name).group(1)) for name in self.files ]\n        return np.sum(n)\n    \n    def transform(self, image, DIM=256):\n        # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n        # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n        # output - image randomly rotated, sheared, zoomed, and shifted\n        XDIM = DIM%2 #fix for size 331\n\n        rot = 180.0 * tf.random.normal([1], dtype='float32')\n        shr = 2.0 * tf.random.normal([1], dtype='float32') \n        h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n        w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n        h_shift = 8.0 * tf.random.normal([1], dtype='float32') \n        w_shift = 8.0 * tf.random.normal([1], dtype='float32') \n\n        # GET TRANSFORMATION MATRIX\n        m = self.get_matrix(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n        # LIST DESTINATION PIXEL INDICES\n        x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n        y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n        z   = tf.ones([DIM*DIM], dtype='int32')\n        idx = tf.stack( [x,y,z] )\n\n        # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n        idx2 = be.dot(m, tf.cast(idx, dtype='float32'))\n        idx2 = be.cast(idx2, dtype='int32')\n        idx2 = be.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n\n        # FIND ORIGIN PIXEL VALUES           \n        idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n        d    = tf.gather_nd(image, tf.transpose(idx3))\n\n        return tf.reshape(d,[DIM, DIM,3])\n    \n    \n    def get_matrix(self, rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n        # https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n        # returns 3x3 transformmatrix which transforms indicies\n        \n        # CONVERT DEGREES TO RADIANS\n        rotation = math.pi * rotation / 180.\n        shear    = math.pi * shear    / 180.\n\n        def get_3x3_mat(lst):\n            return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n        # ROTATION MATRIX\n        c1   = tf.math.cos(rotation)\n        s1   = tf.math.sin(rotation)\n        one  = tf.constant([1],dtype='float32')\n        zero = tf.constant([0],dtype='float32')\n\n        rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                       -s1,  c1,   zero, \n                                       zero, zero, one])    \n        # SHEAR MATRIX\n        c2 = tf.math.cos(shear)\n        s2 = tf.math.sin(shear)    \n\n        shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                    zero, c2,   zero, \n                                    zero, zero, one])        \n        # ZOOM MATRIX\n        zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                                   zero,            one/width_zoom, zero, \n                                   zero,            zero,           one])    \n        # SHIFT MATRIX\n        shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                    zero, one,  width_shift, \n                                    zero, zero, one])\n\n        return be.dot(be.dot(rotation_matrix, shear_matrix), \n                     be.dot(zoom_matrix,     shift_matrix))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test by parsing a value from testing and training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = Dataset(training, augment=True).stream()\nds = ds.unbatch()\nds = ds.take(1)\n\nfor idx, data in enumerate(iter(ds)):\n    img, img_or_target = data\n    img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n    img = pil.Image.fromarray(img)\n    img = img.resize((400,400), resample=pil.Image.BILINEAR)\n    plt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = Dataset(testing, is_training=False).stream()\nds = ds.unbatch()\nds = ds.take(1)\n\nfor idx, data in enumerate(iter(ds)):\n    img, img_or_target = data\n    img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n    img = pil.Image.fromarray(img)\n    img = img.resize((400,400), resample=pil.Image.BILINEAR)\n    plt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\n    Dataset(training).count(),\n    Dataset(testing, is_training=False).count()\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build a model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model():\n    def __init__(self, name):\n        self.name = name\n        i = Input(shape=(400, 400, 3))\n        base = efn.EfficientNetB2(weights='imagenet', input_shape=(400, 400, 3), include_top=False)\n        x = base(i)\n        x = GlobalAveragePooling2D()(x)\n#         x = BatchNormalization()(x)\n#         x = Dense(256, activation='relu', kernel_regularizer=l2(0.01))(x)\n#         x = BatchNormalization()(x)\n#         x = Dropout(0.1)(x)\n        x = Dense(1, activation='sigmoid')(x)\n        model = tf.keras.Model(inputs=i, outputs=x)\n        model.compile(\n            optimizer=Adam(learning_rate=0.001),\n            loss=BinaryCrossentropy(label_smoothing=0.05),\n            metrics=['binary_crossentropy', AUC(name='auc') ] # 'accuracy'\n        )\n        self.model = model\n\n    def get(self):\n        return self.model\n        \n    def fit(self, train, validate, epochs=10, batch_size=64):\n        self.batch_size = batch_size\n        # tf.keras.utils.plot_model(self.model, show_shapes=True)\n        history = self.model.fit(\n            train.stream(),\n            validation_data=validate.stream(),\n            verbose=1,\n            epochs=epochs,\n            steps_per_epoch=train.count()/batch_size,\n            batch_size=batch_size,\n            callbacks=[\n#                 EarlyStopping(monitor='auc', mode='max', patience=6, verbose=2, restore_best_weights=True),\n                ModelCheckpoint(monitor='val_auc', verbose=1, save_best_only=True, mode='max', \n                                filepath='{val_auc:.5f}_'+self.name+'.h5'),\n                # ReduceLROnPlateau(monitor='auc', factor=0.5, patience=3, verbose=1, mode='auto', cooldown=1, min_lr=0 ),\n                self.get_learning_rate(),\n            ],\n        )\n        return history\n    \n    def get_learning_rate(self):\n        lr_start = 0.000005\n        lr_max = 0.00000125 * self.batch_size\n        lr_min = 0.000001\n        lr_ramp_ep = 5\n        lr_sus_ep = 0\n        lr_decay = 0.8\n   \n        def lrfn(epoch):\n            if epoch < lr_ramp_ep:\n                lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            elif epoch < lr_ramp_ep + lr_sus_ep:\n                lr = lr_max\n            else:\n                lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            return lr\n\n        lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n        return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !rm 0.4*.h5\n# !rm 0.5*.h5\n# !rm 0.6*.h5\n# !rm 0.7*.h5\n# !rm 0.8*.h5\n# # !rm *.png","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nn = Model('efn_pool_batch_256BD10x1')\n\nBATCH_SIZE=256\n\nfor i in range(FOLDS):\n    train = Dataset(X_train[i], augment=True, batch_size=BATCH_SIZE)\n    validate = Dataset(X_val[i], batch_size=BATCH_SIZE)\n    history = nn.fit(train, validate, epochs=30, batch_size=BATCH_SIZE)\n    \n    print(history.history)\n    \n    plt.plot(range(len(history.history['val_auc'])), history.history['val_auc'], color='green')\n    plt.plot(range(len(history.history['auc'])), history.history['auc'], color='red')\n    plt.plot(range(len(history.history['binary_crossentropy'])), history.history['binary_crossentropy'], color='blue')\n    plt.plot(range(len(history.history['val_binary_crossentropy'])), history.history['val_binary_crossentropy'], color='purple')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# mm = nn.get()\n# sub_model = tf.keras.models.load_model('0.90119_efn_pool_batch_256BD10x1.h5') # 0.8681\n# sub_model = tf.keras.models.load_model('0.89918_efn_pool_batch_256BD10x1.h5') # 0.8591\n# Longer epochs, 16\n# sub_model = tf.keras.models.load_model('0.92810_efn_pool_batch_256BD10x1.h5') # 0.9055\nsub_model = tf.keras.models.load_model('0.93747_efn_pool_batch_256BD10x1.h5') # 0.9074\nds = Dataset(testing, is_training=False)\npredicted = sub_model.predict(ds.stream(), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame(dict(\n    image_name = np.array([img_name.numpy().decode('utf-8') for img, img_name in iter(ds.stream().unbatch()) ]),\n    target = np.reshape(predicted, (1, predicted.shape[0]))[0]\n))\n\nsub.to_csv('efn_pool_batch3_submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}