{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"## EfficientNet\nfrom efficientnet import EfficientNetB0\n\neffnet = EfficientNetB0(include_top=False, weights=None, pooling='avg')\neffnet.load_weights('../input/efficientnet-keras-weights-b0b5/efficientnet-b0_imagenet_1000_notop.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Globals\nSZ = 224\nSEED = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false},"cell_type":"code","source":"## Prepare Data\nimport numpy as np\nimport pandas as pd\n\ntrain = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n\ntrain.diagnosis.hist(); train.diagnosis.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Our models will have a tendency to overfit to classes 0 and 2. A stratified train/test split will provide a validation set with a similar distribution as the training data, but will improperly test the model's ability to classify the disease intensity across the board. We need a validation set that will represent the model's true discriminative power. \n\nLet's take 15% of each class stratified, but limit the over-represented classes (0 and 2) to 15% of the least-represented class (3). "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train/Test Split\nnp.random.seed(SEED)\n\nlimit = int(193 * 0.15)\nzer = train[train.diagnosis == 0].sample(n=limit)\none = train[train.diagnosis == 1].sample(frac=0.15)\ntwo = train[train.diagnosis == 2].sample(n=limit)\nthr = train[train.diagnosis == 3].sample(frac=0.15)\nfou = train[train.diagnosis == 4].sample(frac=0.15)\nvalid = pd.concat([zer, one, two, thr, fou])\ntrain = train.drop(valid.index)\n\nxt, yt = train.id_code, train.diagnosis\nxv, yv = valid.id_code, valid.diagnosis\n\nyt.hist(); yv.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract\nfrom keras.utils.data_utils import _extract_archive\n\ndef extract_imgs(ds):\n    path = f'../input/aptos2019-224/{ds}_images.tar.gz'\n    _extract_archive(path)\n\ndef append_ext(fname):\n    if '_' in fname:\n        return fname + '.jpeg'\n    return fname + '.png'\n\nextract_imgs('train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Manage Preprocessing\nfrom keras.applications.imagenet_utils import preprocess_input\ndef preprocess(img):\n    img = preprocess_input(img, mode='torch')\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data Generator\nfrom keras.preprocessing.image import ImageDataGenerator\n\nbs = 128\nshape = (SZ, SZ, 3)\ntrain = pd.concat((xt, yt), axis=1).astype(str)\nvalid = pd.concat((xv, yv), axis=1).astype(str)\ntrain.id_code = train.id_code.apply(append_ext)\nvalid.id_code = valid.id_code.apply(append_ext)\ntrain_dir = './train_images'\n\ndef dataflow(df, mode='t'):\n    assert mode in ['t', 'v']\n    tfms = {'rotation_range': 360,\n            'preprocessing_function': preprocess}\n    if mode == 't': tfms.update({'zoom_range': 0.1,\n                                 'horizontal_flip': True,\n                                 'vertical_flip': True})\n    datagen = ImageDataGenerator(**tfms)\n    return datagen.flow_from_dataframe(\n        df, directory=train_dir, x_col='id_code', y_col='diagnosis',\n        target_size=(SZ, SZ), class_mode='sparse',\n        batch_size=bs, shuffle=True, seed=SEED,\n        validate_filenames=False, drop_duplicates=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Training\n# Metrics\n# https://www.kaggle.com/carlolepelaars/efficientnetb3-with-keras-aptos-2019\nfrom keras.callbacks import Callback\nfrom sklearn.metrics import cohen_kappa_score\n\ndef get_preds_and_labels(model, datagen):\n    \"\"\"\n    Get predictions and labels from the data generator.\n    Referencing `EmbedWrapper`, note that our data\n    generator outputs two y targets and our model does\n    the same.\n    \"\"\"\n    y_pred, y_true = [], []\n    for _ in range(datagen.samples // bs):\n        x, y = next(datagen)\n        yh = model.predict(x)\n        yh = np.argmax(yh, axis=1)\n        y_pred.append(yh)\n        y_true.append(y)\n    return np.concatenate(y_pred).ravel(), np.concatenate(y_true).ravel()\n\nclass QWK(Callback):\n    def on_train_begin(self, logs={}):\n        \"\"\"Initialize best score variable.\"\"\"\n        self.best = 0.\n    \n    def on_epoch_end(self, epoch, logs={}):\n        \"\"\"\n        Gets QWK score on the validation data and saves best weights.\n        Assumes both the model and validation data generator exist.\n        \"\"\"\n        y_pred, y_true = get_preds_and_labels(model, dataflow(valid, mode='v'))\n        y_pred = np.rint(y_pred).astype(np.uint8).clip(0, 4)\n        score = cohen_kappa_score(y_true, y_pred, weights='quadratic')\n        if score > self.best:\n            self.best = score\n            self.model.save_weights('weights.h5')\n            print(f'val_qwk: {score:.4f}')\n        return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model\nfrom keras import Input, Model\nfrom keras.layers import Dropout, Dense\nfrom keras.optimizers import SGD\n\nsgd = SGD(1e-4, decay=1e-5, momentum=0.9, nesterov=True, clipnorm=10.)\ndef effnet_aptos(shape):\n    i = Input(shape=shape)\n    x = effnet(i)\n    x = Dropout(0.2, seed=SEED)(x)\n    o = Dense(5, activation='softmax', kernel_initializer='zeros')(x)\n    model = Model(inputs=i, outputs=o)\n    for layer in model.layers[:-1]:\n        layer.trainable = False\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile Params\nloss = 'sparse_categorical_crossentropy'\nmetrics = ['accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile\nmodel = effnet_aptos(shape)\nmodel.compile(loss=loss, optimizer=sgd, metrics=metrics)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Top\ntsteps = np.ceil(len(train) / bs)\nvsteps = np.ceil(len(valid) / bs)\n\nhistory = model.fit_generator(dataflow(train), epochs=3, steps_per_epoch=tsteps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Unfreeze\nfor layer in model.layers:\n    layer.trainable = True\nmodel.compile(loss=loss, optimizer=sgd, metrics=metrics)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cosine Anneal, Model Checkpoint, QWK\nfrom cosineanneal import CosineAnnealingScheduler\nfrom keras.callbacks import ModelCheckpoint\n\nepochs = 30\ncallbacks = [CosineAnnealingScheduler(T_max=epochs, eta_max=1e-2, eta_min=1e-3),\n             QWK()]\n\n# Train Full\nhistory = model.fit_generator(dataflow(train),\n                              epochs=epochs,\n                              steps_per_epoch=tsteps,\n                              validation_data=dataflow(valid, mode='v'),\n                              validation_steps=vsteps,\n                              callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cleanup\nimport shutil\nshutil.rmtree('./train_images')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}