{"cells":[{"metadata":{"_uuid":"a4564135dfdccf82159309eb7a9bf2fc3fa6d84d"},"cell_type":"markdown","source":"- Stratified K fold + optimal LR with keras"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Sequential\nfrom keras.models import model_from_json\nfrom keras.layers import Conv2D, MaxPooling2D, MaxPool2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization\nfrom keras.optimizers import RMSprop, Adam\nfrom keras import backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.optimizers import SGD\nfrom keras.callbacks import EarlyStopping,ReduceLROnPlateau\n\nfrom keras.layers import Dense\nfrom keras.models import Model\nimport keras\nimport os\nimport glob\nfrom skimage.io import imread\nfrom keras.preprocessing import image\n\n\nprint(os.listdir(\"../input\"))\n\ntrain_dir = \"../input/train/\"\ntest_dir = \"../input/test/\"\n\ndataset= pd.read_csv('../input/train_labels.csv',dtype='str')\n\ndef append_ext(fn):\n    return fn+\".tif\"\ndataset[\"id\"]=dataset[\"id\"].apply(append_ext)\n\ndatapath='../input/'\ntrain_path = datapath+'train'\nvalid_path =  datapath+'train'\ntest_path=datapath+'test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc22cb3685d8acb22290528c097b725d3806a65e"},"cell_type":"code","source":"testing_files = glob.glob(os.path.join(test_dir,'*.tif'))\nTESTING_BATCH_SIZE=10\n\nsubmission = pd.DataFrame()\ntest_files = glob.glob(os.path.join(test_dir,'*.tif'))\nsubmission = pd.DataFrame()\nmax_idx = len(test_files)\nfile_batch=5000\nprint('reading files')\n\ntest_df = pd.DataFrame({'path': test_files})\nprint(test_df.shape)\ntest_df['id'] = test_df.path.map(lambda x: x.split('/')[3].split(\".\")[0])\ntest_df['image'] = test_df['path'].map(imread)\ntest_df['label']=0\nprint('end of reading')\nX_test = np.stack(test_df['image'].values)\nX_test = keras.applications.resnet50.preprocess_input(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"013d7515a497c33b103227cf8cbea4c5f8f5e397"},"cell_type":"code","source":"k_folds=1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63995e10973e3eb6a81c64942b4294368f67f341"},"cell_type":"code","source":"#Import Keras\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, Dropout, MaxPooling2D, Flatten, Dense\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.convolutional import SeparableConv2D\nfrom keras.layers.core import Activation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54dfb35243a438bb012315c5f941dcfd69ee0080"},"cell_type":"code","source":"# from https://www.kaggle.com/soumya044/histopathologic-cancer-detection\nclass CancerNet:\n    @staticmethod\n    def build(width, height, depth, classes):\n        \n        # initialize the model along with the input shape to be\n        # \"channels last\" and the channels dimension itself\n        model = Sequential()\n        inputShape = (height, width, depth)\n        chanDim = -1\n        \n        # CONV => RELU => POOL\n        model.add(SeparableConv2D(32, (3, 3), padding=\"same\",input_shape = inputShape))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n\n        # (CONV => RELU => POOL) * 2\n        model.add(SeparableConv2D(64, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(SeparableConv2D(64, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n\n        # (CONV => RELU => POOL) * 3\n        model.add(SeparableConv2D(128, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(SeparableConv2D(128, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(SeparableConv2D(128, (3, 3), padding=\"same\"))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization(axis=chanDim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n        \n        # first (and only) set of FC => RELU layers\n        model.add(Flatten())\n        model.add(Dense(256))\n        model.add(Activation(\"relu\"))\n        model.add(BatchNormalization())\n        model.add(Dropout(0.2))\n\n        # softmax classifier\n        model.add(Dense(classes))\n        model.add(Activation(\"sigmoid\"))\n        \n        model.summary()\n\n        # return the constructed network architecture\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2e17ed07f25911b768d387b143427a3682a3282"},"cell_type":"code","source":"model= CancerNet.build(width = 96, height = 96, depth = 3, classes = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7514f2f7761902106cdca50419480c31acc4f143"},"cell_type":"code","source":"# source :https://github.com/surmenok/keras_lr_finder\nfrom matplotlib import pyplot as plt\nimport math\nfrom keras.callbacks import LambdaCallback\nimport keras.backend as K\n\n\nclass LRFinder:\n    \"\"\"\n    Plots the change of the loss function of a Keras model when the learning rate is exponentially increasing.\n    See for details:\n    https://towardsdatascience.com/estimating-optimal-learning-rate-for-a-deep-neural-network-ce32f2556ce0\n    \"\"\"\n    def __init__(self, model):\n        self.model = model\n        self.losses = []\n        self.lrs = []\n        self.best_loss = 1e9\n\n    def on_batch_end(self, batch, logs):\n        # Log the learning rate\n        lr = K.get_value(self.model.optimizer.lr)\n        self.lrs.append(lr)\n\n        # Log the loss\n        loss = logs['loss']\n        self.losses.append(loss)\n\n        # Check whether the loss got too large or NaN\n        if math.isnan(loss) or loss > self.best_loss * 4:\n            self.model.stop_training = True\n            return\n\n        if loss < self.best_loss:\n            self.best_loss = loss\n\n        # Increase the learning rate for the next batch\n        lr *= self.lr_mult\n        K.set_value(self.model.optimizer.lr, lr)\n\n    def find(self, x_train, y_train, start_lr, end_lr, batch_size=64, epochs=1):\n        num_batches = epochs * x_train.shape[0] / batch_size\n        self.lr_mult = (float(end_lr) / float(start_lr)) ** (float(1) / float(num_batches))\n\n        # Save weights into a file\n        self.model.save_weights('tmp.h5')\n\n        # Remember the original learning rate\n        original_lr = K.get_value(self.model.optimizer.lr)\n\n        # Set the initial learning rate\n        K.set_value(self.model.optimizer.lr, start_lr)\n\n        callback = LambdaCallback(on_batch_end=lambda batch, logs: self.on_batch_end(batch, logs))\n\n        self.model.fit(x_train, y_train,\n                        batch_size=batch_size, epochs=epochs,\n                        callbacks=[callback])\n\n        # Restore the weights to the state before model fitting\n        self.model.load_weights('tmp.h5')\n\n        # Restore the original learning rate\n        K.set_value(self.model.optimizer.lr, original_lr)\n\n    def plot_loss(self, n_skip_beginning=10, n_skip_end=5):\n        \"\"\"\n        Plots the loss.\n        Parameters:\n            n_skip_beginning - number of batches to skip on the left.\n            n_skip_end - number of batches to skip on the right.\n        \"\"\"\n        plt.ylabel(\"loss\")\n        plt.xlabel(\"learning rate (log scale)\")\n        plt.plot(self.lrs[n_skip_beginning:-n_skip_end], self.losses[n_skip_beginning:-n_skip_end])\n        plt.xscale('log')\n\n    def plot_loss_change(self, sma=1, n_skip_beginning=10, n_skip_end=5, y_lim=(-0.01, 0.01)):\n        \"\"\"\n        Plots rate of change of the loss function.\n        Parameters:\n            sma - number of batches for simple moving average to smooth out the curve.\n            n_skip_beginning - number of batches to skip on the left.\n            n_skip_end - number of batches to skip on the right.\n            y_lim - limits for the y axis.\n        \"\"\"\n        assert sma >= 1\n        derivatives = [0] * sma\n        for i in range(sma, len(self.lrs)):\n            derivative = (self.losses[i] - self.losses[i - sma]) / sma\n            derivatives.append(derivative)\n\n        plt.ylabel(\"rate of loss change\")\n        plt.xlabel(\"learning rate (log scale)\")\n        plt.plot(self.lrs[n_skip_beginning:-n_skip_end], derivatives[n_skip_beginning:-n_skip_end])\n        plt.xscale('log')\n        plt.ylim(y_lim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b2ceb671b33a1336be5e4d4c2e119a6e1976f35"},"cell_type":"code","source":"sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)\nadam=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\nfor i in range(len(model.layers)):\n    model.layers[i].trainable = True\nmodel.compile(loss='binary_crossentropy', optimizer=adam, metrics=['acc'])\n\nearly_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2)\nlr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=2, min_lr=0.5e-6)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70d9703039d0f679fc3f16a44debe57c372e0d57"},"cell_type":"code","source":"lr_finder = LRFinder(model)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f95243b561e51bce43317f31facdcb0fffb2e799"},"cell_type":"markdown","source":"Optimal learning rate finding https://github.com/surmenok/keras_lr_finder"},{"metadata":{"trusted":true,"_uuid":"275aafa61fedf79e7e4aaddc525c0fe8a5d6071b"},"cell_type":"code","source":"#help function\ndef append_adress(fn):\n    return \"../input/train/\"+fn ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa649f2c0a59e24df7c45a5ff15d8b36d2df7503"},"cell_type":"code","source":"#optimal LR will be search on 0.005 fraction of dataset\ndataset_lr_finder=dataset.sample(frac=0.01)\nprint(dataset_lr_finder.columns)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"367a09d063674d43bfedc490ef9340e1ec99a248"},"cell_type":"code","source":"dataset_lr_finder[\"path\"]=dataset_lr_finder[\"id\"].apply(append_adress) # adding image's path on optimal LR dataset\nprint(dataset_lr_finder.head(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4fc942f3adb7efc2ae2fa852b4fd5f03dff47dc"},"cell_type":"code","source":"dataset_lr_finder['image'] = dataset_lr_finder['path'].map(imread)\nx_train = np.stack(dataset_lr_finder['image'].values)\nx_train = keras.applications.resnet50.preprocess_input(x_train)\ny_train =dataset_lr_finder.label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4392f00c196dc19e6cc207c71f5b18697039ce84"},"cell_type":"code","source":"# Train a model with batch size 512 for 5 epochs\n# with learning rate growing exponentially from 0.0001 to 1\nlr_finder.find(x_train, y_train, start_lr=0.000001, end_lr=0.3, batch_size=100, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dcca38bc2365e99b81d51959e1d17d69af31e53"},"cell_type":"code","source":"lr_finder.plot_loss()\n#3.5.10-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5266af5656405bbedb488603639106febcb453b5"},"cell_type":"code","source":"sgd = SGD(lr=3e-3, decay=1e-6, momentum=0.9, nesterov=True)\nmodel.compile(loss='binary_crossentropy', optimizer=adam, metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af1e069c39f3e441e08e131f47a2d7972214018f"},"cell_type":"code","source":"import keras\ntrain_datagen=ImageDataGenerator(\n #   width_shift_range=0.5,\n #   height_shift_range=0.5,\n #   shear_range=0.5,\n #   zoom_range=0.5,\n #   horizontal_flip=True,vertical_flip=True,\n #   rotation_range=90,fill_mode = 'nearest',\n    rescale=1/255,validation_split=0.3\n    \n  #  preprocessing_function= keras.applications.resnet50.preprocess_input\n    )\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f17ae52491350681e24f95a3f56a29c6aa6085d4"},"cell_type":"code","source":"train_generator = train_datagen.flow_from_dataframe(dataframe=dataset,directory=train_path,x_col = 'id',y_col = 'label',has_ext=False,\n                subset='training',\n                target_size=(96, 96),\n                batch_size=32,\n                class_mode='binary'\n                )\nSTEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size\n    \n\nvalidation_generator = train_datagen.flow_from_dataframe(\n                dataframe=dataset,\n                directory=valid_path,\n                x_col = 'id',\n                y_col = 'label',\n                has_ext=False,\n                subset='validation', # This is the trick to properly separate train and validation dataset\n                target_size=(96, 96),\n                batch_size=32,\n                shuffle=False,\n                class_mode='binary'\n                )\nSTEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size\nmodel.fit_generator(generator=train_generator,\n              steps_per_epoch=STEP_SIZE_TRAIN,\n              nb_epoch=10,\n              shuffle=True,verbose=1,\n              callbacks=[lr_reducer, early_stop],\n              validation_data=validation_generator,\n              validation_steps=STEP_SIZE_VALID)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93fd8f349ec2c6a1afa91da1d57d1935cbd11328"},"cell_type":"code","source":"predictions = model.predict(X_test)\n    \ntest_df['label']+=predictions[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"f8254249278f5ee69ab93cc9b0ce762485abff64"},"cell_type":"markdown","source":"from sklearn.model_selection import StratifiedKFold\n# Only one fold... \nskf=StratifiedKFold(n_splits=k_folds)\nfor i,(trdex,valdex) in enumerate(skf.split(X=dataset.id.values,y=dataset.label.values)): # trdex valdex contienent la position de l'image dans le dataframe, i est le numero de fold\n    train_sub_dataframe=dataset.iloc[trdex]\n    print(trdex.shape)\n\n    train_generator = train_datagen.flow_from_dataframe(dataframe=train_sub_dataframe,directory=train_path,x_col = 'id',y_col = 'label',has_ext=False,\n                #subset='training',\n                target_size=(96, 96),\n                batch_size=32,\n                class_mode='binary'\n                )\n    STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size\n    \n    val_sub_dataframe=dataset.iloc[valdex]\n    validation_generator = train_datagen.flow_from_dataframe(\n                dataframe=val_sub_dataframe,\n                directory=valid_path,\n                x_col = 'id',\n                y_col = 'label',\n                has_ext=False,\n                #subset='validation', # This is the trick to properly separate train and validation dataset\n                target_size=(96, 96),\n                batch_size=32,\n                shuffle=False,\n                class_mode='binary'\n                )\n    STEP_SIZE_VALID=validation_generator.n//validation_generator.batch_size\n    model.fit_generator(generator=train_generator,\n              steps_per_epoch=STEP_SIZE_TRAIN,\n              nb_epoch=10,\n              shuffle=True,verbose=1,\n              callbacks=[lr_reducer, early_stop],\n              validation_data=validation_generator,\n              validation_steps=STEP_SIZE_VALID)\n    print('prediction for fold n '+str(i))\n    predictions = model.predict(X_test)\n    \n    test_df['label']+=predictions[:,0]/k_folds\n"},{"metadata":{"trusted":true,"_uuid":"a1f6600425a6516ed1e0f19ee24e2d0b21c5aeac"},"cell_type":"code","source":"submission = test_df[[\"id\", \"label\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c9c5fee1ec466857418020b6b9231dcd20836f5"},"cell_type":"code","source":"\nsubmission.to_csv(\"submission_14_03.csv\", index = False, header = True)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}