{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom glob import glob\nfrom random import shuffle\nimport cv2\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Convolution1D, concatenate, SpatialDropout1D, GlobalMaxPool1D, GlobalAvgPool1D, Embedding, Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten\nfrom keras.layers import Dense, Input, Dropout, MaxPooling2D, Concatenate, GlobalMaxPooling2D, GlobalAveragePooling2D, Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D\nfrom keras.layers.pooling import _GlobalPooling1D\nfrom keras.losses import mae, sparse_categorical_crossentropy, binary_crossentropy\nfrom keras.models import Model\nfrom keras.applications.nasnet import NASNetMobile, NASNetLarge, preprocess_input\nfrom keras.optimizers import Adam, RMSprop\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom imgaug import augmenters as iaa\nimport imgaug as ia #not from keras, from other image aug library\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"folderLoc = \"../input\"\nprint(os.listdir(folderLoc))# listing of the names of the files and folder in folderloc\ntrainData = pd.read_csv(folderLoc+\"/train_labels.csv\")\nidLabel = {col1:col2 for col1,col2 in zip(trainData.id.values, trainData.label.values)}# to map between columns abd labels\ntrainData.head()# to display all the information in the files","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def getId(filePath):# get the location and give all the tif format files\n    return filePath.split(os.path.sep)[-1].replace('.tif', '')# find all tif files and return ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"trainFiles = glob(folderLoc+'/train/*.tif')# find all train TIF files and place in the variable \ntestFiles = glob(folderLoc+'/test/*.tif') # find all test TIF files and place in the variable\nprint(\"labeled_files size :\", len(trainFiles))# lenght of Train/labeled file\nprint(\"test_files size :\", len(testFiles)) # lenght of test file","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# split the train file into 95% train and 5% val\ntrainSet, validationSet = train_test_split(trainFiles, test_size=0.05, random_state=123456)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def getChunk(sequence, seqSize):# agreegate sequences with some size, chunk and return\n    return (sequence[position:position + seqSize] for position in range(0, len(sequence), seqSize))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\ndef getSequence():\n    sometimes = lambda aug: iaa.Sometimes(0.5, aug) # flip negative 0.5% sometimes\n    seq = iaa.Sequential(\n        [\n            # apply the following augmenters to most images\n            iaa.Fliplr(0.5), # horizontally flip 50% of all images\n            iaa.Flipud(0.2), # vertically flip 20% of all images\n            sometimes(iaa.Affine(\n                scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)}, # scale images to 80-120% of their size, individually per axis\n                translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)}, # translate by -20 to +20 percent (per axis)\n                rotate=(-10, 10), # rotate by -45 to +45 degrees\n                shear=(-5, 5), # shear by -16 to +16 degrees\n                order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n            )),\n            # execute 0 to 5 of the following (less important) augmenters per image\n            # don't execute all of them, as that would often be way too strong\n            iaa.SomeOf((0, 5),\n                [\n                    sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n                    iaa.OneOf([\n                        iaa.GaussianBlur((0, 1.0)), # blur images with a sigma between 0 and 3.0\n                        iaa.AverageBlur(k=(3, 5)), # blur image using local means with kernel sizes between 2 and 7\n                        iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7\n                    ]),\n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                    # search either for all edges or for directed edges,\n                    # blend the result with the original image using a blobby mask\n                    iaa.SimplexNoiseAlpha(iaa.OneOf([\n                        iaa.EdgeDetect(alpha=(0.5, 1.0)),\n                        iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n                    ])),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5), # add gaussian noise to images\n                    iaa.OneOf([\n                        iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 10% of the pixels\n                        iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2),\n                    ]),\n                    iaa.Invert(0.01, per_channel=True), # invert color channels\n                    iaa.Add((-2, 2), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n                    iaa.AddToHueAndSaturation((-1, 1)), # change hue and saturation\n                    # either change the brightness of the whole image (sometimes\n                    # per channel) or change the brightness of subareas\n                    iaa.OneOf([\n                        iaa.Multiply((0.9, 1.1), per_channel=0.5),\n                        iaa.FrequencyNoiseAlpha(\n                            exponent=(-1, 0),\n                            first=iaa.Multiply((0.9, 1.1), per_channel=True),\n                            second=iaa.ContrastNormalization((0.9, 1.1))\n                        )\n                    ]),\n                    sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n                    sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around\n                    sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True\n    )\n    return seq","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def dataGeneration(listFiles, idLabel, batchSize, augment=False):\n    seq = getSequence()\n    while True:\n        shuffle(listFiles) # shuffle all files from the folder\n        for batch in getChunk(listFiles, batchSize): # read the files as a chunk based on size and send it to a batch\n            X = [cv2.imread(x) for x in batch] # X is collection of images from that batch\n            Y = [idLabel[getId(x)] for x in batch] # Y is labels getting from the train_label.csv file\n            if augment:\n                X = seq.augment_images(X)\n            X = [preprocess_input(x) for x in X]\n                \n            yield np.array(X), np.array(Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def getModel():\n    inputs = Input((96, 96, 3)) # input size based keras implementation\n    base_model = NASNetMobile(include_top=False, input_shape=(224, 224, 3))#input_shape should be 128, 160, 192, 224\n    x = base_model(inputs) # base model from keras library\n    out1 = GlobalMaxPooling2D()(x)\n    out2 = GlobalAveragePooling2D()(x)\n    out3 = Flatten()(x)\n    out = Concatenate(axis=-1)([out1, out2, out3])\n    out = Dropout(0.5)(out)\n    out = Dense(1, activation=\"sigmoid\", name=\"3_\")(out)\n    model = Model(inputs, out)\n    model.compile(optimizer=Adam(0.0001), loss=binary_crossentropy, metrics=['acc'])\n    model.summary()\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model = getModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"batchSize=32\nh5Path = \"model.h5\"\ncheckpoint = ModelCheckpoint(h5Path, monitor='val_acc', verbose=2, save_best_only=True, mode='max')\n\nhistory = model.fit_generator(\n    dataGeneration(trainSet, idLabel, batchSize, augment=True),\n    validation_data=dataGeneration(validationSet, idLabel, batchSize),\n    epochs=2, verbose=1,\n    callbacks=[checkpoint],\n    steps_per_epoch=len(trainSet) // batchSize,\n    validation_steps=len(validationSet) // batchSize)\nbatchSize=64\nhistory = model.fit_generator(\n    dataGeneration(trainSet, idLabel, batchSize, augment=True),\n    validation_data=dataGeneration(validationSet, idLabel, batchSize),\n    epochs=6, verbose=1,\n    callbacks=[checkpoint],\n    steps_per_epoch=len(trainSet) // batchSize,\n    validation_steps=len(validationSet) // batchSize)\n\nmodel.load_weights(h5Path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"predictions = []\nids = []\nfor batch in getChunk(testFiles, batchSize):\n    X = [preprocess_input(cv2.imread(x)) for x in batch]\n    idsBatch = [getId(x) for x in batch]\n    X = np.array(X)\n    predsBatch = ((model.predict(X).ravel()*model.predict(X[:, ::-1, :, :]).ravel()*model.predict(X[:, ::-1, ::-1, :]).ravel()*model.predict(X[:, :, ::-1, :]).ravel())**0.25).tolist()\n    predictions += predsBatch\n    ids += idsBatch","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Trian and validation loss graph\nval_loss = history.history['val_loss']\nloss = history.history['loss']\nplt.plot(range(len(val_loss)),val_loss,'c',label='Validation loss')\nplt.plot(range(len(loss)),loss,'m',label='Train loss')\n\nplt.title('Training and validation losses')\nplt.legend()\nplt.xlabel('epochs')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataFrame = pd.DataFrame({'id':ids, 'label':predictions})\ndataFrame.to_csv(\"result.csv\", index=False)\ndataFrame.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}