{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os, random, sys, cv2, matplotlib, csv, keras\nfrom subprocess import check_output\nfrom datetime import datetime\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\nfrom keras.optimizers import SGD, Adam\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\nfrom keras.preprocessing import image","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"NUM_CLASSES = 5\n\n# we need images of same size so we convert them into the size\nWIDTH = 128\nHEIGHT = 128\nDEPTH = 3\ninputShape = (HEIGHT, WIDTH, DEPTH)\n\n# initialize number of epochs to train for, initial learning rate and batch size\nEPOCHS = 15\nINIT_LR = 1e-3\nBS = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49074dd69c14b4533243ccaa7db8f630856664d0","_kg_hide-output":false,"scrolled":false},"cell_type":"code","source":"#Storing images in image-data pair\n\nprint(\"Loading images at...\"+ str(datetime.now()))\nsys.stdout.flush()\n\nImageNameDataHash = {}            #For storing image information in image-data pair\nimages = os.listdir(\"/kaggle/working/../input/\")\n\nfor imageFileName in images:\n    if(imageFileName == \"trainLabels.csv\"):\n        continue\n    img = load_img(os.path.join(os.path.sep, \"/kaggle/working/../input/\", imageFileName))\n    arr = img_to_array(img)\n    dim1 = arr.shape[0]\n    dim2 = arr.shape[1]\n    dim3 = arr.shape[2]\n    if (dim1 < HEIGHT or dim2 < WIDTH or dim3 < DEPTH):\n        print(\"Error image dimensions are less than expected \"+str(arr.shape))\n    \n    arr = cv2.resize(arr, (HEIGHT,WIDTH))       \n    \n    dim1 = arr.shape[0]\n    dim2 = arr.shape[1]\n    dim3 = arr.shape[2]\n    \n    if (dim1 != HEIGHT or dim2 != WIDTH or dim3 != DEPTH):\n        print(\"Error after resize, image dimensions are not equal to expected \"+str(arr.shape))\n    \n    arr = np.array(arr, dtype=\"float\") / 255.0\n    imageFileName = imageFileName.replace('.jpeg','')\n    \n    ImageNameDataHash[str(imageFileName)] = np.array(arr)       #Storing Image-data pair\n        \nprint(\"Loaded \" + str(len(ImageNameDataHash)) + \" images at...\"+ str(datetime.now())) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dce731a7ad43adc4091339f165a96e5bb4a59e0"},"cell_type":"code","source":"random.seed(10)\nprint(\"Reading trainLabels.csv...\")\ndf = pd.read_csv('/kaggle/working/../input/trainLabels.csv', sep=',')\nprint(type(df))\n\nrow_count = df.shape[0]\ncol_count = df.shape[1]\nprint(\"row_count=\"+str(row_count)+\" col count=\"+str(col_count))\n\ndf[\"PatientID\"] = ''\nheader_list = list(df.columns)\nprint(header_list)\n\nImageLevelHash = {}     # Storing Image-level pair\npatientIDList = []\nuniquePatientIDList = []\n\nfor index, row in df.iterrows():\n    key = row[0] + ''\n    patientID = row[0] + ''\n    patientID = patientID.replace('_right', '')\n    patientID = patientID.replace('_left', '')\n    df.at[index, 'PatientID'] = patientID\n    patientIDList.append(patientID)\n    ImageLevelHash[key] = str(row[1])\n    \nuniquePatientIDList = sorted(set(patientIDList))\ncount = 0\n\nfor patientID in uniquePatientIDList:\n    left_level = ImageLevelHash[str(patientID + '_left')]\n    right_level = ImageLevelHash[str(patientID + '_right')]\n    \n    if(left_level != right_level):\n        count = count + 1\n\nprint(\"count of images with both left and right eye level not matching=\"+str(count))\nprint(\"number of unique patients=\"+str(len(uniquePatientIDList)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aa1a36377436490060579f1b5cdf9f4916ed760"},"cell_type":"code","source":"#Merging two dataframes to give columns/features as follows: ['image', 'data', 'level', 'PatientID']\n\nimageNameArr = []\ndataArr = []\n\nkeepImages =  list(ImageNameDataHash.keys())\ndf = df[df['image'].isin(keepImages)]\nfor index, row in df.iterrows():\n    key = str(row[0])\n    if key in ImageNameDataHash:\n        imageNameArr.append(key)\n        dataArr.append(np.array(ImageNameDataHash[key]))\n        \ndf2 = pd.DataFrame({'image': imageNameArr, 'data': dataArr})\ndf2_header_list = list(df2.columns)\n        \ndf = pd.merge(df2, df, left_on='image', right_on='image', how='outer')\ndf_header_list = list(df.columns) \nprint(df_header_list) # 'image', 'data', level', 'PatientID'\nprint(len(df)) # 1000\nprint(df.sample())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc6a37972c9b09ecdfa03a4573715633467b86df"},"cell_type":"code","source":"#Plotting a sample data\n\nsample = df.loc[df.index[0], 'data']\nprint(\"Sample Image\")\nprint(type(sample)) # <class 'numpy.ndarray'>\nprint(sample.shape) # 128,128,3\nfrom matplotlib import pyplot as plt\nplt.imshow(sample, interpolation='nearest')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f879a7908365f3f364285bdd8985b7fe455676aa"},"cell_type":"code","source":"#Partitioning of data in 60:20:20 ratio for training, evaluation, testing respectively\n\nX = df['data']\nY = to_categorical(np.array(df['level']), num_classes=NUM_CLASSES)\nprint(\"Partition of image into 60:20:20\")\nsys.stdout.flush()\nunique_ids = df.PatientID.unique()\nprint('unique_ids shape='+ str(len(unique_ids))) #500\n\ntrain_ids, not_train_ids = train_test_split(unique_ids, test_size = 0.40, random_state = 10)\nvalid_ids, test_ids = train_test_split(not_train_ids, test_size = 0.50, random_state = 10)\n\ntrainid_list = train_ids.tolist()\nvalidid_list = valid_ids.tolist()\ntestid_list = test_ids.tolist()\n\ntraindf = df[df.PatientID.isin(trainid_list)]\nvalSet = df[df.PatientID.isin(validid_list)]\ntestSet = df[df.PatientID.isin(testid_list)]\n\ntraindf = traindf.reset_index(drop=True)\nvalSet = valSet.reset_index(drop=True)\ntestSet = testSet.reset_index(drop=True)\nprint(traindf.head())\nprint(valSet.head())\nprint(testSet.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c986f647b09b9388cb7e04ab01660577807633a"},"cell_type":"code","source":"#Saving data and level of each image in separate lists, each for training, evaluating and testing\n\ntrainX = traindf['data']\ntrainY = traindf['level']\n\nvalX = valSet['data']\nvalY = valSet['level']\n\ntestX = testSet['data']\ntestY = testSet['level']\n\nprint('trainX shape=', trainX.shape[0], 'valX shape=', valX.shape[0], 'testX shape=', testX.shape[0]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9c47a97a6d5069c4d252042c48c238d69bcc580"},"cell_type":"code","source":"trainY =  to_categorical(trainY, num_classes=NUM_CLASSES)\nvalY =  to_categorical(valY, num_classes=NUM_CLASSES)\ntestY =  to_categorical(testY, num_classes=NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"009f6f8546fd0d351d47a0abd054eb6dc3cc1b89"},"cell_type":"code","source":"#Reshaping Image Data\n\nfrom numpy import zeros\n\nXtrain = np.zeros([trainX.shape[0],HEIGHT, WIDTH, DEPTH])\nfor i in range(trainX.shape[0]): # 0 to (traindf Size - 1)\n    Xtrain[i] = trainX[i]\nXval = np.zeros([valX.shape[0],HEIGHT, WIDTH, DEPTH])\nfor i in range(valX.shape[0]): # 0 to (traindf Size - 1)\n    Xval[i] = valX[i]\nXtest = np.zeros([testX.shape[0],HEIGHT, WIDTH, DEPTH])\nfor i in range(testX.shape[0]): # 0 to (traindf Size - 1)\n    Xtest[i] = testX[i]\n\nprint(Xtrain.shape) # (600,128,128,3)\nprint(Xval.shape) # (200,128,128,3)\nprint(Xtest.shape) # (200,128,128,3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f1e4efadfe5e985327143a2385d6173de314c07"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=inputShape))\nmodel.add(Conv2D(32, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"775d8d0b6f561e7fc5587bbf0efbef9449fd544d"},"cell_type":"code","source":"model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b8c90d1e776679ba2ab5425236b6b400a464ecc"},"cell_type":"code","source":"model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64a507c2f6d8dbf1d6c55dde6baaf79dd30525ab"},"cell_type":"code","source":"model.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(output_dim=NUM_CLASSES, activation='softmax')) \nopt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cecbde20080e812eeeff2c6262e7995b5614de1d"},"cell_type":"code","source":"print(\"compiling model..\")\nsys.stdout.flush()\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=opt, metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80d6284200bbc247f76e331aa6f2a6ed90af5ef6"},"cell_type":"code","source":"#summary\nfrom keras.utils import print_summary\nprint_summary(model, line_length=None, positions=None, print_fn=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44ebe6d43752bfa48af0eede58d65499da316d82"},"cell_type":"code","source":"#construct the image generator for data augmentation\nprint(\"Generating images...\")\nsys.stdout.flush()\naug = ImageDataGenerator(rotation_range=30, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode=\"nearest\")\n\nprint(\"training network...\")\nsys.stdout.flush()\n\nH = model.fit_generator(aug.flow(Xtrain, trainY, batch_size=BS), validation_data=(Xval, valY), steps_per_epoch=len(trainX) // BS, epochs=EPOCHS, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6823c8dc01df8dc5fadee60b6a5dc29786243a50","scrolled":true},"cell_type":"code","source":"#Prediction\n\npredict = model.predict(Xtest, batch_size=BS, verbose = 1, steps = None)\nprint(predict)\nevaluate = model.evaluate(Xtest, testY, verbose = 1, steps = None)\nprint(evaluate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"837b3e2f8e4d6871f04c0b9e00a4b708a1810bcd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a8e2924fcb7766b53ab71435c45671b2b6c699a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5500f6b64267580f5cc26be91835f511a075eb5b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f2063de29763bd352e978cd6e4cd370735b9db1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85bf9813c44a52a86d8ef68815c5c0ec3cce84ae"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}