{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os, random, sys, cv2, matplotlib, csv, keras\nfrom subprocess import check_output\nfrom datetime import datetime\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\nfrom keras.optimizers import SGD, Adam\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\nfrom keras.models import load_model\n\nfrom keras.preprocessing import image","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"NUM_CLASSES = 5\n\n# we need images of same size so we convert them into the size\nWIDTH = 128\nHEIGHT = 128\nDEPTH = 3\ninputShape = (HEIGHT, WIDTH, DEPTH)\n\n# initialize number of epochs to train for, initial learning rate and batch size\nEPOCHS = 15\nINIT_LR = 1e-3\nBS = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Loading images at...\"+ str(datetime.now()))\nsys.stdout.flush()\n\nImageNameDataHash = {}\nimages = os.listdir(\"/kaggle/working/../input/\")\n\nfor imageFileName in images:\n    if(imageFileName == \"trainLabels.csv\"):\n        continue\n    img = load_img(os.path.join(os.path.sep, \"/kaggle/working/../input/\", imageFileName))\n    arr = img_to_array(img)\n    dim1 = arr.shape[0]\n    dim2 = arr.shape[1]\n    dim3 = arr.shape[2]\n    if (dim1 < HEIGHT or dim2 < WIDTH or dim3 < DEPTH):\n        print(\"Error image dimensions are less than expected \"+str(arr.shape))\n    \n    arr = cv2.resize(arr, (HEIGHT,WIDTH))\n    \n    dim1 = arr.shape[0]\n    dim2 = arr.shape[1]\n    dim3 = arr.shape[2]\n    \n    if (dim1 != HEIGHT or dim2 != WIDTH or dim3 != DEPTH):\n        print(\"Error after resize, image dimensions are not equal to expected \"+str(arr.shape))\n    \n    arr = np.array(arr, dtype=\"float\") / 255.0\n    imageFileName = imageFileName.replace('.jpeg','')\n    \n    ImageNameDataHash[str(imageFileName)] = np.array(arr) \n        \nprint(\"Loaded \" + str(len(ImageNameDataHash)) + \" images at...\"+ str(datetime.now())) # 1000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(10)\nprint(\"Reading trainLabels.csv...\")\ndf = pd.read_csv('/kaggle/working/../input/trainLabels.csv', sep=',')\nprint(type(df))\n\nrow_count = df.shape[0]\ncol_count = df.shape[1]\nprint(\"row_count=\"+str(row_count)+\" col count=\"+str(col_count))\n\ndf[\"PatientID\"] = ''\nheader_list = list(df.columns)\nprint(header_list)\n\nImageLevelHash = {}\npatientIDList = []\nuniquePatientIDList = []\n\nfor index, row in df.iterrows():\n    key = row[0] + ''\n    patientID = row[0] + ''\n    patientID = patientID.replace('_right', '')\n    patientID = patientID.replace('_left', '')\n    df.at[index, 'PatientID'] = patientID\n    patientIDList.append(patientID)\n    ImageLevelHash[key] = str(row[1])\n    \nuniquePatientIDList = sorted(set(patientIDList))\ncount = 0\n\nfor patientID in uniquePatientIDList:\n    left_level = ImageLevelHash[str(patientID + '_left')]\n    right_level = ImageLevelHash[str(patientID + '_right')]\n    \n    if(left_level != right_level):\n        count = count + 1\n\nprint(\"count of images with both left and right eye level not matching=\"+str(count))\nprint(\"number of unique patients=\"+str(len(uniquePatientIDList)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imageNameArr = []\ndataArr = []\n\nkeepImages =  list(ImageNameDataHash.keys())\ndf = df[df['image'].isin(keepImages)]\nfor index, row in df.iterrows():\n    key = str(row[0])\n    if key in ImageNameDataHash:\n        imageNameArr.append(key)\n        dataArr.append(np.array(ImageNameDataHash[key]))\n        \ndf2 = pd.DataFrame({'image': imageNameArr, 'data': dataArr})\ndf2_header_list = list(df2.columns)\n\nif len(df) != len(df2):\n    print(\"Error length of df != df2\")\nfor idx in range(0, len(df)):\n    if (df.loc[df.index[idx], 'image'] != df2.loc[df2.index[idx], 'image']):\n        print(\"Error \" + df.loc[df.index[idx], 'image'] + \"==\" + df2.loc[df2.index[idx], 'image'])\n        \ndf = pd.merge(df2, df, left_on='image', right_on='image', how='outer')\ndf_header_list = list(df.columns) \nprint(df_header_list) # 'image', 'data', level', 'PatientID'\nprint(len(df)) # 1000\nprint(df.sample())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = df.loc[df.index[0], 'data']\nprint(\"Sample Image\")\nprint(type(sample)) # <class 'numpy.ndarray'>\nprint(sample.shape) # 128,128,3\nfrom matplotlib import pyplot as plt\nplt.imshow(sample, interpolation='nearest')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df['data']\nY = to_categorical(np.array(df['level']), num_classes=NUM_CLASSES)\nprint(\"Partition of image into 60:20:20\")\nsys.stdout.flush()\nunique_ids = df.PatientID.unique()\nprint('unique_ids shape='+ str(len(unique_ids))) #500\n\ntrain_ids, not_train_ids = train_test_split(unique_ids, test_size = 0.40, random_state = 10)\nvalid_ids, test_ids = train_test_split(not_train_ids, test_size = 0.50, random_state = 10)\n\ntrainid_list = train_ids.tolist()\nvalidid_list = valid_ids.tolist()\ntestid_list = test_ids.tolist()\n\ntraindf = df[df.PatientID.isin(trainid_list)]\nvalSet = df[df.PatientID.isin(validid_list)]\ntestSet = df[df.PatientID.isin(testid_list)]\n\ntraindf = traindf.reset_index(drop=True)\nvalSet = valSet.reset_index(drop=True)\ntestSet = testSet.reset_index(drop=True)\nprint(traindf.head())\nprint(valSet.head())\nprint(testSet.head())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX = traindf['data']\ntrainY = traindf['level']\n\nvalX = valSet['data']\nvalY = valSet['level']\n\ntestX = testSet['data']\ntestY = testSet['level']\n\nprint('trainX shape=', trainX.shape[0], 'valX shape=', valX.shape[0], 'testX shape=', testX.shape[0]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainY =  to_categorical(trainY, num_classes=NUM_CLASSES)\nvalY =  to_categorical(valY, num_classes=NUM_CLASSES)\ntestY =  to_categorical(testY, num_classes=NUM_CLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from numpy import zeros\n\nXtrain = np.zeros([trainX.shape[0],HEIGHT, WIDTH, DEPTH])\nfor i in range(trainX.shape[0]): # 0 to traindf Size -1\n    Xtrain[i] = trainX[i]\nXval = np.zeros([valX.shape[0],HEIGHT, WIDTH, DEPTH])\nfor i in range(valX.shape[0]): # 0 to traindf Size -1\n    Xval[i] = valX[i]\nXtest = np.zeros([testX.shape[0],HEIGHT, WIDTH, DEPTH])\nfor i in range(testX.shape[0]): # 0 to traindf Size -1\n    Xtest[i] = testX[i]\n\nprint(Xtrain.shape) # (750,128,128,3)\nprint(Xval.shape) # (250,128,128,3)\nprint(Xtest.shape) # (750,128,128,3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=inputShape))\nmodel.add(Conv2D(32, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.add(Flatten())\nmodel.add(Dense(512, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(output_dim=NUM_CLASSES, activation='softmax')) \nopt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"compiling model...\")\nsys.stdout.flush()\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=opt, metrics=[\"accuracy\"])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import print_summary\nprint_summary(model, line_length=None, positions=None, print_fn=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Generating images...\")\nsys.stdout.flush()\naug = ImageDataGenerator(rotation_range=30, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode=\"nearest\")\n\nprint(\"training network...\")\nsys.stdout.flush()\n\nH = model.fit_generator(aug.flow(Xtrain, trainY, batch_size=BS), validation_data=(Xval, valY), steps_per_epoch=len(trainX) // BS, epochs=EPOCHS, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def letsPredict(predictit):\n    array = [[0 for x in range(5)] for y in range(len(predictit))] \n    for i, value in enumerate(predictit):\n        if max(value[0], value[1], value[2], value[3], value[4]) == value[0]: \n            array[i] = [1., 0., 0., 0., 0.]\n        elif max(value[0], value[1], value[2], value[3], value[4]) == value[1]: \n            array[i] = [0., 1., 0., 0., 0.]\n        elif max(value[0], value[1], value[2], value[3], value[4]) == value[2]: \n            array[i] = [0., 0., 1., 0., 0.]\n        elif max(value[0], value[1], value[2], value[3], value[4]) == value[3]: \n            array[i] = [0., 0., 0., 1., 0.]\n        elif max(value[0], value[1], value[2], value[3], value[4]) == value[4]: \n            array[i] = [0., 0., 0., 0., 1.]\n        else:\n            array[i] = [1., 0., 0., 0., 0.]\n    return array\nprint(Xtest[0]);\npredict = model.predict(Xtest, batch_size=BS, verbose = 1, steps = None)\nprint(predict)\nXtest1 = letsPredict(predict)\nprint(Xtest1)   \nprint(testY)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"evaluate = model.evaluate(Xtest, testY, verbose = 1, steps = None)\nprint(evaluate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set the matplotlib backend so figures can be saved in the background\n# plot the training loss and accuracy\nprint(\"Generating plots...\")\nsys.stdout.flush()\nmatplotlib.use(\"Agg\")\nmatplotlib.pyplot.style.use(\"ggplot\")\nmatplotlib.pyplot.figure()\nN = EPOCHS\nmatplotlib.pyplot.plot(np.arange(0, N), H.history[\"loss\"], label=\"train_loss\")\nmatplotlib.pyplot.plot(np.arange(0, N), H.history[\"val_loss\"], label=\"val_loss\")\nmatplotlib.pyplot.plot(np.arange(0, N), H.history[\"acc\"], label=\"train_acc\")\nmatplotlib.pyplot.plot(np.arange(0, N), H.history[\"val_acc\"], label=\"val_acc\")\nmatplotlib.pyplot.title(\"Training Loss and Accuracy on diabetic retinopathy detection\")\nmatplotlib.pyplot.xlabel(\"Epoch #\")\nmatplotlib.pyplot.ylabel(\"Loss/Accuracy\")\nmatplotlib.pyplot.legend(loc=\"lower left\")\nmatplotlib.pyplot.savefig(\"plot.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(\"model.h5\")\nprint(\"Saved model to disk\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = load_model('model.h5')\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t=178\ntest=Xtest[t].reshape(-1,128,128,3)\narr=model.predict(test)\narr=arr.flatten()\n\nmaxm=arr[0]\nfor i in range(0,5):\n    if(arr[i]>=maxm):\n        predicted_label=i+1\n        maxm=arr[i]\nfor i in range(0,5):\n    if(int(testY[t][i])==1):\n        actual_label=i+1\n        break\nname={1:\"NO level of Diabetic Retinopathy\",2:\"MILD level of Diabetic Retinopathy\",3:\"MODERATE level of Diabetic Retinopathy\",4:\"SEVERE level of Diabetic Retinopathy\",5:\"PROLIFERATIVE level of Diabetic Retinopathy\"\n}\n\nimport matplotlib.pyplot as plt\nplt.imshow(Xtest[t])\nplt.show\n\nprint(\"Predicted  : The patient is having \" + name[predicted_label])\nprint(\"Expected   : The patient should have \"+name[actual_label])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}