{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport cv2\nimport scipy.stats as sp\nimport pydicom\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"class_info = pd.read_csv('../input/stage_2_detailed_class_info.csv')\ntr_labels = pd.read_csv('../input/stage_2_train_labels.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea7c29502337c4a9ba70e11b17ebfac35698b039"},"cell_type":"code","source":"print(len(class_info))\nclass_info.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f26ecb3c8c1bb20eae5bae0f81f3342e91d67903"},"cell_type":"code","source":"# Acccessing rows by index in pandas\nclass_info.loc[2][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"296b6b55516ce946bdef3d9e987d3c2b5aa66116"},"cell_type":"code","source":"class_info['class'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b76db95511a42f115761257629afbc791f6d01b9"},"cell_type":"code","source":"tr_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9b9529dabdc2701b4be92cf4e4b216cf1d653bb"},"cell_type":"code","source":"images_list = os.listdir(\"../input/stage_2_train_images\")\ntest_image = os.listdir(\"../input/stage_2_test_images\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c3698746b9c4e10be7ebe8f44654b62e2e43d07"},"cell_type":"code","source":"print(images_list[0][:-4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6eafc874d479947c1cc1acf4599fe323f123dbc0"},"cell_type":"code","source":"ty=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fa6df88a71e708d206f9df93d03b737542a80ec"},"cell_type":"code","source":"c=class_info[class_info['patientId']==images_list[1][:-4]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9223ade028f60b92680d84b760f4453aeb414cc2"},"cell_type":"code","source":"ty.append(c['class'].unique()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a66f965631f895bdd92cebeb131cda7ea34eb86"},"cell_type":"code","source":"ty","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2a6261c11368195f01e87034c508a92cf26205f"},"cell_type":"code","source":"print(len(images_list))\nprint(len(test_image))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d47e60a813c266788145622c4ebcc7e9e46be9f"},"cell_type":"code","source":"def show(image):\n    plt.figure(figsize=(10,10))\n    plt.imshow(image,cmap='gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c2fde3d80238e96ded759a7ac9a326d95e1a9ed"},"cell_type":"code","source":"path=\"../input/stage_2_train_images/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edb50ec3f8d00409e8539c86b254b35c28d957ec"},"cell_type":"code","source":"dcm_data = pydicom.read_file(path+images_list[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b779206daf487ab69e524f7cf158a44b9fcb1238"},"cell_type":"code","source":"print(dcm_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cda8b8a145d314a64f2d9024f7d200ab87f11d5"},"cell_type":"code","source":"img = dcm_data.pixel_array","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a30a62d4e3f59caf9c6e5e163b782ec785ba6e9"},"cell_type":"code","source":"show(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dc43523e9db0ec7511e4d220701ca4cbc18b61d"},"cell_type":"code","source":"l= list(dcm_data.elements())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8674ab085f34bb67a2798a92bbc9159493c501c"},"cell_type":"code","source":"dcm_data.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27037d02f4e7970c80cbc9389108b1e59f7e1b95"},"cell_type":"code","source":"dcm_data[0x10,0x10].value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e68557d001b5aa71386701f652c97d52168c5e4"},"cell_type":"code","source":"# if we do nat exactly remember the keywords the following way can be conveniently used to first get\n# the keys first and then using that key to access corresponding values.\ndcm_data.dir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed03d2d4bd3bdba066c91304fa2761f65973ecbc"},"cell_type":"code","source":"dcm_data.PatientID","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1345ac328865f5afaf70114c81e98b2127ed76d"},"cell_type":"code","source":"dcm_data.PatientSex","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1766acfa8baac38fb6ee51c4039383647922c4b4"},"cell_type":"code","source":"sample_sub = pd.read_csv(\"../input/stage_2_sample_submission.csv\")\nsample_sub.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"924be64d8675054f721b25ce363f1d8623852440"},"cell_type":"code","source":"l1 = list(class_info['patientId'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45623c37eae440aa1eac95e82c463b0233eeda3d"},"cell_type":"code","source":"not_labeled =[]\n\nfor i in range(0,1000):\n    if test_image[i][:-4] not in l1:\n        not_labeled.append(test_image[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c320e43391b16fd3534ecb72412e3c2f83c7537"},"cell_type":"code","source":"# all image of test_images are not labeled \nlen(not_labeled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34369962eb960381becc2f096ee168b313eab61e"},"cell_type":"code","source":"labeled =[]\n\nfor i in range(0,26684):\n    if images_list[i][:-4] not in l1:\n        labeled.append(images_list[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fad6ea469f7d37f6f47dc4f1845ad41a1adedbd"},"cell_type":"code","source":"len(labeled)\n# so all traing data is lebelled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bfcc463847a5149f016ade80a393bacc071a856"},"cell_type":"code","source":"len(set(l1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c62254ec7adc19f86d3dab7e3084a2cbc97f57a"},"cell_type":"code","source":"s =test_image[10][:-4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00b8cf0100c4b371b51344d99e61635de210b50c"},"cell_type":"code","source":"s in l1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20d3d4f69f7fc3200c66fc9b0a1c674be0716fc7"},"cell_type":"code","source":"s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"908801265aac8ffd5a6a9a176055e2f533476189"},"cell_type":"code","source":"pid = sample_sub['patientId']\nstring = sample_sub['PredictionString']\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68b0ba3c65f83d297a1c0d47ab27a189770bc16c"},"cell_type":"code","source":"my_submission = pd.DataFrame({'patientId': pid, 'PredictionString': string})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a3c452d34080c437e3f77a57c6c9a5662e0aa17"},"cell_type":"code","source":"my_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"526a138a92582ec910cd6b1ecef343b67eba437d"},"cell_type":"code","source":"import csv\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42e98a4aafea4a75a47f48a531830bab748bccae"},"cell_type":"code","source":"# empty dictionary\npneumonia_locations = {}\n# load table\nwith open(os.path.join('../input/stage_1_train_labels.csv'), mode='r') as infile:\n    # open reader\n    reader = csv.reader(infile)\n    # skip header\n    next(reader, None)\n    # loop through rows\n    for rows in reader:\n        # retrieve information\n        filename = rows[0]\n        location = rows[1:5]\n        pneumonia = rows[5]\n        # if row contains pneumonia add label to dictionary\n        # which contains a list of pneumonia locations per filename\n        if pneumonia == '1':\n            # convert string to float to int\n            location = [int(float(i)) for i in location]\n            # save pneumonia location in dictionary\n            if filename in pneumonia_locations:\n                pneumonia_locations[filename].append(location)\n            else:\n                pneumonia_locations[filename] = [location]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d31e4a0cd5170d28686e1be42adfb251256ba33","scrolled":false},"cell_type":"code","source":"img_with_pneumonia={}\nfor index,row in tr_labels.iterrows():\n    filename=row['patientId']\n    pneumonia= row['Target']\n    if pneumonia==1:\n        if filename in img_with_pneumonia:\n            img_with_pneumonia[filename].append([int(row['x']),int(row['y']),int(row['height']),int(row['width'])])\n        else:\n            img_with_pneumonia[filename]=[[int(row['x']),int(row['y']),int(row['height']),int(row['width'])]]\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37358d6b131ad431971fc0f820b860d28697d518"},"cell_type":"code","source":"img2= img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d45e1bdc7008be65ea16ac2cc095ab4301893011"},"cell_type":"code","source":"print(img2.shape)\nimg2 = np.expand_dims(img2,-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8df59691354454c32f7c39b356be92445d0380c"},"cell_type":"code","source":"len(img2[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdcafb8132611e98a9cbdc09cd00447bc8d045fa"},"cell_type":"markdown","source":"**Date 10/27/2018**"},{"metadata":{"trusted":true,"_uuid":"4aab3c3fd23999c390e1736ec7d5b1287548cbd7"},"cell_type":"code","source":"import keras\nfrom keras.models import Sequential,Input,Model\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe8c14931e7a9224f784f57fb0e96e36a07bf949"},"cell_type":"code","source":"train_x=[]\ntrain_y=[]\ndef makeDataset():\n    for i in range(len(images_list)%1500):\n        d=pydicom.read_file(path+images_list[i])\n        c=class_info[class_info['patientId']==images_list[i][:-4]]\n        train_y.append(c['class'].unique()[0])\n        train_x.append(d.pixel_array)\nmakeDataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df14d17941dbae4a134bab66d50aa89673356042"},"cell_type":"code","source":"#function to convert string label to integer value\ndef label(s):\n    if s=='Normal':\n        return 0\n    if s=='No Lung Opacity / Not Normal':\n        return 1\n    if s=='Lung Opacity':\n        return 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"583e527d0c58f45ec7c8742f4998f8ecc6fa5a3d"},"cell_type":"code","source":"train_y = list(map(label,train_y))\ntrain_y= np.stack(train_y)\ntrain_x = np.stack(train_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ccc22f4ac86ff50e4a1fd491a01ab10cbfa52a6"},"cell_type":"code","source":"train_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0750000d3a5bdbb914a08681fa19b33fb3fb0fbe"},"cell_type":"code","source":"# converting train_y to one hot\ntrain_y = to_categorical(train_y)\n# reshaping train_x to standard form\ntrain_x=train_x.reshape(-1,1024,1024,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b381a3a32bd5e4ac045f1b2501c09b4d7a5f07f4"},"cell_type":"code","source":"train_x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f0324bb59e98921b65a69fe8bb646c95e5e9939"},"cell_type":"code","source":"# Normalizing the pixel value\ntrain_x = train_x.astype('float32')\ntrain_x = train_x / 255.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"484684472e7a5f4592d71e7bc09f69d0e16ffc93"},"cell_type":"code","source":"print(train_x.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0eaf0d763ec5176a5c981d05ad37f409e971db4b"},"cell_type":"code","source":"#del train_x\n#del train_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fc3e5b93aa8100e9bcfd06c94ba857213e654f3"},"cell_type":"code","source":"train_y[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d57f332d37393757ccad1480949309ded8f37fe"},"cell_type":"code","source":"#The image have 1024X1024 and only one channel\nimg = pydicom.read_file(path+images_list[1])\nimg=img.pixel_array\nimg.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"446363eda00b07a9922dbacbf52572ec0763fe46"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b16f87498205e1789a7523cf65bef01c9ecf22b4"},"cell_type":"code","source":"batch_size = 16\nepochs = 15\nnum_classes = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a74116ba06e377f8d5571eee5f31f8466dbbe50"},"cell_type":"code","source":"model= Sequential()\n# here input shape is (1024,1024) because the image is of size 1024X1024 and have only one channel\n# here 32 is number of filters of kernel size (3,3)\n# generally number of filters are increased and kernel size decreased but here is is constant\n\nmodel.add(Conv2D(32,kernel_size=(7,7),activation='linear',input_shape=(1024,1024,1),padding='same'))\nmodel.add(LeakyReLU(alpha=0.1))\nmodel.add(MaxPooling2D(pool_size=(7,7),padding='same'))\nmodel.add(Conv2D(64,kernel_size=(7,7),activation='linear',padding='same'))\nmodel.add(LeakyReLU(alpha=0.1))\nmodel.add(MaxPooling2D(pool_size=(7,7),padding='same'))\nmodel.add(Conv2D(128,kernel_size=(7,7),activation='linear',padding='same'))\nmodel.add(LeakyReLU(alpha=0.1))\nmodel.add(MaxPooling2D(pool_size=(7,7),padding='same'))\nmodel.add(Flatten())\nmodel.add(Dense(128,activation='linear'))\nmodel.add(LeakyReLU(alpha=0.1))\nmodel.add(Dense(num_classes,activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d625a9ddcbbd031dc9c0fff1473fdda6dd4d5717"},"cell_type":"code","source":"model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(),metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad7d926beac388cfeb88af6aca769b8796e99302"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beca2ec13391f2a9b607ddad9c1a9e59b6f087c2"},"cell_type":"code","source":"model_train = model.fit(train_x,train_y,batch_size=batch_size,epochs=epochs,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f0d13fbcc6e6e500550111e5fe8703458bc9ce2"},"cell_type":"code","source":"# Model Training Results\naccuracy = model_train.history['acc']\nloss = model_train.history['loss']\nepochs = range(len(accuracy))\nplt.plot(epochs, accuracy, 'r', label='Training accuracy')\nplt.title('Training accuracy')\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, 'g', label='Training loss')\nplt.title('Training Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc7bf20fd77d0d256cff249d422196e6a1406198"},"cell_type":"markdown","source":"**Now checking Model performance **"},{"metadata":{"trusted":true,"_uuid":"e219dd6e79e6aa4798ed74e93d321bf274ce610a"},"cell_type":"code","source":"# check performance on train_data\ntest_eval = model.evaluate(train_x, train_y, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abe8a83cf600b97a20900a31a27ed43813ffc23f"},"cell_type":"code","source":"print('Test loss:', test_eval[0])\nprint('Test accuracy:', test_eval[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26cbdb7c1a67a1b0e809f3d58c8cc4117ea79f3c"},"cell_type":"code","source":"# check individual\nprint(\"Predicted label of the image\",np.argmax(np.round(model.predict(np.expand_dims(train_x[3],axis=0)))))\nprint(\"Actual label of the image \",np.argmax(train_y[3]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6508d13f9977fc507e022e1460c271dfea44c00a"},"cell_type":"code","source":"\ndef check(i):\n    dcm_img = pydicom.read_file(path+class_info.loc[i][0]+\".dcm\")\n    img=dcm_img.pixel_array\n    img2=img.reshape(1024,1024,1)\n    plt.figure(figsize=(10,10))\n    plt.imshow(img,cmap='gray')\n    p=np.argmax(np.round(model.predict(np.expand_dims(img2,axis=0))))\n    plt.title(\"Predicted {} ,Actual {}\".format(p,label(class_info.loc[i][1])))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bca660e36f00e896441acba7956d829c250f70a7"},"cell_type":"code","source":"check(25000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0e901575dfa1e0dc3db5d5abeddcc78fcf97290"},"cell_type":"markdown","source":"**Date 29/10/2018**"},{"metadata":{"trusted":true,"_uuid":"600bd586a03dd80842d746ed6681ca68cef3f4c4"},"cell_type":"code","source":"batch_sz=20\nfor e in range(2):\n    batch=0\n    print(\"epochs : \",e)\n    for image in range(int(100/batch_sz)):\n        print(\"batch : \",batch)\n        for i in range(batch_sz):\n            print(batch+i)\n        batch+=batch_sz\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d710ddf9178fed88c8d7eb5b8500f45dd26ef4d"},"cell_type":"code","source":"# Training on ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}