{"cells":[{"metadata":{},"cell_type":"markdown","source":"# OSIC Pulmonary Fibrosis Progression Kaggle Competition Analysis\n#### I was working on google colab so the first couple of lines are about how to get the data using Kaggle API to colab"},{"metadata":{"id":"Lj0OVQRsG0K8","outputId":"f45859e4-b9a2-4695-b94e-02219264a74a","trusted":true},"cell_type":"code","source":"# # import the kaggle.json file to download the dataset from the API\n# from google.colab import files\n# files.upload()","execution_count":null,"outputs":[]},{"metadata":{"id":"zYwkjHGOG7Sd","outputId":"62bf4164-2e5a-4497-a7f8-38a366b7fa1a","trusted":true},"cell_type":"code","source":"#######to mount data from Google drive########\n# from google.colab import drive\n# drive.mount('/content/drive')","execution_count":null,"outputs":[]},{"metadata":{"id":"4W_pmOwEIEi6","outputId":"c5a7f6c3-332d-4a70-b812-97bcfba93cf5","trusted":true},"cell_type":"code","source":"# pip install --upgrade pip","execution_count":null,"outputs":[]},{"metadata":{"id":"1OFrWttaIEpG","outputId":"ff82267a-3b05-4642-af04-cb37a0bfd0ff","trusted":true},"cell_type":"code","source":"# to easy download your dataset without errors use kaggle==1.5.6 version\n# pip install kaggle==1.5.6","execution_count":null,"outputs":[]},{"metadata":{"id":"Cvg6mIhNH4vs","trusted":true},"cell_type":"code","source":"#  ! mkdir ~/.kaggle","execution_count":null,"outputs":[]},{"metadata":{"id":"kLJp38qMHWjg","trusted":true},"cell_type":"code","source":"# ! cp kaggle.json ~/.kaggle/","execution_count":null,"outputs":[]},{"metadata":{"id":"6972cDT3HWmc","trusted":true},"cell_type":"code","source":"#  ! chmod 600 ~/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"id":"qv07Z3-QHWzZ","outputId":"fc911f92-3e75-450e-f941-f55c53c75259","trusted":true},"cell_type":"code","source":"# ! kaggle datasets list","execution_count":null,"outputs":[]},{"metadata":{"id":"YPX_Y0wsH9ob","outputId":"682a489a-05aa-4140-daa0-f7e14891c8e3","trusted":true},"cell_type":"code","source":"# ! kaggle competitions download -c 'osic-pulmonary-fibrosis-progression'","execution_count":null,"outputs":[]},{"metadata":{"id":"I7vbySRYIUuy","trusted":true},"cell_type":"code","source":"# ! mkdir train","execution_count":null,"outputs":[]},{"metadata":{"id":"ofSvnxx9V7ui","outputId":"23e69974-785f-4048-8ab6-affd28c88741","trusted":true},"cell_type":"code","source":"# ! unzip /content/osic-pulmonary-fibrosis-progression.zip -d train","execution_count":null,"outputs":[]},{"metadata":{"id":"oN2l2Lsj9vBT","outputId":"82617721-11ef-4462-a02f-e731646c334f","trusted":true},"cell_type":"code","source":"# import pydicom to read dcm images \n# !pip install pydicom ","execution_count":null,"outputs":[]},{"metadata":{"id":"B5GsuNBNWAup","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport numpy as np\nimport os \nimport pydicom","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Section:1 Handling Data:\nAssuming you've downloaded the data , what exactly are we working with here? The data consists of many 2D \"slices,\" which, when combined, produce a 3-dimensional rendering of whatever was scanned."},{"metadata":{"id":"A_umsjQLa0qn","trusted":true},"cell_type":"code","source":"testing = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"id":"6UQC-KSea02g","outputId":"d9f519cf-f632-4434-a4d8-377bdd539f9a","trusted":true},"cell_type":"code","source":"testing","execution_count":null,"outputs":[]},{"metadata":{"id":"YhOmO65Oa06F","outputId":"0ea1d355-4755-45e8-e194-853184a8be8c","trusted":true},"cell_type":"code","source":"testing.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"id":"aIe5cNXwbX8A","trusted":true},"cell_type":"code","source":"data_dir = '../input/osic-pulmonary-fibrosis-progression/train/'\npatients = os.listdir(data_dir)\nlabels_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv',index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"id":"LbE9y7sCF4-3","outputId":"758a20a1-83e8-4e4d-f465-8ebcf5fd920d","trusted":true},"cell_type":"code","source":"labels_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"Pym2zNGA95PG","outputId":"ab4e6f4b-2fb1-4ec4-c033-6be078439634","trusted":true},"cell_type":"code","source":"testing.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"id":"MXEP3vCykiHI","outputId":"c3dd53b1-9bcc-4a67-b0eb-02927e5543b2","trusted":true},"cell_type":"code","source":"for patient in patients[:2]:\n    label = labels_df._get_value(patient, 'FVC')\n    path = data_dir + patient\n    \n    # a couple great 1-liners from: https://www.kaggle.com/gzuidhof/data-science-bowl-2017/full-preprocessing-tutorial\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    print(len(slices),label)\n    print(slices[0])","execution_count":null,"outputs":[]},{"metadata":{"id":"KCuzDYVnLN3j","outputId":"4793d913-aecb-4ab7-9b77-8af09a6aec78","trusted":true},"cell_type":"code","source":"plt.bar(labels_df['Sex'],labels_df['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"id":"bsBMPJErOnl7","outputId":"7503a3a9-22f1-47bf-fc92-efd2abf04997","trusted":true},"cell_type":"code","source":"plt.bar(labels_df['SmokingStatus'],labels_df['FVC'],color='red')","execution_count":null,"outputs":[]},{"metadata":{"id":"uyPZxznDOd8l","outputId":"e56e0bcb-492a-4150-e965-b9238eebef68","trusted":true},"cell_type":"code","source":"plt.hist(labels_df['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"id":"eYW7-_T1d2xV","outputId":"765034bf-3710-4bcb-cace-80291d6a5fca","trusted":true},"cell_type":"code","source":"plt.plot(labels_df['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"id":"hmb6cGobkiYw","outputId":"0c3f546f-a188-4478-abdd-0b05536ec7bf","trusted":true},"cell_type":"code","source":"for patient in patients[:10]:\n    label = labels_df._get_value(patient, 'FVC')\n    path = data_dir + patient\n    \n    # a couple great 1-liners from: https://www.kaggle.com/gzuidhof/data-science-bowl-2017/full-preprocessing-tutorial\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    try:\n      print(slices[0].pixel_array.shape, len(slices))\n    except:\n      print('')","execution_count":null,"outputs":[]},{"metadata":{"id":"6vr5RHXlkihS","outputId":"12179095-3908-42e7-e97b-1ff73e4d27e8","trusted":true},"cell_type":"code","source":"len(patients)","execution_count":null,"outputs":[]},{"metadata":{"id":"PgGl9HRukieL","outputId":"1636185b-b7c0-48ff-82c0-81f2e3fdee77","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfor patient in patients[:5]:\n    label = labels_df._get_value(patient, 'FVC')\n    path = data_dir + patient\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    \n    #          the first slice\n    try:\n      plt.imshow(slices[0].pixel_array)\n      plt.show()\n    except:\n      print('None')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Section 2: Processing and viewing our Data\nOkay, so we know what we've got, and what we need to do with it.\nWe have a few options at this point, we could take the code that we have already and do the processing \"online.\" By this, I mean, while training the network, we can actually just loop over our patients, resize the data, then feed it through our neural network. We actually don't have to have all of the data prepared before we go through the network."},{"metadata":{"id":"SDCJrOK5kiVo","outputId":"26c07578-db27-48f9-b6f1-b0f07dbac46e","trusted":true},"cell_type":"code","source":"import cv2\nimport numpy as np\n\nIMG_PX_SIZE = 50\n\nfor patient in patients[:1]:\n    label = labels_df._get_value(patient, 'FVC')\n    path = data_dir + patient\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    fig = plt.figure()\n    for num,each_slice in enumerate(slices[:12]):\n        y = fig.add_subplot(3,4,num+1)\n        new_img = cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE))\n        y.imshow(new_img)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"3PJHKp_PkiTs","outputId":"f53aa24a-2c1c-4e99-c474-f1a15d68a319","trusted":true},"cell_type":"code","source":"import math\ndef chunks(l, n):\n    # Credit: Ned Batchelder\n    # Link: http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks\n    \"\"\"Yield successive n-sized chunks from l.\"\"\"\n    for i in range(0, len(l), n):\n        yield l[i:i + n]\n\ndef mean(l):\n    return sum(l) / len(l)\n\nIMG_PX_SIZE = 50\nhm_slices = 20\n\ndata_dir = '../input/osic-pulmonary-fibrosis-progression/train/'\npatients = os.listdir(data_dir)\nlabels_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv',index_col=0)\n\nfor patient in patients[:10]:\n    try:\n        label = labels_df._get_value(patient, 'FVC')\n        path = data_dir + patient\n        slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n        slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n        new_slices = []\n        slices = [cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE)) for each_slice in slices]\n        chunk_sizes = math.ceil(len(slices) / hm_slices)\n        for slice_chunk in chunks(slices, chunk_sizes):\n            slice_chunk = list(map(mean, zip(*slice_chunk)))\n            new_slices.append(slice_chunk)\n\n        print(len(slices), len(new_slices))\n    except:\n        # some patients don't have labels, so we'll just pass on this for now\n        pass","execution_count":null,"outputs":[]},{"metadata":{"id":"bD1I2W7kP2Up","outputId":"b11236cd-f7ab-419d-ef83-20be72e9d4a1","trusted":true},"cell_type":"code","source":"len(patients)","execution_count":null,"outputs":[]},{"metadata":{"id":"6hVVJ7dtGe69","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"id":"M6Sxx6zUkiFZ","outputId":"fdfcb2f5-583b-481a-bad1-658605e6cb72","trusted":true},"cell_type":"code","source":"for patient in patients[:10]:\n    try:\n        label = labels_df._get_value(patient, 'FVC')\n        path = data_dir + patient\n        slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n        slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n        new_slices = []\n\n        slices = [cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE)) for each_slice in slices]\n\n        chunk_sizes = math.ceil(len(slices) / hm_slices)\n\n\n        for slice_chunk in chunks(slices, chunk_sizes):\n            slice_chunk = list(map(mean, zip(*slice_chunk)))\n            new_slices.append(slice_chunk)\n\n        if len(new_slices) == hm_slices-1:\n            new_slices.append(new_slices[-1])\n\n        if len(new_slices) == hm_slices-2:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n\n        if len(new_slices) == hm_slices-3:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n\n        if len(new_slices) == hm_slices-4:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n\n        if len(new_slices) == hm_slices-5:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])            \n            new_slices.append(new_slices[-1]) \n\n        if len(new_slices) == hm_slices-6:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n\n        if len(new_slices) == hm_slices-7:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n\n        if len(new_slices) == hm_slices-8:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n        if len(new_slices) == hm_slices-9:\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1]) \n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])\n            new_slices.append(new_slices[-1])                     \n            new_slices.append(new_slices[-1])\n\n\n        if len(new_slices) == hm_slices+2:\n            new_val = list(map(mean, zip(*[new_slices[hm_slices-1],new_slices[hm_slices],])))\n            del new_slices[hm_slices]\n            new_slices[hm_slices-1] = new_val\n\n        if len(new_slices) == hm_slices+1:\n            new_val = list(map(mean, zip(*[new_slices[hm_slices-1],new_slices[hm_slices],])))\n            del new_slices[hm_slices]\n            new_slices[hm_slices-1] = new_val\n\n        print(len(slices), len(new_slices))\n    except Exception as e:\n        # again, some patients are not labeled, but JIC we still want the error if something\n        # else is wrong with our code\n        print(str(e))","execution_count":null,"outputs":[]},{"metadata":{"id":"C5v7Aw46Mjir","outputId":"14be784a-9b21-45a8-ff1e-b766f74e919d","trusted":true},"cell_type":"code","source":"for patient in patients[:15]:\n    label = labels_df._get_value(patient, 'FVC')\n    path = data_dir + patient\n    \n    # a couple great 1-liners from: https://www.kaggle.com/gzuidhof/data-science-bowl-2017/full-preprocessing-tutorial\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    print(len(slices),label)\n    # print(slices[0])","execution_count":null,"outputs":[]},{"metadata":{"id":"HvoZyHlvkiDp","outputId":"fdf5dd9c-3dd7-4983-8839-28100185d3f4","trusted":true},"cell_type":"code","source":"for patient in patients[:3]:\n    label = labels_df._get_value(patient, 'FVC')\n    path = data_dir + patient\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    new_slices = []\n\n    slices = [cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE)) for each_slice in slices]\n    \n    chunk_sizes = math.ceil(len(slices) / hm_slices)\n    for slice_chunk in chunks(slices, chunk_sizes):\n      slice_chunk = list(map(mean, zip(*slice_chunk)))\n      new_slices.append(slice_chunk)\n\n    if len(new_slices) == hm_slices-1:\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-2:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-3:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-4:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-5:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])            \n      new_slices.append(new_slices[-1]) \n    if len(new_slices) == hm_slices-6:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-7:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-8:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices-9:\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1]) \n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])\n      new_slices.append(new_slices[-1])                     \n      new_slices.append(new_slices[-1])\n    if len(new_slices) == hm_slices+2:\n        new_val = list(map(mean, zip(*[new_slices[hm_slices-1],new_slices[hm_slices],])))\n        del new_slices[hm_slices]\n        new_slices[hm_slices-1] = new_val\n        \n    if len(new_slices) == hm_slices+1:\n        new_val = list(map(mean, zip(*[new_slices[hm_slices-1],new_slices[hm_slices],])))\n        del new_slices[hm_slices]\n        new_slices[hm_slices-1] = new_val\n    \n    fig = plt.figure()\n    for num,each_slice in enumerate(new_slices):\n        y = fig.add_subplot(4,5,num+1)\n        y.imshow(each_slice, cmap='gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Section 3: Preprocessing our Data"},{"metadata":{"id":"u694r_ci8Z3U","outputId":"3202d49b-9c51-48a5-ed59-6e00070121cc","trusted":true},"cell_type":"code","source":"IMG_SIZE_PX=20\nSLICE_COUNT=10\n\n\ndef chunks(l, n):\n  for i in range(0, len(l), n):\n    yield l[i:i + n]\n\n\ndef mean(l):\n  return sum(l) / len(l)\n\ndef process_data(patient,labels_df,img_px_size=10, hm_slices=10, visualize=False):\n  label = labels_df._get_value(patient, 'FVC')\n  path = data_dir + patient\n  slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n  try:\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n  except:\n    print('')\n  new_slices = []\n  try:\n    slices = [cv2.resize(np.array(each_slice.pixel_array),(img_px_size,img_px_size)) for each_slice in slices]\n  except:\n    return [0,0]\n  chunk_sizes = math.ceil(len(slices) / hm_slices)\n  for slice_chunk in chunks(slices, chunk_sizes):\n    slice_chunk = list(map(mean, zip(*slice_chunk)))\n    new_slices.append(slice_chunk)\n\n  if len(new_slices) == hm_slices-1:\n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices-2:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices-3:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices-4:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices-5:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])            \n    new_slices.append(new_slices[-1]) \n\n  if len(new_slices) == hm_slices-6:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices-7:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices-8:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n  if len(new_slices) == hm_slices-9:\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1]) \n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])\n    new_slices.append(new_slices[-1])                     \n    new_slices.append(new_slices[-1])\n\n  if len(new_slices) == hm_slices+2:\n    new_val = list(map(mean, zip(*[new_slices[hm_slices-1],new_slices[hm_slices],])))\n    del new_slices[hm_slices]\n    new_slices[hm_slices-1] = new_val\n        \n  if len(new_slices) == hm_slices+1:\n    new_val = list(map(mean, zip(*[new_slices[hm_slices-1],new_slices[hm_slices],])))\n    del new_slices[hm_slices]\n    new_slices[hm_slices-1] = new_val\n\n  if visualize:\n    fig = plt.figure()\n    for num,each_slice in enumerate(new_slices):\n      y = fig.add_subplot(4,5,num+1)\n      y.imshow(each_slice, cmap='gray')\n      plt.show()\n\n  if label.all == 1: label=np.array([0,1])\n  elif label.all == 0: label=np.array([1,0])\n        \n  return np.array(new_slices),label\n\n#                                               stage 1 for real.\ndata_dir = '../input/osic-pulmonary-fibrosis-progression/train/'\npatients = os.listdir(data_dir)\nlabels_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv',index_col=0)\n\n\nmuch_data = []\nfor num,patient in enumerate(patients):\n    if num % 100 == 0:\n        print(num)\n    try:\n        img_data,label = process_data(patient,labels_df,img_px_size=IMG_SIZE_PX, hm_slices=SLICE_COUNT)\n        #print(img_data.shape,label)\n        much_data.append([img_data,label])\n    except KeyError as e:\n        print('This is unlabeled data!')\n\nnp.save('much_data-{}-{}-{}.npy'.format(IMG_SIZE_PX,IMG_SIZE_PX,SLICE_COUNT), much_data)","execution_count":null,"outputs":[]},{"metadata":{"id":"oCTG-oLNkh_b","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.compat.v1 as tf\ntf.disable_v2_behavior() \nimport numpy as np\n\nIMG_PXL_SIZE = 10\nSLICE_COUNT = 10\n\nn_classes = 2\nbatch_size = 10\n\nx = tf.placeholder('float')\ny = tf.placeholder('float')\n\nkeep_rate = 0.8","execution_count":null,"outputs":[]},{"metadata":{"id":"t22vREq9kh7S","trusted":true},"cell_type":"code","source":"def conv3d(x, W):\n  return tf.nn.conv3d(x, W, strides=[1,1,1,1,1], padding='SAME')\n\ndef maxpool3d(x):\n  return tf.nn.max_pool3d(x, ksize=[1,2,2,2,1], strides=[1,2,2,2,1], padding='SAME')","execution_count":null,"outputs":[]},{"metadata":{"id":"0xL5juoDkh5O","trusted":true},"cell_type":"code","source":"def convolutional_neural_network(x):\n    #                # 5 x 5 x 5 patches, 1 channel, 32 features to compute.\n    weights = {'W_conv1':tf.Variable(tf.random_normal([3,3,3,1,32])),\n               #       5 x 5 x 5 patches, 32 channels, 64 features to compute.\n               'W_conv2':tf.Variable(tf.random_normal([3,3,3,32,64])),\n               #                                  64 features\n               'W_fc':tf.Variable(tf.random_normal([13824  ,1024])),\n               'out':tf.Variable(tf.random_normal([1024, n_classes]))}\n\n    biases = {'b_conv1':tf.Variable(tf.random_normal([32])),\n               'b_conv2':tf.Variable(tf.random_normal([64])),\n               'b_fc':tf.Variable(tf.random_normal([1024])),\n               'out':tf.Variable(tf.random_normal([n_classes]))}\n\n    #                            image X      image Y        image Z\n    x = tf.reshape(x, shape=[-1, 10, 10, 10, 1])\n\n    conv1 = tf.nn.relu(conv3d(x, weights['W_conv1']) + biases['b_conv1'])\n    conv1 = maxpool3d(conv1)\n\n\n    conv2 = tf.nn.relu(conv3d(conv1, weights['W_conv2']) + biases['b_conv2'])\n    conv2 = maxpool3d(conv2)\n\n    fc = tf.reshape(conv2,[-1, 13824])\n    fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])\n    fc = tf.nn.dropout(fc, keep_rate)\n\n    output = tf.matmul(fc, weights['out'])+biases['out']\n\n    return output","execution_count":null,"outputs":[]},{"metadata":{"id":"603bD41uwbBA","outputId":"193039dc-a367-4b42-8b1e-5d66a4033c85","trusted":true},"cell_type":"code","source":"much_data = np.load('./much_data-20-20-10.npy',allow_pickle=True)\n# len(much_data)\n\nmuch_data[:-2]","execution_count":null,"outputs":[]},{"metadata":{"id":"GcPMuGHnkh2j","trusted":true},"cell_type":"code","source":"much_data = np.load('./much_data-20-20-10.npy',allow_pickle=True)\n# # If you are working with the basic sample data, use maybe 2 instead of 100 here... you don't have enough data to really do this\ntrain_data = much_data[:-2]\nvalidation_data = much_data[-2:]\n\ndef train_neural_network(x):\n  prediction = convolutional_neural_network(x)\n  cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels = y,logits = prediction) )\n  optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)\n  hm_epochs = 5\n  with tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n    successful_runs = 0\n    total_runs = 0\n        \n    for epoch in range(hm_epochs):\n      epoch_loss = 0\n      for data in train_data:\n        total_runs += 1\n        try:\n          X = data[0]\n          Y = data[1]\n          _, c = sess.run([optimizer, cost], feed_dict={x: X, y: Y})\n          epoch_loss += c\n          successful_runs += 1\n        except Exception as e:\n          pass\n          # print(str(e))\n            \n      print('Epoch', epoch+1, 'completed out of',hm_epochs,'loss:',epoch_loss)\n\n      correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))\n      accuracy = tf.reduce_mean(tf.cast(correct, 'float'))\n\n      print('Accuracy:',accuracy.eval({x:[i[0] for i in validation_data], y:[i[0] for i in validation_data]}))      \n\n    print('Done. Finishing accuracy:')\n    print('Accuracy:',accuracy.eval({x:[i[0] for i in validation_data], y:[i[0] for i in validation_data]}))    \n    print('fitment percent:',successful_runs/total_runs)\n\n# Run this locally:\n# train_neural_network(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}