{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"collapsed":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nimport cv2\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import OneHotEncoder\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense , Conv3D , MaxPool3D , BatchNormalization , Dropout , Concatenate ,Input , Flatten\nfrom tensorflow.keras.models import Model , Sequential\nimport PIL\nimport gdcm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = os.path.join('../input/osic-pulmonary-fibrosis-progression/train/')\nprint('train_dir = ' , train_dir)\ntest_dir = os.path.join('../input/osic-pulmonary-fibrosis-progression/test/')\nprint('test_dir = ' , test_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#training data\ntrain_labels = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\nprint(train_labels.shape)\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_labels = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nprint(test_labels.shape)\ntest_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_patients = train_labels['Patient'].values\ntrain_patients.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = 0\nfor p in train_patients[27:28]:\n        path = train_dir + p\n        slices = [pydicom.dcmread(path+'/'+s) for s in os.listdir(path)]\n        slices.sort(key = lambda x: int(x.InstanceNumber))\n        a = slices[0].pixel_array.shape\n        #print(len(slices) , slices[0].pixel_array.shape)\n        #print(len(slices))\n        print(slices[0])\n#     except RuntimeError as e:\n#         count += 1\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 150 \n\nfor p in train_patients[:1]:\n    path = train_dir + p\n    slices = [pydicom.read_file(path+'/'+s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n    fig = plt.figure(figsize=(8 , 8))\n    for i , scan in enumerate(slices[:12]):\n        p = fig.add_subplot(3, 4 , i+1)\n        img = cv2.resize(np.array(scan.pixel_array) , (IMG_SIZE , IMG_SIZE))\n        p.imshow(img , cmap = 'gray')\n    plt.show()\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_SLICES = 30\n\ndef chunks(lst, n):\n    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n    for i in range(0, len(lst), n):\n        yield lst[i:i + n]\n        \ndef mean(l):\n    return sum(l)/len(l)\n\n\ndef resize_slices(slices):\n    slices = [cv2.resize(np.array(slice.pixel_array) , (IMG_SIZE , IMG_SIZE)) for slice in slices]\n    if len(slices) == NUM_SLICES:\n        return slices\n    else:\n        chunk_size = int(np.ceil(len(slices) / NUM_SLICES))\n        new_slices = []\n        for chunk in chunks(slices , chunk_size):\n            chunk = list(map(mean , zip(*chunk)))\n            new_slices.append(chunk)\n        if len(new_slices) < NUM_SLICES:\n            for i in range(NUM_SLICES - len(new_slices)):\n                new_slices.append(new_slices[-1])\n        elif len(new_slices) > NUM_SLICES:\n            extra = new_slices[NUM_SLICES-1:]\n            last = list(map(mean , zip(*extra)))\n            del new_slices[NUM_SLICES:]\n            new_slices[-1] = last\n        return new_slices\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in train_patients[22:28]:\n    path = train_dir + p\n    slices = [pydicom.read_file(path+'/'+s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.InstanceNumber))\n    new_slices = resize_slices(slices)\n    print(np.shape(new_slices))\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets look at our numerical and categorical attributes","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_smoke_classes = train_labels['SmokingStatus'].unique()\nprint(len(num_smoke_classes))\nprint(num_smoke_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_attribute = train_labels['SmokingStatus'].tolist()\nfor i in range(len(categorical_attribute)):\n    if categorical_attribute[i] == 'Ex-smoker':\n        categorical_attribute[i] = [0,0,1]\n    elif categorical_attribute[i] == 'Never smoked':\n        categorical_attribute[i] = [0,1,0]\n    elif categorical_attribute[i] == 'Currently smokes':\n        categorical_attribute[i] = [1,0,0]\ncategorical_attribute = np.array(categorical_attribute)\nprint(categorical_attribute[:10])\nprint(categorical_attribute.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_features = ['Weeks' , 'Age' , 'Sex' , 'SmokingStatus']\nnumerical_data = train_labels[['Weeks','Age' ,]]\nnumerical_data = np.array(numerical_data)\nnumerical_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_data[:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_data = np.concatenate((numerical_data , categorical_attribute) , axis = 1)\nprint(numerical_data.shape)\nnumeraical_data = np.array(numerical_data)\nnumerical_data[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_labels['Sex'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gender_oh = train_labels['Sex'].tolist()\nfor i in range(len(gender_oh)):\n    if gender_oh[i] == 'Male':\n        gender_oh[i] = [0,1]\n    elif gender_oh[i] == 'Female':\n        gender_oh[i] = [1,0]\ngender_oh = np.array(gender_oh)\nprint(gender_oh[:10])\nprint(gender_oh.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_data = np.concatenate((numerical_data , gender_oh) , axis = 1)\nprint(numerical_data.shape)\nnumeraical_data = np.array(numerical_data)\nnumerical_data[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_data[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('Train_X_numerical.npy' , numerical_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pydicom.config.image_handlers = ['pillow_handler']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ct_data = []\nfor p in train_patients:\n    path = train_dir + p\n    slices = [pydicom.read_file(path+'/'+s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: int(x.InstanceNumber))\n    try:\n        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n    except:\n        slice_thickness = slices[0].SliceThickness\n    for s in slices:\n        s.SliceThickness = slice_thickness\n    new_slices = resize_slices(slices)\n    new_slices = np.array(new_slices).astype(np.int16)\n    ct_data.append(new_slices)\n\nct_data = np.array(ct_data)\nprint(ct_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ct_data = np.array(ct_data)\nnp.save('Train_ct_processed.npy' , ct_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}