{"cells":[{"metadata":{"trusted":true},"cell_type":"markdown","source":"!pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index\n!pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport pydicom\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom tensorflow.keras.layers import (Dense, Dropout, Activation, Flatten, Input, Add,\n                                    BatchNormalization, LeakyReLU, Concatenate, GlobalAveragePooling2D,Conv2D, AveragePooling2D)\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.utils import Sequence\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.applications as tfa\n#import efficientnet.tfkeras as efn\nfrom sklearn.model_selection import train_test_split, KFold\nimport seaborn as sns\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tab(df):\n    ''' \n    This function gives an array wrt each patient containing\n    feature like age, gender and smoking status\n    '''\n    vector = [(df.Age.values[0]-30)/30]\n    \n    if df.Sex.values[0].lower() == 'male':\n        vector.append(0)\n    else:\n        vector.append(1)\n        \n    if df.SmokingStatus.values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df.SmokingStatus.values[0] == 'Ex-smoker':\n        vector.extend([1,1])\n    elif df.SmokingStatus.values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    else:\n        vector.extend([1,0])\n        \n    return np.array(vector)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"A = {} #Stores slope value for each of the patient\nTAB = {} #Stores training data wrt each patient\nP = [] #Stores all unique patient id's\n\nfor i,p in enumerate(train.Patient.unique()):\n    sub = train.loc[train.Patient == p, :]\n    fvc = sub.FVC.values\n    week = sub.Weeks.values\n    c = np.vstack([week, np.ones(len(week))]).T\n    a, b = np.linalg.lstsq(c,fvc)[0]\n    \n    A[p] = a # Contains slope\n    TAB[p] = get_tab(sub) #Contains gender and smoking feature\n    P.append(p) #contains unique id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating CNN architecture for coeficient prediction:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize(d.pixel_array/2**11 ,(512,512))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x, y = [], []\n# for p in tqdm(train.Patient.unique()):\n#     try:\n#         ldir = os.listdir(f'osic-pulmonary-fibrosis-progression-lungs-mask/mask_noise/mask_noise/{p}/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class IGenerator(Sequence):\n    \n    ''' \n    This is the generator class, which generates an input of batch size 32\n    i.e 32 patient's 2 dicom image, and features from tabular data is generated. As output \n    from his generator x and y contains pixel_data of a dicom image, tab conatins patient's meta\n    information, and 'a' is the coeffiecient wrt each patient. \n    '''\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys, a, tab, batch_size=16):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.a = a\n        self.tab = tab\n        self.batch_size = batch_size\n        \n        self.train_data = {}\n        for p in train.Patient.values:\n            self.train_data[p] = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n    \n    def __len__(self):\n        return 1000\n    \n    def __getitem__(self, idx):\n        x, y = [], []\n        a, tab = [], [] \n        keys = np.random.choice(self.keys, size = self.batch_size)\n        \n        for k in keys:\n            try:\n                i = np.random.choice(self.train_data[k], size=1)[0]\n                j = np.random.choice(self.train_data[k], size=1)[0]\n                img1 = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                img2 = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{j}')\n                \n                x.append(img1)\n                y.append(img2)\n                \n                a.append(self.a[k])\n                tab.append(self.tab[k])\n            except:\n                print(k, i)\n        \n        \n        x,y,a,tab = np.array(x),np.array(y), np.array(a), np.array(tab)\n        x = np.expand_dims(x, axis=-1)\n        y = np.expand_dims(y, axis=-1)\n        return [x,y, tab] , a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_architechture(shape=(512,512,1)):\n    '''Architecture used here is inspired by this kaggle notebook \n    https://www.kaggle.com/miklgr500/linear-decay-based-on-resnet-cnn/notebook'''\n    \n    def res_block(x, filter_number):\n        _x = x\n        x = Conv2D(filter_number, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = BatchNormalization()(x)\n        x = LeakyReLU(0.05)(x)\n        x = Conv2D(filter_number, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = LeakyReLU(0.05)(x)\n        \n        x = Add()([_x, x])\n        return x\n    \n    #two input branch for images\n    input1 = Input(shape=shape, name= 'dicom_image_1')\n    input2 = Input(shape=shape, name= 'dicom_image_2')\n    \n    #image input branch 1 begins\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(input1)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(0.05)(x)\n    \n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(0.05)(x)\n    \n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    #image input branch 2 begins\n    y = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(input2)\n    y = BatchNormalization()(y)\n    y = LeakyReLU(0.05)(y)\n    \n    y = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(y)\n    y = BatchNormalization()(y)\n    y = LeakyReLU(0.05)(y)\n    \n    y = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(y)\n    \n    #Concatinating image inputs\n    x_and_y = Concatenate()([x, y])\n    \n    x_and_y = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x_and_y)\n    x_and_y = BatchNormalization()(x_and_y)\n    x_and_y = LeakyReLU(0.05)(x_and_y)\n    \n    x_and_y = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x_and_y)\n    for _ in range(2):\n        x_and_y = res_block(x_and_y, 16)\n    x_and_y = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x_and_y)\n    \n    x_and_y = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x_and_y)\n    for _ in range(3):\n        x_and_y = res_block(x_and_y, 64)\n    x_and_y = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x_and_y)    \n    \n    x_and_y = Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same')(x_and_y)\n    for _ in range(1):\n        x_and_y = res_block(x_and_y, 128)\n        \n   \n    x_and_y = GlobalAveragePooling2D()(x_and_y)\n    \n    #Patient tabular data input\n    input3 = Input(shape=(4,))\n    z = tf.keras.layers.GaussianNoise(0.2)(input3)\n    xyz = Concatenate()([x_and_y, z])\n    xyz = Dropout(0.6)(xyz) \n    xyz = Dense(1)(xyz)\n    return Model([input1, input2, input3] , xyz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model_architechture()\n\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), loss='mae') \n\ntr_p, vl_p = train_test_split(P, shuffle=True, train_size= 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nkeras.utils.plot_model(model,'img.png', show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"er = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=1e-3,\n    patience=5,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=True,\n)\n\nmodel.fit_generator(IGenerator(keys=tr_p, \n                               a = A, \n                               tab = TAB), \n                    steps_per_epoch = 200,\n                    validation_data=IGenerator(keys=vl_p, \n                               a = A, \n                               tab = TAB),\n                    validation_steps = 20, \n                    callbacks = [er], \n                    epochs=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('best_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model = tf.keras.models.load_model('./best_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(fvc_true, fvc_pred, sigma):\n    sigma_clip = np.maximum(sigma,70)\n    delta = np.abs(fvc_true - fvc_pred)\n    delta = np.minimum(delta,1000)\n    sqrt = np.sqrt(2)\n    metric = (delta/sigma_clip)*sqrt + np.log(sigma_clip*sqrt)\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm\n\nmetric = []\nfor q in tqdm(range(1, 10)):\n    m = []\n    for p in vl_p:\n        x, y = [], []\n        tab = [] \n        \n        if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n            continue\n            \n        img_set = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n        img_set = np.random.choice(img_set, size=20)\n        for i in img_set:\n            x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}')) \n            y.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))\n            tab.append(get_tab(train.loc[train.Patient == p, :])) \n        tab = np.array(tab) \n    \n        x = np.expand_dims(x, axis=-1)\n        y = np.expand_dims(y, axis=-1)\n        _a = model.predict([x,y, tab]) \n        a = np.quantile(_a, q / 10)\n        \n        percent_true = train.Percent.values[train.Patient == p]\n        fvc_true = train.FVC.values[train.Patient == p]\n        weeks_true = train.Weeks.values[train.Patient == p]\n        \n        fvc = a * (weeks_true - weeks_true[0]) + fvc_true[0]\n        percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])\n        m.append(score(fvc_true, fvc, percent))\n    print(np.mean(m))\n    metric.append(np.mean(m))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"q = (np.argmin(metric) + 1)/ 10\nq","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv') \nsub.head() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv') \ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"A_test, B_test, P_test,W, FVC= {}, {}, {},{},{} \nSTD, WEEK = {}, {} \nfor p in test.Patient.unique():\n    x,y = [],[]\n    tab = [] \n    img_set = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/')\n    img_set = np.random.choice(img_set, size=20)\n    for i in img_set:\n        x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/{i}')) \n        y.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))\n        tab.append(get_tab(test.loc[test.Patient == p, :])) \n    tab = np.array(tab) \n            \n    x = np.expand_dims(x, axis=-1) \n    y = np.expand_dims(y, axis=-1) \n    _a = model.predict([x,y, tab]) \n    a = np.quantile(_a, q)\n    A_test[p] = a\n    B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]\n    P_test[p] = test.Percent.values[test.Patient == p] \n    WEEK[p] = test.Weeks.values[test.Patient == p]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k in sub.Patient_Week.values:\n    p, w = k.split('_')\n    w = int(w) \n    \n    fvc = A_test[p] * w + B_test[p]\n    sub.loc[sub.Patient_Week == k, 'FVC'] = fvc\n    sub.loc[sub.Patient_Week == k, 'Confidence'] = (\n        P_test[p] - A_test[p] * abs(WEEK[p] - w) \n) \nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def get_model(shape=(512, 512, 1)):\n    def res_block(x, n_features):\n        _x = x\n        x = BatchNormalization()(x)\n        x = LeakyReLU(0.05)(x)\n    \n        x = Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = Add()([_x, x])\n        return x\n    \n    input1 = Input(shape=shape, name= 'dicom_image_1')\n    #input2 = Input(shape=shape, name= 'dicom_image_2')\n    \n    # 512\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(input1)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(0.05)(x)\n    \n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(0.05)(x)\n    \n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 256\n    x = Conv2D(8, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 8)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 128\n    x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 16)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 64\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 32)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 32\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)    \n    \n    # 16\n    x = Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 128)\n        \n    # 16\n    x = GlobalAveragePooling2D()(x)\n    \n    inp2 = Input(shape=(4,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2]) \n    x = Dropout(0.6)(x) \n    x = Dense(1)(x)\n    #x2 = Dense(1)(x)\n    return Model([input1, inp2] , x)\n"},{"metadata":{},"cell_type":"markdown","source":"# Defining model"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"def get_efficientnet(model, shape):\n    models_dict = {\n        'b0': efn.EfficientNetB0(input_shape=shape, weights=None,include_top=False),\n        'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),\n        'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),\n        'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),\n        'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),\n        'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),\n        'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),\n        'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)\n        }\n    return models_dict[model]\n\ndef build_model(shape=(512,512,1), model_class=None):\n    inp = Input(shape=shape)\n    base = get_efficientnet(model_class, shape)\n    x = base(inp)\n    x = GlobalAveragePooling2D()(x)\n    inp2 = Input(shape=(4,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x,x2])\n    x = Dropout(0.5)(x)\n    x = Dense(1)(x)\n    model = Model([inp,inp2],x)\n    return model"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"P = np.array(P)\nsubs = []\nfolds_history = []\n\ner = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=1e-3,\n    patience=10,\n    verbose=1,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=True,\n)\n\ncpt = tf.keras.callbacks.ModelCheckpoint(\n    filepath=f'effnet_{EPOCHS}.h5',\n    monitor='val_loss',\n    verbose=1,\n    save_best_only = SAVE_BEST,\n    mode = 'auto'\n)\n\nrlp = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.5,\n    patience=5,\n    verbose=1,\n    min_lr=1e-8\n)\nmodel = build_model(model_class = MODEL_CLASS)\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=LR), loss='mae')\nhistory = model.fit_generator(IGenerator(keys=P,\n                                        a= A,\n                                        tab= TAB),\n                             steps_per_epoch = 32,\n                             validation_data = IGenerator(keys=P,\n                                                         a = A,\n                                                         tab = TAB),\n                             validation_steps = 16,\n                             callbacks = [cpt,rlp],\n                             epochs = EPOCHS)\nfolds_history.append(history.history)\nprint('Trainig Done!')"},{"metadata":{},"cell_type":"markdown","source":"\n<a href='./training_1'> Download File</a>"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}