{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport pydicom\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Activation,Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D,LeakyReLU, ReLU, Concatenate, GaussianNoise,MaxPooling2D\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.optimizers import Nadam, Adam, SGD, Adadelta, Adamax\nfrom tensorflow.keras.applications import VGG19\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/osic-pulmonary-fibrosis-progression/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"items = os.listdir(path)\nprint(items)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(path + 'train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.SmokingStatus.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def changing_columns(df):\n    age_sex_smoke = [(df.Age.values[0]-30) / 30]\n    \n    if df['Sex'].values[0] == 'Male':\n        age_sex_smoke.append(0)\n    else:\n        age_sex_smoke.append(1)\n    \n    if df['SmokingStatus'].values[0]== 'Never smoked':\n        age_sex_smoke.extend([0,0])\n    elif df['SmokingStatus'].values[0]== 'Ex-smoker':\n        age_sex_smoke.extend([1,1])\n    elif df['SmokingStatus'].values[0]== 'Currently smokes':\n        age_sex_smoke.extend([0,1])\n    else :\n        age_sex_smoke.extend([1,0])\n    \n    return np.array(age_sex_smoke)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"changing_columns(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"A = {}\nT = {}\nP = []\n\nfor i,p in enumerate(train_df['Patient'].unique()):\n    sub = train_df.loc[train_df.Patient == p, :]\n   # print(sub)\n    fvc = sub['FVC'].values\n   # print(fvc)\n    weeks = sub['Weeks'].values\n   # print(weeks)\n    ver_stack= np.vstack([weeks,np.ones(len(weeks))]).T\n   # print(ver_stack)\n    a,b = np.linalg.lstsq(ver_stack, fvc)[0]\n   # print(a)\n   # print(b)\n    \n    A[p] = a\n    T[p] = changing_columns(sub)\n    P.append(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_S = 32\nshape = 299","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_im_from_dicom(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (512, 512))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class IGenerator(Sequence):\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys, a, tab):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.a = a\n        self.tab = tab\n        self.batch_size = BATCH_S\n        \n        self.train_data = {}\n        for p in train_df.Patient.unique():\n            ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n            numb = [float(i[:-4]) for i in ldir]\n            self.train_data[p] = [i for i in os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/') if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15]\n    \n    def __len__(self):\n        return 1000\n    def __getitem__(self, idx):\n        x = []\n        a, tab = [], [] \n        keys = np.random.choice(self.keys, size = self.batch_size)\n        for k in keys:\n            try:\n                i = np.random.choice(self.train_data[k], size=1)[0]\n                img = get_im_from_dicom(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                mask = cv2.resize(cv2.imread(f'../input/osic-pulmonary-fibrosis-progression-lungs-mask/mask_clear/mask_clear/{k}/{i[:-4]}.jpg', 0), (512, 512))> 0\n                img[~mask] = 0\n                x.append(img)\n                a.append(self.a[k])\n                tab.append(self.tab[k])\n            except:\n                print('ooooo, we have a problem')\n                print(k)\n                print(i)\n       \n        x,a,tab = np.array(x), np.array(a), np.array(tab)\n        x = np.expand_dims(x, axis=-1)\n        return [x, tab] , a\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def the_model(input_shape=(shape,shape,1)):\n   \n    vgg = VGG19(include_top = False, weights= None, input_shape= input_shape)\n   \n    inp_1 = Input(shape = input_shape)\n    \n    half_1 = vgg(inp_1)\n    half_1 = GlobalAveragePooling2D()(half_1)\n    \n    inp_2 = Input(shape=(4,))\n    half_2 = GaussianNoise(0.3)(inp_2)\n    \n    whole = Concatenate()([half_1, half_2])\n    whole = Dense(100, activation='relu')(whole)\n    whole = Dense(100, activation='relu')(whole)\n    whole = Dropout(0.2)(whole)\n    whole = Dense(10, activation='relu')(whole)\n    whole = Dropout(0.2)(whole)\n    whole = Dense(1)(whole)\n    \n    return Model([inp_1,inp_2], whole)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = the_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part, valid_part = train_test_split(P, shuffle= True, train_size= 0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = IGenerator(keys= train_part, a=A, tab=T)\nvalid_generator = IGenerator(keys= valid_part, a=A, tab=T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s_p_e = 200\nepochs = 25\nlearn_r = 0.001","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nearly_stopp = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",\n                                               min_delta=1e-3,patience=10,\n                                               verbose=1,mode=\"auto\",\n                                               baseline=None,\n                                               restore_best_weights=True)\n\nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', \n                                                 factor=0.2,\n                                                 patience=5, min_lr=0.0001)\n\nopt_1 = SGD(learning_rate=learn_r, momentum=0.9)\n\nopt_2 = Adam(learning_rate= learn_r)\n\nopt_3 = Nadam(learning_rate= learn_r)\n\nopt_4 = Adamax(learning_rate= learn_r, beta_1=0.9, beta_2=0.999, epsilon=1e-07)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer= opt_4, loss= 'mae')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist = model.fit_generator(train_generator, steps_per_epoch=s_p_e, validation_data= valid_generator,\n                   validation_steps=20, callbacks= [reduce_lr, early_stopp], epochs= epochs, workers= 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(fvc_true, fvc_pred, sigma):    \n    sigma_clip = np.maximum(sigma, 70)\n    delta = np.abs(fvc_true - fvc_pred)\n    delta = np.minimum(delta, 1000)\n    sq2 = np.sqrt(2)\n    metric = (delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmetric = []\nfor q in range(1, 10):\n    m = []\n    for p in valid_part:\n        x = [] \n        tab = [] \n        \n        if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n            continue\n            \n        ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n        for i in ldir:\n            if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n                x.append(get_im_from_dicom(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}')) \n                tab.append(changing_columns(train_df.loc[train_df.Patient == p, :])) \n        if len(x) < 1:\n            continue\n        tab = np.array(tab) \n    \n        x = np.expand_dims(x, axis=-1)\n        _a = model.predict([x, tab])\n        a = np.quantile(_a, q / 10)\n        \n        percent_true = train_df.Percent.values[train_df.Patient == p]\n        fvc_true = train_df.FVC.values[train_df.Patient == p]\n        weeks_true = train_df.Weeks.values[train_df.Patient == p]\n        \n        fvc = a * (weeks_true - weeks_true[0]) + fvc_true[0]\n        percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])\n        m.append(score(fvc_true, fvc, percent))\n    print(np.mean(m))\n    metric.append(np.mean(m))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"q = (np.argmin(metric) + 1)/ 10\nprint(q)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv') \nsub.head() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv') \ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"A_test, B_test, P_test,W, FVC= {}, {}, {},{},{} \nSTD, WEEK = {}, {} \nfor p in test.Patient.unique():\n    x = [] \n    tab = [] \n    ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/')\n    for i in ldir:\n        if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n            x.append(get_im_from_dicom(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/{i}')) \n            tab.append(changing_columns(test.loc[test.Patient == p, :])) \n    if len(x) <= 1:\n        continue\n    tab = np.array(tab) \n            \n    x = np.expand_dims(x, axis=-1) \n    _a = model.predict([x, tab]) \n    a = np.quantile(_a, q)\n    A_test[p] = a\n    B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]\n    P_test[p] = test.Percent.values[test.Patient == p] \n    WEEK[p] = test.Weeks.values[test.Patient == p]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k in sub.Patient_Week.values:\n    p, w = k.split('_')\n    w = int(w) \n    \n    fvc = A_test[p] * w + B_test[p]\n    sub.loc[sub.Patient_Week == k, 'FVC'] = fvc\n    sub.loc[sub.Patient_Week == k, 'Confidence'] = (\n        P_test[p] - A_test[p] * abs(WEEK[p] - w)\n)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}