{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"Dropout_model = 0.385\nFVC_weight = 0.2\nConfidence_weight = 0.2\n!pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index\n!pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index\nimport os\nimport cv2\nimport pydicom\nimport pandas as pd\nimport numpy as np \nimport tensorflow as tf \nimport matplotlib.pyplot as plt \nimport random\nfrom tqdm.notebook import tqdm \nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow_addons.optimizers import RectifiedAdam\nfrom tensorflow.keras import Model\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom tensorflow.keras.optimizers import Nadam\nimport seaborn as sns\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(1)\nconfig = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)\ntrain = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_age_max = train.Age.values.max()\ntr_age_min = train.Age.values.min()\ntr_age_mean = train.Age.values.mean()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def expandFVC_Percent(DS):\n    DS['FVC_Percent'] = DS['FVC'] / DS['Percent']\n    DS['FVC_Percent'] = (DS['FVC_Percent'] - DS['FVC_Percent'].min()) / (DS['FVC_Percent'].max() - DS['FVC_Percent'].min())\n    return DS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def expandMinFVC(data_set):\n    data_set['MinWeeks'] = data_set.groupby('Patient')['Weeks'].transform('min')\n    sub_tr = data_set.loc[data_set['Weeks']==data_set['MinWeeks']]\n\n    sub_tr['MinFVC'] = sub_tr['FVC']\n\n    sub_tr = sub_tr[['Patient', 'MinFVC']]\n\n    merged = data_set.merge(sub_tr, on='Patient', how='outer')\n\n\n    data_set=merged\n    \n    data_set['MinFVC'] = (data_set['MinFVC'] - data_set['MinFVC'].min())/(data_set['MinFVC'].max()-data_set['MinFVC'].min())\n    print(data_set.sample(5))\n    return data_set\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = expandMinFVC(train)\n# train = expandFVC_Percent(train)\n\nprint(train.head())\nprint(train.sample(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_tab(df):\n    # df: data frame\n    # [age, male/female[0/1], SmokingStatus[00/11/01], BaseFvc] (5,)\n    vector = [(df.Age.values[0] - tr_age_min) / (tr_age_max-tr_age_min)] \n    \n    if df.Sex.values[0] == 'male':\n       vector.append(0)\n    else:\n       vector.append(1)\n    \n    if df.SmokingStatus.values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df.SmokingStatus.values[0] == 'Ex-smoker':\n        vector.extend([1,1])\n    elif df.SmokingStatus.values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    else:\n        vector.extend([1,0])\n    vector.extend([df['MinFVC'].values[0]])\n#     vector.extend([df['FVC_Percent'].values[0]])\n    return np.array(vector) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# s = f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}'\nprint(train.columns)\nk =  train.Patient[102]\nprint(k)\ni = 10\npath = f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}.dcm'\nd = pydicom.dcmread(path)\nprint(d.pixel_array/(2**11))\n# print(dir(d))\nprint(d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nA = {} \nTAB = {} \nP = [] \nfor i, p in tqdm(enumerate(train.Patient.unique())):\n    sub = train.loc[train.Patient == p, :] \n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    A[p] = a\n    TAB[p] = get_tab(sub)\n    P.append(p)\n\ndef get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize(d.pixel_array / 2**11, (512, 512))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\nimport math\n\nclass IGenerator(Sequence):\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys, a, tab, batch_size=4):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.a = a\n        self.tab = tab\n        self.batch_size = batch_size\n        \n        self.train_data = {}  # imgs for each person; self.trian_data[pi] = [img1, img2, ..., imgn]\n        for p in train.Patient.values:\n            # filenames in \"~/train/personid/\"\n            self.train_data[p] = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n    \n    def __len__(self):\n        return 100\n#         return math.ceil(len(self.keys) / float(self.batch_size)) \n    \n    def __getitem__(self, idx):\n        x = []\n        a, tab = [], [] \n        keys = np.random.choice(self.keys, size = self.batch_size)  # persons in batch size\n        for k in keys:\n            #  chose a {[img, tab], a}for each person (only 1 img)\n            try:\n                self.train_data[k].sort()\n                mx_len = len(self.train_data[k])\n                i = np.random.choice(self.train_data[k][mx_len//3:mx_len//3*2], size=1)[0]\n#                 i = self.train_data[k][max_len//2]\n                img = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                x.append(img)\n                a.append(self.a[k])\n                tab.append(self.tab[k])\n            except:\n                print(k, i)\n       \n        x, a, tab = np.array(x), np.array(a), np.array(tab)\n        x = np.expand_dims(x, axis=-1)\n        return [x, tab] , a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)\nimport efficientnet.tfkeras as efn\n\ndef get_efficientnet(model, shape):\n    models_dict = {\n        'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),\n        'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),\n        'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),\n        'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),\n        'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),\n        'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),\n        'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),\n        'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)\n    }\n    return models_dict[model]\n\ndef build_model(shape=(512, 512, 1), model_class=None):\n    inp = Input(shape=shape)\n    base = get_efficientnet(model_class, shape)\n    x = base(inp)\n    x = GlobalAveragePooling2D()(x)\n#     inp2 = Input(shape=(4,))  # original is 4\n    inp2 = Input(shape=(5,))  # add the feature of MinFVC\n#     inp2 = Input(shape=(6,))  # add the feature of MinFVC + FVC/Percent\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2]) \n    x = Dropout(Dropout_model)(x)\n    x = Dense(1)(x)\n    model = Model([inp, inp2] , x)\n    \n#     weights = [w for w in os.listdir('../input/osic-model-weights') if model_class in w][0]\n#     model.load_weights('../input/osic-model-weights/' + weights)\n    return model\n\ndef get_model(shape=(512, 512, 1)):\n    def res_block(x, n_features):\n        _x = x\n        x = BatchNormalization()(x)\n        x = LeakyReLU(0.05)(x)\n    \n        x = Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = Add()([_x, x])\n        return x\n    \n    inp = Input(shape=shape)\n    \n    # 512\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(inp)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(0.05)(x)\n    \n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = BatchNormalization()(x)\n    x = LeakyReLU(0.05)(x)\n    \n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 256\n    x = Conv2D(8, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 8)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 128\n    x = Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 16)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 64\n    x = Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 32)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 32\n    x = Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)    \n    \n    # 16\n    x = Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 128)\n        \n    # 16\n    x = GlobalAveragePooling2D()(x)\n    \n    inp2 = Input(shape=(5,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2]) \n    x = Dropout(0.6)(x) \n    x = Dense(1)(x)\n    #x2 = Dense(1)(x)\n    return Model([inp, inp2] , x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nmodel_classes = ['b3'] #['b0','b1','b2','b3',b4','b5','b6','b7']\nmodels = [build_model(shape=(512, 512, 1), model_class=m) for m in model_classes]\nprint('Number of models: ' + str(len(models)))\n\nmodel = models[0]\n\n# model = get_model()\nmodel.summary()\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss='mae')  # original is lr=0.001\n\nfrom sklearn.model_selection import train_test_split \ntr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8) \n\ner = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=1e-5\n    patience=5,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=True,\n)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(IGenerator(keys=tr_p, \n                               a = A, \n                               tab = TAB), \n#                     steps_per_epoch = 100,\n                    validation_data=IGenerator(keys=vl_p, \n                               a = A, \n                               tab = TAB),\n#                     validation_steps = 20, \n#                     callbacks = [er], \n                    epochs=300)\n\n# model.fit(Xtrain, Ytrain, batch_size = 32, epochs = 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights_path = \"./efn.ckpt\"\nif not os.path.exists(weights_path):\n    os.mkdir(weights_path)\n    \nmodel.save_weights(weights_path)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef score(fvc_true, fvc_pred, sigma):\n    sigma_clip = np.maximum(sigma, 70) # changed from 70, trie 66.7 too\n    delta = np.abs(fvc_true - fvc_pred)\n    delta = np.minimum(delta, 1000)\n    sq2 = np.sqrt(2)\n    metric = (delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)\n    return np.mean(metric)\nsubs = []\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split \ntr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8) \n\n\nmetric = []\nfor q in tqdm(range(1, 10)):\n    m = []\n    for p in vl_p:\n        x = [] \n        tab = [] \n\n        if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n            continue\n\n        ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n        for i in ldir:\n            if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n                x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}')) \n                tab.append(get_tab(train.loc[train.Patient == p, :])) \n        if len(x) < 1:\n            continue\n        tab = np.array(tab) \n\n        x = np.expand_dims(x, axis=-1) \n        _a = model.predict([x, tab]) \n        a = np.quantile(_a, q / 10)\n\n        percent_true = train.Percent.values[train.Patient == p]\n        fvc_true = train.FVC.values[train.Patient == p]\n        weeks_true = train.Weeks.values[train.Patient == p]\n\n        fvc = a * (weeks_true - weeks_true[0]) + fvc_true[0]\n        percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])\n#         m.append(score(fvc_true, fvc, percent))\n        m.append(score(fvc_true, fvc, np.sqrt(2) * (np.abs(fvc_true - fvc))))\n    print(np.mean(m))\n    metric.append(np.mean(m))\nq = (np.argmin(metric) + 1)/ 10\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(q)\nwith open('q.txt', 'w') as f:\n    f.write(str(q)+'\\n')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !ls\n# !rm -r checkpoint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = build_model(shape=(512, 512, 1), model_class='b0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mode.load_weights(weigths_path)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}