{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport pydicom\n\nplt.style.use(\"dark_background\")\n\nmain_dir = \"../input/osic-pulmonary-fibrosis-progression\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_imgs = tf.io.gfile.glob(main_dir + \"/train/*/*\")\ntest_files = tf.io.gfile.glob(main_dir+\"/test/*/*\")\ndf_train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.isna().sum().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking to see which columns change with time.\n(df_train.groupby('Patient').nunique() != 1).sum() == 0 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = pydicom.dcmread(train_imgs[1])\ntype(temp)\ntemp","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"elements are stored as tuple key value pairs."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('\\n'.join(str(temp).split('\\n')[:15]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# temp.keys()\nlist(temp.values())[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp.dir()[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#displaying the image\nplt.figure(figsize=(10,10))\nplt.axis('off')\nplt.imshow(temp.pixel_array,cmap = 'gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's Explore the dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['SmokingStatus'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ages = df_train.groupby('Patient').Age.head(1)\nprint('Maximum Patient Age: {} \\n Minimum Patient Age: {}'.format(ages.max(),ages.min()))\nax = ages.plot(kind='hist', bins=50, edgecolor='red', color='y', figsize=(15, 5), xticks=range(49, 89))\nages.plot(kind='kde', ax=ax, xlim=(47, 90), color='w', secondary_y=True);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f,ax = plt.subplots(figsize=(15,5) ,ncols=2)\n\ndf_train.groupby('Patient')['SmokingStatus'].head(1).value_counts().plot(kind='pie', ax=ax[0], autopct= lambda x: str(int(x))+\"%\",\ntitle = 'Smoking Status pie Chart', colors = ['orange', 'blue', 'green'])\n\ndf_train.groupby('Patient')['Sex'].head(1).value_counts().plot(kind='pie', ax=ax[1],autopct= lambda x: str(int(x))+\"%\",\ntitle = 'Sex pie Chart', colors = ['red', 'green'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.groupby(['SmokingStatus', 'Sex'])['Patient'].nunique().unstack().plot(\n    kind='bar', stacked=True, figsize=(10, 6), yticks=range(0, 130, 10),\n    rot=0, title='Gender Across Smoking Status')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.groupby(['SmokingStatus','Sex'])[['Weeks', 'FVC', 'Percent', 'Age']].agg({\n    'Weeks': 'count',\n    'FVC' : ['min', 'max', 'mean', 'std'],\n    'Age': ['min', 'max', 'mean', 'std'],\n    'Percent' : ['min', 'max', 'mean', 'std']}).rename({'Weeks': \"cumulative Records\"}, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.signal import savgol_filter\n\ndef display_FVC_progress(data, title, smooth=True, drop=1, median=True):\n    agg = ['count', 'min', 'max', 'median']\n    if not median:\n        agg.remove('median')\n    \n    temp = data.groupby('Weeks')[['FVC']].agg(agg)\n    temp = temp[temp['FVC']['count'] > drop].drop(('FVC', 'count'), axis=1)\n    \n    if smooth:\n        temp['FVC', 'max'] = savgol_filter(temp['FVC', 'max'], 9, 3)\n        temp['FVC', 'min'] = savgol_filter(temp['FVC', 'min'], 9, 3)\n\n    ax = temp.plot(\n        figsize=(15, 5), \n        title=f'Variation & progress of FVC over the Weeks ({title})', \n        legend=True, xticks=range(-10, 150, 5)\n    );\n\n    ax.fill_between(temp.index, temp['FVC', 'max'], temp['FVC', 'min'], color='green');\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_FVC_progress(df_train, 'All Categories')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_FVC_progress(df_train[df_train['Sex']== 'Male'], 'Only Males' , smooth = True, median=False)\ndisplay_FVC_progress(df_train[df_train['Sex']== 'Female'], 'Only Females' , smooth = True, median=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_FVC_progress(df_train[df_train['SmokingStatus']== 'Currently smokes'], 'Current Smokers' , smooth = True, median=False)\ndisplay_FVC_progress(df_train[df_train['SmokingStatus']== 'Ex-smoker'], 'Ex-Smokers' , smooth = True, median=False)\ndisplay_FVC_progress(df_train[df_train['SmokingStatus']== 'Never smoked'], 'Never Smoked' , smooth = True, median=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Building"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\nfrom tqdm.notebook import tqdm \nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow_addons.optimizers import RectifiedAdam\nfrom tensorflow.keras import Model\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom tensorflow.keras.optimizers import Nadam\nimport seaborn as sns\nfrom PIL import Image\nimport cv2\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tab(df):\n    vector = [(df['Age'].values[0]-30)/30]\n            \n    if df['Sex'].values[0] == 'Male':\n        vector.append(0)\n    else:\n        vector.append(1)\n    \n    if df['SmokingStatus'].values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df['SmokingStatus'].values[0] == 'Ex-smoker':\n        vector.extend([1,1])\n    elif df['SmokingStatus'].values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    else:\n        vector.extend([1,0])\n    return np.array(vector)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"A = {}\nTAB = {}\nP = []\n\nfor i, p in tqdm(enumerate(df_train.Patient.unique())):\n    sub = df_train.loc[df_train.Patient == p, :]\n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a,b = np.linalg.lstsq(c, fvc)[0]\n    \n    A[p] = a\n    TAB[p] = get_tab(sub)\n    P.append(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize(d.pixel_array/ 2**11, (512,512))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\n\n\nclass IGenerator(Sequence):\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys, a, tab, batch_size=32):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.a = a\n        self.tab = tab\n        self.batch_size = batch_size\n        self.train_data = {}\n        for p in df_train.Patient.values:\n            self.train_data[p] = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n            \n    def __len__(self):\n        return 1000\n    def __getitem__(self, idx):\n        x = []\n        a, tab = [], []\n        keys = np.random.choice(self.keys, size = self.batch_size)\n        for k in keys:\n            try:\n                i = np.random.choice(self.train_data[k], size=1)[0]\n                img = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                x.append(img)\n                a.append(self.a[k])\n                tab.append(self.tab[k])\n            except:\n                print(k,i)\n                \n        x,a,tab = np.array(x), np.array(a), np.array(tab)\n        x = np.expand_dims(x, axis = -1)\n        return [x, tab], a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import (Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D,\n                                    Add, Conv2D, AveragePooling2D, LeakyReLU, Concatenate)\nimport efficientnet.tfkeras as efn\n\n\ndef get_efficientnet(model, shape):\n    models_dict = {\n        'b0' : efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),\n        'b1' :efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),\n        'b2' :efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),\n        'b3' :efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),\n        'b4' :efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),\n        'b5' :efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),\n        'b6' :efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),\n        'b7' :efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)        \n    }\n    \n    return models_dict[model]\n\n\ndef build_model(shape=(512, 512, 1), model_class = None):\n    inp = Input(shape = shape)\n    base = get_efficientnet(model_class, shape)\n    x = base(inp)\n    x = GlobalAveragePooling2D()(x)\n    inp2 = Input(shape=(4,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2])\n    x = Dropout(0.35)(x)\n    x = Dense(1)(x)\n    model = Model([inp, inp2], x)\n    \n    weights = [w for w  in os.listdir('../input/osic-model-weights') if model_class in w][0]\n    model.load_weights('../input/osic-model-weights/' + weights)\n    return model\n\nmodel_classes = ['b5']\nmodels = [build_model(shape=(512, 512, 1), model_class = m) for m in model_classes]\nprint('Number of Models: '+str(len(models)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(list(A.values()));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(fvc_true, fvc_pred, sigma):\n    sigma_clip = np.maximum(sigma, 70)\n    delta = np.abs(fvc_true - fvc_pred)\n    delta = np.minimum(delta, 1000)\n    sq2 = np.sqrt(2)\n    metric = (delta / sigma_clip)*sq2 + np.log(sigma_clip*sq2)\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subs = []\nfor model in models:\n    metric = []\n    for q in tqdm(range(1, 10)):\n        m = []\n        for p in vl_p:\n            x = [] \n            tab = [] \n\n            if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n                continue\n\n            ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n            for i in ldir:\n                if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n                    x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}')) \n                    tab.append(get_tab(df_train.loc[df_train.Patient == p, :])) \n            if len(x) < 1:\n                continue\n            tab = np.array(tab) \n\n            x = np.expand_dims(x, axis=-1) \n            _a = model.predict([x, tab]) \n            a = np.quantile(_a, q / 10)\n\n            percent_true = df_train.Percent.values[df_train.Patient == p]\n            fvc_true = df_train.FVC.values[df_train.Patient == p]\n            weeks_true = df_train.Weeks.values[df_train.Patient == p]\n\n            fvc = a * (weeks_true - weeks_true[0]) + fvc_true[0]\n            percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])\n            m.append(score(fvc_true, fvc, percent))\n        print(np.mean(m))\n        metric.append(np.mean(m))\n\n    q = (np.argmin(metric) + 1)/ 10\n\n    sub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv') \n    test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv') \n    A_test, B_test, P_test,W, FVC= {}, {}, {},{},{} \n    STD, WEEK = {}, {} \n    for p in test.Patient.unique():\n        x = [] \n        tab = [] \n        ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/')\n        for i in ldir:\n            if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n                x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/{i}')) \n                tab.append(get_tab(test.loc[test.Patient == p, :])) \n        if len(x) <= 1:\n            continue\n        tab = np.array(tab) \n\n        x = np.expand_dims(x, axis=-1) \n        _a = model.predict([x, tab]) \n        a = np.quantile(_a, q)\n        A_test[p] = a\n        B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]\n        P_test[p] = test.Percent.values[test.Patient == p] \n        WEEK[p] = test.Weeks.values[test.Patient == p]\n\n    for k in sub.Patient_Week.values:\n        p, w = k.split('_')\n        w = int(w) \n\n        fvc = A_test[p] * w + B_test[p]\n        sub.loc[sub.Patient_Week == k, 'FVC'] = fvc\n        sub.loc[sub.Patient_Week == k, 'Confidence'] = (\n            P_test[p] - A_test[p] * abs(WEEK[p] - w) \n    ) \n\n    _sub = sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].copy()\n    subs.append(_sub)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = len(subs)\nsub = subs[0].copy() # ref\nsub[\"FVC\"] = 0\nsub[\"Confidence\"] = 0\nfor i in range(N):\n    sub[\"FVC\"] += subs[0][\"FVC\"] * (1/N)\n    sub[\"Confidence\"] += subs[0][\"Confidence\"] * (1/N)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}