{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **Introduction:**\n\nThis notebook is dedicated to creating a submission for the https://www.kaggle.com/c/osic-pulmonary-fibrosis-progression competition\n\n**Acknowledgments**\n\nThis notebook contains ideas and code from just like so many fucking places"},{"metadata":{},"cell_type":"markdown","source":"Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\nimport pydicom\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(8675309)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"Importing train and test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nprint(train.head())\nprint(test.head())\nprint(sub.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Examining Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\n\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(test.drop('Weeks', axis=1), on=\"Patient\")\n\nprint(train.shape, test.shape, sub.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.info())\n\n# patient_ids = train.Patient.unique()\n\n# for i in range(0,10):  \n#     single_patient = train[train['Patient'] == patient_ids[i]]\n#     plt.plot(single_patient['Weeks'], single_patient['FVC'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Examining CT Images**"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path = '../input/osic-pulmonary-fibrosis-progression/'\nimage_files_list = []\nfor dirName, subdirList, fileList in os.walk(image_path):\n    for filename in fileList:\n        if \".dcm\" in filename.lower():\n            image_files_list.append(os.path.join(dirName,filename))\n            \n\nimage = pydicom.dcmread(image_files_list[0])\n\nplt.figure()\nplt.imshow(image.pixel_array, cmap=plt.cm.bone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clean and wranglinate the data\n\ntrain['WHERE'] = 'train'\ntest['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = train.append([test, sub])\n\n\ndata['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\n\nbase = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient', 'FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb == 1]\nbase.drop('nb', axis=1, inplace=True)\n\n\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base\n\n\nCOLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n        \n\ndata['age'] = (data['Age'] - data['Age'].min()) / (data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min()) / (data['min_FVC'].max() - data['min_FVC'].min())\ndata['week'] = (data['base_week'] - data['base_week'].min()) / (data['base_week'].max() - data['base_week'].min())\ndata['percent'] = (data['Percent'] - data['Percent'].min()) / (data['Percent'].max() - data['Percent'].min())\nFE += ['age','percent','week','BASE']\nprint(FE)\n\n\ntrain = data.loc[data.WHERE=='train']\ntest = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data\n\n\n\ntrain.shape, test.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n\n\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return tf.keras.backend.mean(metric)\n\n\ndef qloss(y_true, y_pred):\n    qs = [0.2, 0.50, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return tf.keras.backend.mean(v)\n\n\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n\n\ndef make_model(nh):\n    z = tf.keras.layers.Input((nh,), name=\"Patient\")\n    x = tf.keras.layers.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = tf.keras.layers.Dense(100, activation=\"relu\", name=\"d2\")(x)\n#     x = tf.keras.layers.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = tf.keras.layers.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = tf.keras.layers.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = tf.keras.layers.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = tf.keras.models.Model(z, preds, name=\"definitely_not_a_CNN\")\n    model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['FVC'].values\nz = train[FE].values\nze = sub[FE].values\nnh = z.shape[1]\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\n\nnet = make_model(nh)\nprint(net.summary())\nprint(net.count_params())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD = 5\nkf = KFold(n_splits=NFOLD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\nEPOCHS = 800\nBATCH_SIZE = 64\ndiff_sum = 0\n\nfor tr_idx, val_idx in kf.split(z):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    net = make_model(nh)\n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    train_loss, train_score = net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE)\n    print(f\"Train Loss: {train_loss}  Score: {train_score}\")\n    val_loss, val_score = net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE)\n    print(f\"Val Loss: {val_loss}  Score: {val_score}\")\n    score_diff = val_score - train_score\n    diff_sum += score_diff\n    print(f\"Score diff: {score_diff}\")\n    print(\"Predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"Predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n    \nprint(f\"Score diff sum : {diff_sum}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\n\n\nsub['FVC1'] = 0.996*pe[:, 1]\nsub['Confidence1'] = pe[:, 2] - pe[:, 0]\nsubm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()\n\n\nsubm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1\n    \n\nsubm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}