{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pulmonary Fibrosis EDA 🏥💊"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pandas_profiling import ProfileReport","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')\nprint('Train Data:')\nprint(train.head())\n\ntest=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')\nprint('\\n\\nTest Data:')\nprint(test.head())\n\nsub=pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nprint('\\n\\nSubmission File:')\nprint(sub.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Some patients had their FVC value tested before the CT Scan(negative values for Weeks)"},{"metadata":{"trusted":true},"cell_type":"code","source":"ProfileReport(train,progress_bar=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ProfileReport(test,progress_bar=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ProfileReport(sub,progress_bar=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. EDA with Visualization"},{"metadata":{},"cell_type":"markdown","source":"## # Patients Readings 😷"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('No of unique patients:',len(train.Patient.unique()))\n\nreadings=train.groupby('Patient').Weeks.count()\nprint('Min no. of readings for a patient:', min(readings))\nprint('Max no. of readings for a patient:', max(readings))\n\nfig=plt.figure(figsize=(15,5))\nsns.barplot(readings.index,readings,color='#7AC8BE')\nplt.title('Number of Readings per Patient',size=15)\nplt.xlabel('Patient',size=12)\nplt.ylabel('# Readings',size=12)\nplt.xticks([])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## # Patients Details 👴"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Age\nprint('Minimum aged patient:',min(train['Age']))\nprint('Maximum aged patient:',max(train['Age']))\n\nfig=plt.figure(figsize=(10,5))\nsns.distplot(train['Age'])\nplt.title('Age Distribution',size=15)\nplt.xlabel('Age',size=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Sex\nsex=train.groupby('Patient').Sex.first()\nprint('Male Patients:',sex.value_counts()[0])\nprint('Female Patients:',sex.value_counts()[1])\n\nfig=plt.figure(figsize=(5,5))                                              \nsns.countplot(sex)\nplt.title('Sex Distribution',size=15)\nplt.ylabel('# Patients',size=12)\nplt.xlabel('Sex',size=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Smoking status\nsmoke=train.groupby('Patient').SmokingStatus.first()\nprint('Ex-smokers:',smoke.value_counts()[0])\nprint('Patients who never smoked:',smoke.value_counts()[1])\nprint('Patients who currently smoke:',smoke.value_counts()[2])\n\nfig=plt.figure(figsize=(5,5))                                              \nsns.countplot(smoke)\nplt.title('Smoking Status',size=15)\nplt.ylabel('# Patients',size=12)\nplt.xlabel('Status',size=12)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### * Forced vital capacity (FVC) is the amount of air that can be forcibly exhaled from your lungs after taking the deepest breath possible. The recorded lung capacity in ml."},{"metadata":{"trusted":true},"cell_type":"code","source":"#FVC value\nprint('Maximum FVC value:',max(train['FVC']))\nprint('Minimum FVC value:',min(train['FVC']))\n\nfig=plt.figure(figsize=(10,5))\nsns.distplot(train['FVC'])\nplt.title('FVC Value Distribution',size=15)\nplt.xlabel('FVC Value',size=12)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### * Percent - a computed field which approximates the patient's FVC as a percent of the typical FVC for a person of similar characteristics."},{"metadata":{"trusted":true},"cell_type":"code","source":"#Percent\nprint('Maximum Percentage:',max(train['Percent']))\nprint('Minimum Percentage:',min(train['Percent']))\n\nfig=plt.figure(figsize=(10,5))\nsns.distplot(train['Percent'])\nplt.title('Percentage Distribution',size=15)\nplt.xlabel('Percent',size=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Scatterplot to check correlations\na=train[['Age','SmokingStatus','Percent']]\nfig=plt.figure(figsize=(15,5))\nfor i in range(len(a.columns)):\n    fig.add_subplot(1,3,i+1)\n    sns.scatterplot(x=a.iloc[:,i],y=train['FVC'],hue=train['Sex'],palette=['blue','red'])\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Age and Smoking Status has no correlation with the FVC value.\n* Percent and FVC are highly correlated"},{"metadata":{},"cell_type":"markdown","source":"# 3. Dicom Data 📁"},{"metadata":{},"cell_type":"markdown","source":"-References: \n* https://www.kaggle.com/gzuidhof/full-preprocessing-tutorial\n* https://www.kaggle.com/allunia/pulmonary-dicom-preprocessing\n\nThanks to Guido Zuidhof (@gzuidhof) and Laura Fink (@allunia) for these really insightful notebooks. "},{"metadata":{},"cell_type":"markdown","source":"## # Reading the Metadata 🗃"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom as dicom\nimport cv2\n\ndata_dir='../input/osic-pulmonary-fibrosis-progression/train'\npatients=os.listdir(data_dir)\nlabels_df=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv',index_col=0)\n#labels_df=labels_df[['FVC']]\nlabels_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Viewing the metadata of the dicom file\n\nfor patient in patients[:1]:\n    label=labels_df.loc[patient,'FVC']\n    path=data_dir+'/'+patient\n    slices=[dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    print('No. of scans:',len(slices))\n    print('Height and width of the scan:',slices[0].pixel_array.shape)\n    print('\\nMetadata of the Dicom File:')\n    print(slices[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###     From the above data we can see that:\n* This patient has 258 images of his/her ct scan\n* Each image is a 512x512 pixel image"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Viewing the ct scan size for 5 different patients\nc=0\nfor patient in patients:\n    try:\n        label=labels_df.loc[patient,'FVC']\n        path=data_dir+'/'+patient\n        slices=[dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n        print(len(slices),slices[0].pixel_array.shape)\n        c+=1\n        if c==5:\n            break\n    except:\n        continue","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Each patient has the same ct scan size of 512x512\n* But the no. of scans for each patient is different\n* So we need to resize each image to the same size to feed into our model"},{"metadata":{"trusted":true},"cell_type":"code","source":"min_s=9999\nmax_s=0\nfor patient in patients[:]:\n    label=labels_df.loc[patient,'FVC']\n    path=data_dir+'/'+patient\n    slices=[len(s) for s in os.listdir(path)]\n    if len(slices)<min_s:\n        min_s=len(slices)\n    if len(slices)>max_s:\n        max_s=len(slices)\nprint('Minimum number of scans for any patient:',min_s)\nprint('Maximum number of scans for any patient:',max_s)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization📷"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Single Frame 2D Visualization for a patient\nimport cv2\n\nfor patient in patients[1:2]:\n    label=labels_df.loc[patient,'FVC']\n    path=data_dir+'/'+patient\n    slices=[dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    \n    fig=plt.figure(figsize=(5,5))\n    plt.axis('off')\n    plt.title('CT Scan',size=15)\n    plt.imshow(slices[0].pixel_array,cmap='gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#2D Visualization of all the scans for a patient\nimport cv2\n\nfor patient in patients:\n    label=labels_df.loc[patient,'FVC']\n    path=data_dir+'/'+patient\n    slices=[dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    \n    img_px_size=150\n    \n    try:\n        if len(slices)<=56:\n            fig=plt.figure(figsize=(20,40))\n            for num,each_slice in enumerate(slices):\n                fig.add_subplot(14,4,num+1)\n                new_image=cv2.resize(np.array(each_slice.pixel_array),(img_px_size,img_px_size))\n                plt.axis('off')\n                plt.title(num+1,size=10)\n                plt.imshow(new_image,cmap='gray')\n            plt.show()\n            break\n    except:\n        continue","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## # HU (Hounsfield Scale) Values"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the scans in given folder path\ndef load_scan(path):\n    slices = [dicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    try:\n        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n    except:\n        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n        \n    for s in slices:\n        s.SliceThickness = slice_thickness\n        \n    return slices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_pixels_hu(slices):\n    image = np.stack([s.pixel_array for s in slices])\n    # Convert to int16 (from sometimes int16), \n    # should be possible as values should always be low enough (<32k)\n    image = image.astype(np.int16)\n\n    # Set outside-of-scan pixels to 0\n    # The intercept is usually -1024, so air is approximately 0\n    image[image == -2000] = 0\n    \n    # Convert to Hounsfield units (HU)\n    for slice_number in range(len(slices)):\n        \n        intercept = slices[slice_number].RescaleIntercept\n        slope = slices[slice_number].RescaleSlope\n        \n        if slope != 1:\n            image[slice_number] = slope * image[slice_number].astype(np.float64)\n            image[slice_number] = image[slice_number].astype(np.int16)\n            \n        image[slice_number] += np.int16(intercept)\n    \n    return np.array(image, dtype=np.int16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_patient = load_scan(data_dir + '/' + patients[0])\nfirst_patient_pixels = get_pixels_hu(first_patient)\nplt.hist(first_patient_pixels.flatten(), bins=80, color='c')\nplt.xlabel(\"Hounsfield Units (HU)\")\nplt.ylabel(\"Frequency\")\nplt.show()\n\n# Show some slice in the middle\nplt.imshow(first_patient_pixels[80], cmap=plt.cm.gray)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Preprocessing 📝"},{"metadata":{},"cell_type":"markdown","source":"### # Drop Duplicates "},{"metadata":{"trusted":true},"cell_type":"code","source":"#Drop duplicate values from the training dataset\ndrop=train[train.duplicated(subset=['Patient','Weeks'],keep='last')]\nprint('No. of rows to be dropped:',drop.shape[0])\ntrain.drop_duplicates(subset=['Patient','Weeks'],keep='last',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Not many duplicate values are present in the dataset.\n* We keep the last value and drop all the previous iterations."},{"metadata":{},"cell_type":"markdown","source":"### # Splitting the Submission File and Concatenating"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Split Patient_Week Column from the submission file\nsub[['Patient','Weeks']]=sub.Patient_Week.str.split(\"_\",expand = True)\nsub=sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Merging submission file and test file\nsub=sub.merge(test.drop('Weeks',axis = 1),on=\"Patient\")\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Introduce a column to indicate the source dataset for the data\n#Merge train and test data\ntrain['Dataset']='train'\nsub['Dataset']='test'\n\ndata=train.append([sub])\ndata.reset_index(inplace = True,drop=True)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Conveting categorical data to numerical data and dropping the categorical columns\n#Conversion\ndata = pd.concat([\n    data,\n    pd.get_dummies(data.Sex),\n    pd.get_dummies(data.SmokingStatus)\n],axis=1)\n\n#Dropping\ndata.drop(['Sex','SmokingStatus'],axis=1,inplace=True)\ndata['Weeks']=data['Weeks'].astype('int64')\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Getting the baseline week as every patient had thier first at different points of time w.r.t their CT scans\ndef get_baseline(df):  \n    _df=df.copy()\n    _df['min_week']=_df['Weeks']\n    # as test data is containing all weeks \n    _df.loc[_df.Dataset=='test','min_week']=0\n    _df[\"min_week\"]=_df.groupby('Patient')['Weeks'].transform('min')\n    _df['baselined_week']=_df['Weeks']-_df['min_week']\n    \n    return _df   \n\n\ndata['Weeks']=data['Weeks'].astype('int64')\ndata=get_baseline(data)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_baseline_FVC(df):\n    # same as above\n    _df = df.copy()\n    base = _df.loc[_df.Weeks == _df.min_week]\n    base = base[['Patient','FVC']].copy()\n    base.columns = ['Patient','base_FVC']\n    \n    # add a row which contains the cumulated sum of rows for each patient\n    base['nb'] = 1\n    base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\n    \n    # drop all except the first row for each patient (= unique rows!), containing the min_week\n    base = base[base.nb == 1]\n    base.drop('nb', axis = 1, inplace = True)\n    \n    # merge the rows containing the base_FVC on the original _df\n    _df = _df.merge(base, on = 'Patient', how = 'left')    \n    _df.drop(['min_week'], axis = 1)\n    \n    return _df\n\ndata=get_baseline_FVC(data)\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Scaling Features\ndef scaling(series):\n    return (series-series.min())/(series.max()-series.min())\n\ndata['Age']=scaling(data['Age'])\ndata['Percent']=scaling(data['Percent'])\ndata['baselined_week']=scaling(data['baselined_week'])\ndata['base_FVC']=scaling(data['base_FVC'])\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Model "},{"metadata":{},"cell_type":"markdown","source":"* Reference: https://www.kaggle.com/reighns/higher-lb-score-by-tuning-mloss-around-6-811\n\nThanks to Hongnan Gao (@reighns) for his notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow_addons.layers import WeightNormalization\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Lambda, Input\nfrom tensorflow.keras.models import Sequential, Model\n\n# create constants for the loss function\nC1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n\n# define competition metric\ndef score(y_true, y_pred):\n    \"\"\"Calculate the competition metric\"\"\"\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    sigma_clip = tf.maximum(sigma, C1)\n    # Python is automatically broadcasting y_true with shape (1,0) to \n    # shape (3,0) in order to make this subtraction work\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype = tf.float32) )\n    metric = (delta / sigma_clip) * sq2 + tf.math.log(sigma_clip * sq2)\n    return K.mean(metric)\n\n# define pinball loss\ndef qloss(y_true, y_pred):\n    \"\"\"Calculate Pinball loss\"\"\"\n    # IMPORTANT: define quartiles, feel free to change here!\n    qs = [0.2, 0.50, 0.8]\n    q = tf.constant(np.array([qs]), dtype = tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q * e, (q-1) * e)\n    return K.mean(v)\n\n# combine competition metric and pinball loss to a joint loss function\ndef mloss(_lambda):\n    \"\"\"Combine Score and qloss\"\"\"\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda) * score(y_true, y_pred)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model(nh):\n    z = Input((nh,), name=\"Patient\")\n    x = Dense(100, activation=\"elu\", name=\"d1\")(z)\n    x = Dense(100, activation=\"elu\", name=\"d3\")(x)\n    p1 = Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = Dense(3, activation=\"elu\", name=\"p2\")(x)\n    preds = Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = Model(z,preds,name=\"CNN\")\n    model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## GET TRAINING DATA AND TARGET VALUE\n\n# get back original data split\nfeatures_list=['baselined_week', 'Percent', 'Age', 'base_FVC', 'Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']\ntrain=data.loc[data.Dataset == 'train']\nsub=data.loc[data.Dataset == 'test']\n\n# get target value\ny=train['FVC'].values.astype(float)\n\n# get training & test data\nX_train=train[features_list].values\nX_test=sub[features_list].values\nn_rows=X_train.shape[1]\n\n# instantiate target arrays\ntrain_preds=np.zeros((X_train.shape[0], 3))\ntest_preds=np.zeros((X_test.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=make_model(n_rows)\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\nfrom keras import backend as K\n\nreduce_lr_loss=tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',factor=0.4,patience=150,verbose=0,epsilon=1e-4,mode='min')\n\nNFOLD = 6\nkf = KFold(n_splits=NFOLD)\nOOF_val_score=[]\n\ncnt = 0\nBATCH_SIZE=128\nEPOCHS = 800\nfor tr_idx, val_idx in kf.split(X_train):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    model=make_model(n_rows)\n    history=model.fit(X_train[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, \n            validation_data=(X_train[val_idx], y[val_idx]), verbose=0, callbacks=[reduce_lr_loss])\n    print(\"train\", model.evaluate(X_train[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", model.evaluate(X_train[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    train_preds[val_idx]=model.predict(X_train[val_idx],batch_size=BATCH_SIZE, verbose=0)\n    \n    # append OOF evaluation to calculate OFF_Score\n    OOF_val_score.append(model.evaluate(X_train[val_idx], y[val_idx], verbose = 0, batch_size = BATCH_SIZE, return_dict = True)['score'])\n    \n    print(\"predict test...\")\n    test_preds+=model.predict(X_test, batch_size=BATCH_SIZE, verbose=0)/NFOLD","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Evaluation and Submission ✅🏁"},{"metadata":{"trusted":true},"cell_type":"code","source":"# fetch results from history\nscore = history.history['score']\nval_score = history.history['val_score']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(EPOCHS)\n\n# create subplots\nplt.figure(figsize = (20,5))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, score, label = 'Training Accuracy')\nplt.plot(epochs_range, val_score, label = 'Validation Accuracy')\n# limit y-values for better zoom-scale. Remember that roughly -4.5 is the best possible score\n# plt.ylim(0.8 * np.mean(val_score), 1.2 * np.mean(val_score))\nplt.legend(loc = 'lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label = 'Training Loss')\nplt.plot(epochs_range, val_loss, label = 'Validation Loss')\n# limit y-values for beter zoom-scale\nplt.ylim(0.3 * np.mean(val_loss), 1.8 * np.mean(val_loss))\n\nplt.legend(loc = 'upper right')\nplt.title('Training and Validation Loss')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(OOF_val_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## FIND OPTIMIZED STANDARD-DEVIATION\nsigma_opt = mean_absolute_error(y, train_preds[:,1])\nsigma_uncertain = train_preds[:,2] - train_preds[:,0]\nsigma_mean = np.mean(sigma_uncertain)\nprint(sigma_opt, sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## PREPARE SUBMISSION FILE WITH OUR PREDICTIONS\nsub['FVC1'] = test_preds[:, 1]\nsub['Confidence1'] = test_preds[:,2] - test_preds[:,0]\n\n# get rid of unused data and show some non-empty data\nsubmission = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()\nsubmission.loc[~submission.FVC1.isnull()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.loc[~submission.FVC1.isnull(),'FVC'] = submission.loc[~submission.FVC1.isnull(),'FVC1']\n\nif sigma_mean < 70:\n    submission['Confidence'] = sigma_opt\nelse:\n    submission.loc[~submission.FVC1.isnull(),'Confidence'] = submission.loc[~submission.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"org_test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\n\nfor i in range(len(org_test)):\n    submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'FVC'] = org_test.FVC[i]\n    submission.loc[submission['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'Confidence'] = 70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}