{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport random\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.isotonic import IsotonicRegression\nfrom sklearn.utils import check_random_state\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\n\nimport tables\nfrom os import listdir\nimport glob\nimport tqdm\nfrom typing import Dict\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n\nimport plotly.express as px\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\n\nfrom colorama import Fore, Back, Style\nimport pydicom\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport seaborn as sns\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom tabulate import tabulate\nimport missingno as msno \nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\nfrom scipy.stats import pearsonr\n\nimport pydicom # for DICOM images\nfrom skimage.transform import resize\nimport copy\nimport re\n\nfrom glob import glob\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport scipy.ndimage\nfrom skimage import morphology\nfrom skimage import measure\nfrom skimage.transform import resize\nfrom sklearn.cluster import KMeans\nfrom plotly import __version__\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nfrom plotly.tools import FigureFactory as FF\nfrom plotly.graph_objs import *\ninit_notebook_mode(connected=True) \n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ncustom_colors = ['#74a09e','#86c1b2','#98e2c6','#f3c969','#f2a553', '#d96548', '#c14953']\nsns.palplot(sns.color_palette(custom_colors))\n\nfrom skimage.transform import resize\nsns.set_style(\"whitegrid\")\nsns.despine(left=True, bottom=True)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_lungmask(img, display=False):\n    row_size= img.shape[0]\n    col_size = img.shape[1]\n    \n    mean = np.mean(img)\n    std = np.std(img)\n    img = img-mean\n    img = img/std\n    \n    # Find the average pixel value near the lungs\n        # to renormalize washed out images\n    middle = img[int(col_size/5):int(col_size/5*4),int(row_size/5):int(row_size/5*4)] \n    mean = np.mean(middle)  \n    max = np.max(img)\n    min = np.min(img)\n    \n    # To improve threshold finding, I'm moving the \n    # underflow and overflow on the pixel spectrum\n    img[img==max]=mean\n    img[img==min]=mean\n    \n    # Using Kmeans to separate foreground (soft tissue / bone) and background (lung/air)\n    \n    kmeans = KMeans(n_clusters=2).fit(np.reshape(middle,[np.prod(middle.shape),1]))\n    centers = sorted(kmeans.cluster_centers_.flatten())\n    threshold = np.mean(centers)\n    thresh_img = np.where(img<threshold,1.0,0.0)  # threshold the image\n\n    # First erode away the finer elements, then dilate to include some of the pixels surrounding the lung.  \n    # We don't want to accidentally clip the lung.\n\n    eroded = morphology.erosion(thresh_img,np.ones([3,3]))\n    dilation = morphology.dilation(eroded,np.ones([8,8]))\n\n    labels = measure.label(dilation) # Different labels are displayed in different colors\n    label_vals = np.unique(labels)\n    regions = measure.regionprops(labels)\n    good_labels = []\n    for prop in regions:\n        B = prop.bbox\n        if B[2]-B[0]<row_size/10*9 and B[3]-B[1]<col_size/10*9 and B[0]>row_size/5 and B[2]<col_size/5*4:\n            good_labels.append(prop.label)\n    mask = np.ndarray([row_size,col_size],dtype=np.int8)\n    mask[:] = 0\n\n\n    #  After just the lungs are left, we do another large dilation\n    #  in order to fill in and out the lung mask \n    \n    for N in good_labels:\n        mask = mask + np.where(labels==N,1,0)\n    mask = morphology.dilation(mask,np.ones([10,10])) # one last dilation\n\n    if (display):\n        fig, ax = plt.subplots(3, 2, figsize=[12, 12])\n        ax[0, 0].set_title(\"Original\")\n        ax[0, 0].imshow(img, cmap='gray')\n        ax[0, 0].axis('off')\n        ax[0, 1].set_title(\"Threshold\")\n        ax[0, 1].imshow(thresh_img, cmap='gray')\n        ax[0, 1].axis('off')\n        ax[1, 0].set_title(\"After Erosion and Dilation\")\n        ax[1, 0].imshow(dilation, cmap='gray')\n        ax[1, 0].axis('off')\n        ax[1, 1].set_title(\"Color Labels\")\n        ax[1, 1].imshow(labels)\n        ax[1, 1].axis('off')\n        ax[2, 0].set_title(\"Final Mask\")\n        ax[2, 0].imshow(mask, cmap='gray')\n        ax[2, 0].axis('off')\n        ax[2, 1].set_title(\"Apply Mask on Original\")\n        ax[2, 1].imshow(mask*img, cmap='gray')\n        ax[2, 1].axis('off')\n        \n        plt.show()\n    return mask*img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_dir = \"../input/osic-pulmonary-fibrosis-progression/train/\"\npatient_names = os.listdir(patient_dir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_names.remove('ID00011637202177653955184')\n#patient_names.remove('ID00105637202208831864134')\npatient_names.remove('ID00052637202186188008618')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#np.shape(pixels_mean)\n\nimport h5py\n#filename = '../input/ctscans64x64/my_array.h5'\nfilename = '../input/64-org/org_scans_64_v4.h5'\n\n\nwith h5py.File(filename, \"r\") as f:\n    # List all groups\n    print(\"Keys: %s\" % f.keys())\n    a_group_key = list(f.keys())[0]\n\n    # Get the data\n    data_sample = list(f[a_group_key])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#np.mean(data_sample1[0]) == np.mean(data_sample1[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixels_mean = data_sample","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size_img = 64\ndf = pd.DataFrame([np.array(patient_names),np.array(pixels_mean).reshape(174,size_img*size_img)],['Patient','Pixels'])\n#df_names = pd.DataFrame(data=np.array(patient_names).flatten())\n\ndf = df.T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(df.Pixels)):\n    df.Pixels[i] = df.Pixels[i]/max(df.Pixels[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(df.Pixels[0].reshape(64,64))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(df.Pixels[0].reshape(64,64))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### SAVING THE CT-SCAN PIXELS AS AN HDF5 FILE\n#import numpy as np\n#pixels_mod\n#pixels\n#h5_file = tables.open_file('modified_scans_64.h5', mode='w', titel='many large arrays')\n#h5_file.create_array('/', 'my_array', pixels)\n#h5_file.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_dir_test = \"../input/osic-pulmonary-fibrosis-progression/test/\"\npatient_names_test  = os.listdir(patient_dir_test )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#patient_dir = \"../input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430\"\n\npixels1 = []\npixels_org = []\n\nfor i in range(0,len(patient_names_test)):\n    datasets = []\n    print(patient_names_test[i])\n\n    # First Order the files in the dataset\n    files = []\n    for dcm in list(os.listdir(patient_dir_test+patient_names_test[i])):\n        files.append(dcm) \n    files.sort(key=lambda f: int(re.sub('\\D', '', f)))\n\n    # Read in the Dataset\n    for dcm in files:\n        path = patient_dir_test+patient_names_test[i] + \"/\" + dcm\n        datasets.append(pydicom.dcmread(path))\n\n    imgs = []\n    for data in datasets:\n        img = resize(data.pixel_array, (64, 64), anti_aliasing=True)\n        imgs.append(img)\n        \n    pixels_org.append(np.mean(imgs,axis=0))\n\n    pixels_ = []\n    for i in range(0, len(imgs)):\n        resized_img = resize(datasets[i-1].pixel_array, (64, 64), anti_aliasing=True)\n        img = make_lungmask(resized_img)\n        pixels_.append(img)\n        \n    pixels1.append(np.mean(pixels_,axis=0))\n    if(i%10==0):\n        print(i, ' Patients are analyzed!')\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixels_mean_test = pixels_org","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size_img = 64\n\ndf_test = pd.DataFrame([np.array(patient_names_test),np.array(pixels_mean_test).reshape(len(patient_names_test),size_img*size_img)],['Patient','Pixels'])\ndf_test = df_test.T\n\nfor i in range(len(df_test)):\n    df_test.Pixels[i] = df_test.Pixels[i].flatten()\n    \nfor i in range(len(df_test.Pixels)):\n    df_test.Pixels[i] = df_test.Pixels[i]/max(df_test.Pixels[i])\n    \n\ndf_test2 = pd.DataFrame([np.array(patient_names_test),np.array(pixels_mean_test).reshape(len(patient_names_test),size_img*size_img)],['Patient','Pixels'])\ndf_test2 = df_test2.T\n\nfor i in range(len(df_test2)):\n    df_test2.Pixels[i] = df_test2.Pixels[i].flatten()\n    \nfor i in range(len(df_test2.Pixels)):\n    df_test2.Pixels[i] = df_test2.Pixels[i]/max(df_test2.Pixels[i])\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.loc[173] = ['ID00105637202208831864134'] + [df.loc[df.Patient == 'ID00067637202189903532242'].Pixels.values[0]]\ndf.loc[174] = ['ID00011637202177653955184'] + [df.loc[df.Patient == 'ID00342637202287526592911'].Pixels.values[0]]\ndf.loc[175] = ['ID00052637202186188008618'] + [df.loc[df.Patient == 'ID00165637202237320314458'].Pixels.values[0]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\nBATCH_SIZE= 128","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(f\"{ROOT}/train.csv\")\ntrain.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntest = pd.read_csv(f\"{ROOT}/test.csv\")\n\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(test.drop('Weeks', axis=1), on=\"Patient\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['WHERE'] = 'train'\ndf_test['WHERE'] = 'val'\ndf_test2['WHERE'] = 'test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ct_data = []\nct_data = df.append(df_test)\nct_data = ct_data.append(df_test2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['WHERE'] = 'train'\ntest['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = train.append([test, sub])\n\ndata['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['max_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','max_week'] = np.nan\ndata['max_week'] = data.groupby('Patient')['max_week'].transform('max')\n\ndata['DLCO'] = data.FVC\ndata['min_FEV1'] = data.FVC","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC','Percent','DLCO','min_FEV1']].copy()\nbase.columns = ['Patient','min_FVC','min_Percent','min_DLCO','min_FEV']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\ndata = data.merge(base, on='Patient', how='left')\n\n\nbase1 = data.loc[data.Weeks == data.max_week]\nbase1 = base1[['Patient','FVC','Percent']].copy()\nbase1.columns = ['Patient','max_FVC','max_Percent']\nbase1['nb'] = 1\nbase1['nb'] = base1.groupby('Patient')['nb'].transform('cumsum')\nbase1 = base1[base1.nb==1]\nbase1.drop('nb', axis=1, inplace=True)\ndata = data.merge(base1, on='Patient', how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.merge(ct_data, on=['Patient','WHERE'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['min_FVC1'] =  0.84 -0.3/data['min_FVC'] #0.84 -0.3/data['min_FVC']\ndata['FEV1'] =  0.84 -0.3/data['FVC'] #0.84 -0.3/data['min_FVC']\ndata['base_week'] = data['Weeks'] - data['min_week']\ndata['Neg_Age'] = -1/(data['Age'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\n\nmin_max_scaler = preprocessing.MinMaxScaler()\n\n\ndata['age'] = min_max_scaler.fit_transform(data['Age'].values.reshape(-1,1))\ndata['BASE'] = min_max_scaler.fit_transform(data['min_FVC'].values.reshape(-1,1))\ndata['week'] = min_max_scaler.fit_transform(data['base_week'].values.reshape(-1,1))\ndata['org_week'] = min_max_scaler.fit_transform(data['Weeks'].values.reshape(-1,1))\ndata['percent'] = min_max_scaler.fit_transform(data['Percent'].values.reshape(-1,1))\ndata['min_fev'] = min_max_scaler.fit_transform(data['min_FVC1'].values.reshape(-1,1))\ndata['neg_age'] = min_max_scaler.fit_transform(data['Neg_Age'].values.reshape(-1,1))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FE = ['Ex-smoker','org_week','Never smoked','week','percent','Currently smokes','age','Female','Male','BASE','min_fev','neg_age'] \nFE2 = ['Pixels'] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data.loc[data.WHERE=='train']\ntest = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Input, Dense, InputLayer, Flatten, Reshape, BatchNormalization\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.convolutional import MaxPooling1D\nfrom tensorflow.keras.layers import concatenate","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train['FVC'].values\ny = y.astype(float)\nz1 = train[FE].values\nze1 = sub[FE].values\n\nz = train[FE2].values\nze = sub[FE2].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_z = []\n\nfor j in range(len(z)):\n#    print(j)\n    final_z.append(np.squeeze([ x for x in z[j] if not (x==i).all()]))\n    \nfinal_ze = []\n\nfor j in range(len(ze)):\n#    print(j)\n    final_ze.append(np.squeeze([ x for x in ze[j] if not (x==i).all()]))\n    \n    \nz2 = np.squeeze(final_z)\nze2 = np.squeeze(final_ze)\n\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import initializers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Input, Add, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D, AveragePooling2D, MaxPooling2D, GlobalMaxPooling2D\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.50, 0.80]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\n\ndef make_model():\n    z1 = L.Input((len(FE),))\n    z2 = L.Input((64*64,), name=\"Patient\")\n\n    init = tf.keras.initializers.VarianceScaling(scale=0.1, mode='fan_in', distribution='uniform',seed=42)\n\n    x = L.Dense(100,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(z1)\n    x = L.Dense(100,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(x)\n\n    \n    x = M.Model(inputs=z1, outputs=x)\n    \n    ######################\n    \n    y = Reshape((64*64, 1))(z2)\n    \n    shortcut = y\n    \n    y = Conv1D(kernel_initializer=init, activation='relu', \n                    padding=\"same\", filters=4, kernel_size=8)(y)\n    y = Conv1D(kernel_initializer=init, \n                    padding=\"same\", filters=4, kernel_size=8)(y)\n\n    y = Add()([y, shortcut])\n    \n    y= Activation('relu')(y)\n\n\n    # Max pooling layer\n    y = MaxPooling1D(pool_size=4)(y)\n\n    # Flatten the current input for the fully-connected layers\n    y = Flatten()(y)\n\n    # Fully-connected layers\n    \n    y = L.Dense(256,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(y)\n    y = L.Dense(128,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(y)   \n    y = L.Dense(64,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(y)   \n    y = L.Dense(32,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(y)   \n    y = L.Dense(16,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(y)   \n    y = L.Dense(8,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(y)\n\n    y = M.Model(inputs=z2, outputs=y)\n\n\n    combined = concatenate([x.output, y.output])\n\n#    combined = L.Dense(128,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(combined)   \n#    combined = L.Dense(64,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(combined)   \n#    combined = L.Dense(3,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(combined)   \n\n    #    combined = L.Dense(10,kernel_initializer = init,bias_initializer='zeros', activation=\"relu\")(combined)   \n\n\n    \n    p1 = L.Dense(3, activation=\"relu\", name=\"p1\")(combined)\n    p2 = L.Dense(3, activation=\"linear\", name=\"p2\")(combined)\n\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = M.Model([z1, z2], preds, name=\"CNN\")\n    model.compile(loss=mloss(0.80), optimizer=tf.keras.optimizers.Adam(lr=0.11, beta_1=0.92, beta_2=0.998, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = make_model()\nprint(net.summary())\nprint(net.count_params())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nNFOLD = 5\nkf = KFold(n_splits=NFOLD)\n#kf = KFold(n_splits=NFOLD, shuffle=True, random_state=42)\nseed_everything(42)\n\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\n\nresults_t = []\nresults_v = []\n\ncnt = 0\nfor tr_idx, val_idx in kf.split(z1):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    net = make_model()\n    net.fit([z1[tr_idx],z2[tr_idx]], y[tr_idx], batch_size=BATCH_SIZE, epochs=800, \n            validation_data=([z1[val_idx],z2[val_idx]], y[val_idx]), verbose=0)#,callbacks=[early_stopping]) #\n    print(\"train\", net.evaluate([z1[tr_idx],z2[tr_idx]], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate([z1[val_idx],z2[val_idx]], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    results_t.append(net.evaluate([z1[tr_idx],z2[tr_idx]], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    results_v.append(net.evaluate([z1[val_idx],z2[val_idx]], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n\n    print(\"predict val...\")\n    pred[val_idx] = net.predict([z1[val_idx],z2[val_idx]], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n#    net.fit([z1,z2], y, batch_size=BATCH_SIZE, epochs=800, verbose=0)#,callbacks=[early_stopping]) #\n#    pe += net.predict([ze1,ze2], batch_size=BATCH_SIZE, verbose=0) / NFOLD\n#net = make_model()\nnet.fit([z1,z2], y, batch_size=BATCH_SIZE, epochs=800, verbose=0)#,callbacks=[early_stopping]) #\npe = net.predict([ze1,ze2], batch_size=BATCH_SIZE, verbose=0) #/ NFOLD","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('training loss + score: ',np.mean(results_t,axis=0))\nprint('validation loss + score: ',np.mean(results_v,axis=0))\nprint('Final loss + score: ',np.mean(results_t,axis=0)*np.mean(results_v,axis=0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"800 - +neg\ntraining loss + score:  [39.92542267  6.49224253]\n\nvalidation loss + score:  [48.23802948  6.69629803]\n\nFinal loss + score:  [1925.92371568   43.47399082]","metadata":{}},{"cell_type":"markdown","source":"600 - +neg\n\ntraining loss + score:  [40.14292603  6.49475489]\n\nvalidation loss + score:  [47.6293129   6.68711853]\n\nFinal loss + score:  [1911.97998425   43.43119575]","metadata":{}},{"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idxs = np.random.randint(0, y.shape[0], 100)\nplt.plot(y[idxs],'ro', label=\"ground truth\")\nplt.plot(pred[idxs, 0],lw=2, label=\"q25\")\nplt.plot(pred[idxs, 1],lw=2, label=\"q50\")\nplt.plot(pred[idxs, 2],lw=2, label=\"q75\")\nplt.legend(loc=\"best\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(unc.min(), unc.mean(), unc.max(), (unc>=0).mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(unc)\nplt.title(\"uncertainty in prediction\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['FVC1'] = 0.995  * pe[:, 1]\nsub['Confidence1'] = pe[:, 2] - pe[:, 0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull()].head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm.describe().T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"good =pd.read_csv(\"../input/aug30th/submission-14.csv\")\n\n\nplt.figure(figsize=(20,10))\nplt.plot(range(len(good.FVC)), good.FVC,'ro',ms=2,label='Best')\nplt.plot(range(len(subm.FVC)), subm.FVC,'bo',ms=2,label='Current')\n\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nplt.plot(range(len(good.Confidence)), good.Confidence,'ro',ms=2,label='Best')\nplt.plot(range(len(subm.Confidence)), subm.Confidence,'bo',ms=2,label='Current')\n\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}