{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom PIL import Image\nimport pydicom\nimport matplotlib.pyplot as plt\nimport pylab\nimport cv2\nfrom tensorflow.keras.utils import Sequence\n#import gdcm\nfrom tqdm import tqdm\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.regularizers as R\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 5\nNUM_IMAGES = 150\nBATCH_SIZE = 4\nFOLDS = 5\nIMAGE_DIM = (NUM_IMAGES,55,55)\nCOMP_DIR = '../input/osic-pulmonary-fibrosis-progression/'\nTRAIN_PATH = '../input/osic-pulmonary-fibrosis-progression/train'\nTEST_PATH = '../input/osic-pulmonary-fibrosis-progression/test'\nSUB_PATH = '../input/osic-pulmonary-fibrosis-progression/sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"comp_dir = '../input/osic-pulmonary-fibrosis-progression'\n\ntrain_data = pd.read_csv(os.path.join(comp_dir,'train.csv'))\nsub = pd.read_csv(os.path.join(comp_dir,'sample_submission.csv'))\ntest_data = pd.read_csv(os.path.join(comp_dir,'test.csv'))\ntrain_data.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_u = train_data\ntrain_data_u = train_data_u.drop_duplicates(subset=['Patient'])\ntrain_data_u = train_data_u.rename(columns={'Weeks':'Base_Week','FVC':'Base_FVC','Percent':'Base_Percent'})\ntrain_data_u['Typical_FVC'] = (train_data_u.Base_FVC.values/train_data_u.Base_Percent.values)*100\ntrain_data = train_data.merge(train_data_u.drop(['Age','Sex','SmokingStatus'],axis=1),on='Patient',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub = sub.drop(['Confidence'],axis=1)\nsub =  sub[['Patient','Weeks','Patient_Week']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = test_data.rename(columns={'Weeks':'Base_Week','FVC':'Base_FVC','Percent':'Base_Percent'})\ntest_data['Typical_FVC'] = (test_data.Base_FVC.values/test_data.Base_Percent.values)*100\nsub = sub.merge(test_data, how='left', on='Patient')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data['Type'] = 'train'\nsub['Type'] = 'test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train_data.append(sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_col = [\"FVC\"]\nContinuos_cols = [\"Weeks\",\"Base_Week\",\"Base_FVC\",\"Typical_FVC\",\"Age\",\"Percent\",\"Base_Percent\"]\nCategorical_cols = ['Sex','Smoking_status']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = MinMaxScaler()\nconti = scaler.fit_transform(data[Continuos_cols])\ndata[Continuos_cols] = conti","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.mean(train_data_u.query('SmokingStatus == \\'Never smoked\\'').Base_Percent.values))\nprint(np.mean(train_data_u.query('SmokingStatus == \\'Currently smokes\\'').Base_Percent.values))\nprint(np.mean(train_data_u.query('SmokingStatus == \\'Ex-smoker\\'').Base_Percent.values))\nprint(np.mean(train_data_u.query('Sex == \\'Male\\'').Base_Percent.values))\nprint(np.mean(train_data_u.query('Sex == \\'Female\\'').Base_Percent.values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sex_m = np.zeros((len(data['Sex'].values),1))\nsex_f = np.zeros((len(data['Sex'].values),1))\nsm_es = np.zeros((len(data['Sex'].values),1))\nsm_ns = np.zeros((len(data['Sex'].values),1))\nsm_cs = np.zeros((len(data['Sex'].values),1))\nfor i in range(len(data['Sex'].values)):\n    if data['Sex'].values[i] == 'Male':\n        sex_m[i] = 1\n    elif data['Sex'].values[i] == 'Female':\n        sex_f[i] = 1\nfor i in range(len(data['SmokingStatus'].values)):\n    if data['SmokingStatus'].values[i] =='Ex-smoker':\n        sm_es[i] = 1\n    elif data['SmokingStatus'].values[i] =='Never smoked':\n        sm_ns[i] = 1\n    else:\n        sm_cs[i] = 1\n\ndata['sex_m'] = sex_m\ndata['sex_f'] = sex_f\ndata['sm_es'] = sm_es\ndata['sm_ns'] = sm_ns\ndata['sm_cs'] = sm_cs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_cols = ['Weeks','Base_Week','Base_FVC','Age','sex_m','sex_f','sm_es','sm_ns','sm_cs']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = data[x_cols].loc[data['Type'] == \"train\"].values.astype(np.float)\ny_train = data[prediction_col].loc[data['Type'] == \"train\"].values.astype(np.float)\nx_test = data[x_cols].loc[data['Type'] == \"test\"].values.astype(np.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.astype(np.float32)\ny_train = y_train.astype(np.float32)\nx_test = x_test.astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape,y_train.shape,x_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(x_train[0][3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(x_train),type(y_train),type(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Data_Generator(tf.keras.utils.Sequence):\n    \n    def __init__(self,batch_size,patient_ids,tab_data,dim,target=None,train=True,augment=False):\n        self.batch_size = batch_size\n        self.image_ids = patient_ids\n        self.augment = augment\n        self.dim = dim\n        self.target = target\n        self.indices = range(len(self.image_ids))\n        self.train = train\n        self.tab_data = tab_data\n        #self.on_epoch_end()\n    \n    def getimage(self,image_id):\n        X1 = np.zeros((NUM_IMAGES,self.dim[1],self.dim[2], 1))\n        if self.train:\n            path = TRAIN_PATH\n        else:\n            path = TEST_PATH\n        im_num = len(os.listdir(os.path.join(path,image_id)))\n        if im_num < NUM_IMAGES+1:\n            for i,dcm_i in enumerate(os.listdir(os.path.join(path,image_id))):\n                try:\n                    im = pydicom.dcmread(os.path.join(TRAIN_PATH,f'{image_id}/{dcm_i}'))\n                    img = im.pixel_array/255\n                    img = cv2.resize(img, (self.dim[1],self.dim[2]))\n                    img = np.reshape(img,(IMAGE_DIM[1],IMAGE_DIM[2],1))\n                    X1[i,] = img\n                    if i>=NUM_IMAGES-1:\n                        break\n                except:\n                    continue\n        else:\n            val = (im_num - NUM_IMAGES)//2\n            dir_list = os.listdir(os.path.join(path,image_id))\n            dir_list.sort()\n            for i,dcm_i in enumerate(dir_list[val:]):\n                try:\n                    im = pydicom.dcmread(os.path.join(TRAIN_PATH,f'{image_id}/{dcm_i}'))\n                    img = im.pixel_array/255\n                    img = cv2.resize(img, (self.dim[1],self.dim[2]))\n                    img = np.reshape(img,(IMAGE_DIM[1],IMAGE_DIM[2],1))\n                    X1[i,] = img\n                    if i>=NUM_IMAGES-1:\n                        break\n                except:\n                    continue\n               \n        if self.augment == True:\n            img = self.ImageAugment(img)\n            return img\n        return X1\n    \n    def on_epoch_end(self):\n        return self.indices\n    \n    def getdata(self, image_id_list):\n        X = np.empty((self.batch_size,*self.dim, 1))\n        for i, im_id in enumerate(image_id_list):\n            #print(i)\n            X[i,] = self.getimage(im_id)\n        \n        return X\n    '''\n    def ImageAugment(self,image):\n        augmentor = ImageAugmentor(image,axis_point=[self.dim/2,self.dim/2])\n        augmentor.cutmix()\n        #augmentor.zoom()\n        augmentor.flip()\n        augmentor.rotate()\n        return augmentor.get_image()\n    ''' \n    \n    def __getitem__(self,index):\n        indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n        \n        image_id_list = [self.image_ids[k] for k in indices]\n        tab_X = np.array([self.tab_data[k] for k in indices]).astype(np.float32)\n        X = self.getdata(image_id_list)\n        if self.train == True:\n            target_list = [self.target[k] for k in indices]\n            y = np.array(target_list).astype(np.float32)\n            return X,y\n        return X\n    \n    def __len__(self):\n        return int(np.floor(len(self.indices)/self.batch_size))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:,2]-y_pred[:,0]\n    fvc_pred = y_pred[:,1]\n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:,0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2,0.50,0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e,(q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = data[x_cols].loc[data['Type'] == \"train\"].values\ny_train = data[prediction_col].loc[data['Type'] == \"train\"].values\nx_test = data[x_cols].loc[data['Type'] == \"test\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dense = M.load_model('../input/tab-data-osic/dense_model.h5',custom_objects={'loss':mloss,'score':score})\nmodel_dense.summary()\nmodel_cnn = M.load_model('../input/tab-data-osic/model.h5',custom_objects={'loss':mloss,'score':score})\nmodel_cnn.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_patient_ids = data['Patient'].loc[data['Type'] == \"test\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1,6):\n    if len(test_patient_ids)%i ==0:\n        TEST_BATCH_SIZE = i \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen = Data_Generator(TEST_BATCH_SIZE,test_patient_ids,x_test,IMAGE_DIM,train=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Inferencing\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(test_patient_ids))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_dense = model_dense.predict(x_test,verbose=1)\npred_cnn = model_cnn.predict(test_gen,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conf_dense = pred_dense[:,2] - pred_dense[:,0]\nconf_cnn = pred_cnn[:,2] - pred_cnn[:,0]\nfor i in range(len(conf_cnn)):\n    conf_dense[i] = max(conf_dense[i],70)\n    conf_cnn[i] = max(conf_cnn[i],70)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(pred_dense),len(pred_cnn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = pred_dense*0.8 + pred_cnn*0.2\nconf = conf_dense*0.8 + conf_cnn*0.2\npred_dict = {'FVC':pred[:,1],'Confidence':conf}\npred_df = pd.DataFrame(pred_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Confidence'] = pred_df['Confidence']\nsub['FVC'] = pred_df['FVC']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm = sub[['Patient_Week','FVC','Confidence']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}