{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index\n!pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list(range(5))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pydicom\nimport re\nimport cv2\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras.utils import Sequence\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold,GroupKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\ntr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data=tr\ndata['min_week'] = data['Weeks']\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\nbase = data.loc[data.Weeks == data.min_week].copy()\nbase['normal_FVC']=base.FVC/base.Percent*100\n\nbase = base[['Patient','FVC','normal_FVC','Percent']].copy()\nbase.columns = ['Patient','min_FVC','normal_FVC','base_percent']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base\nCOLS = ['Sex','SmokingStatus'] #,'Age'\nFE = [\"Patient\"]\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n#data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nnum_enc = StandardScaler()\nnum_cols_to_scale = ['base_week','base_percent','min_FVC','Age',\"min_week\"]\nnum_enc.fit(data[num_cols_to_scale])\ndata[num_cols_to_scale]= num_enc.transform(data[num_cols_to_scale])\ntrain_data=data[[\"Patient\",\"Male\",\"Female\",\"Ex-smoker\",\"Never smoked\",\"Currently smokes\",'base_week','base_percent','min_FVC','Age',\"min_week\",\"FVC\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\na=train_data.loc[train_data[\"Patient\"]=='ID00011637202177653955184' ]\nb=train_data.loc[train_data[\"Patient\"]=='ID00052637202186188008618' ]\n#train_data.drop(train_data[\"Patient\"]='ID00011637202177653955184' )\ntrain_data=train_data[(train_data[\"Patient\"]!= 'ID00011637202177653955184') & (train_data[\"Patient\"]!='ID00052637202186188008618' )]\ntrain_data.reset_index(drop=True,inplace=True)\n#train_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize(d.pixel_array  / 2**11, (384, 384))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# y=train_data.loc[:,\"FVC\"].values #\n\n# tab=train_data.iloc[:,1:-1].values #\n\n# pred = np.zeros((tab.shape[0], 1)) #","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" # Prepocessing (Test)"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv') \ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv') \n#sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.rename(columns={\"Weeks\": \"min_week\", \"FVC\": \"min_FVC\",\"Percent\":\"base_percent\"},inplace=True)\n#test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(test, on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['base_week'] = sub['Weeks'] - sub['min_week']\n#sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] \n\nfor col in COLS:\n    \n    for mod in tr[col].unique():\n        \n        sub[mod] = (sub[col] == mod).astype(int)\n#sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cols_to_scale = ['base_week','base_percent','min_FVC','Age',\"min_week\"]\n\nsub[num_cols_to_scale]= num_enc.transform(sub[num_cols_to_scale])\ntest_data=sub[[\"Patient\",\"Male\",\"Female\",\"Ex-smoker\",\"Never smoked\",\"Currently smokes\",'base_week','base_percent','min_FVC','Age',\"min_week\"]]\n#test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_data.iloc[:,1:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import layers as L\nfrom tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EFN = efn.EfficientNetB4\ndef build_model(shape=(384, 384, 1)):\n    # input layers \n    inp = tf.keras.layers.Input(shape=shape)\n    \n\n    base = EFN (input_shape=shape,weights=None,include_top=False)\n    x = base(inp)\n    x = GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1,activation='relu')(x)\n    #x = tf.keras.layers.Dense(1,activation='relu')(x)\n    inp2 = tf.keras.layers.Input(shape=(10,))\n    x = Concatenate()([x, inp2]) \n    \n    x = tf.keras.layers.Dense(64,activation='relu')(x)\n    \n    #x = tf.keras.layers.Dense(64,activation='relu')(inp2)\n    x = tf.keras.layers.Dense(64,activation='relu')(x)\n    x = tf.keras.layers.Dense(1,activation='relu')(x)\n    model = tf.keras.Model(inputs=[inp,inp2] ,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.MeanAbsoluteError()\n    #loss = tf.keras.losses.MeanSquaredError()\n    model.compile(optimizer=opt,loss=loss)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=build_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weight_path=\"../input/image-1-model-b4\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fold=0\n# print(weight_path + \"/fold-{}.h5\".format(fold))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#BATCH_SIZE=1\nBATCH_SIZE=int(len(sub)/5)\nBATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class IGenerator(Sequence):\n    def __init__(self, keys, test_data, batch_size=BATCH_SIZE):\n        self.keys = [k for k in keys ]\n        self.train_data = test_data\n        \n        self.tab=self.train_data.iloc[:,1:].values\n        self.batch_size = batch_size\n        \n\n    \n    def __len__(self):\n        return math.ceil(len(self.keys)/self.batch_size)\n    \n    def __getitem__(self, index):\n        x = []\n        a, tab = [], [] \n        key0 = keys[index*self.batch_size:(index+1)*self.batch_size]\n        #print(index)\n        for k in key0:\n            try:  \n                \n                patient=self.train_data.loc[k,\"Patient\"]\n                \n                \n                imgs= os.listdir(f'../input/osic-pulmonary-fibrosis-progression/test/{patient}/')\n                \n                tmp0=math.ceil(len(imgs)/2)-1\n                i0=np.random.choice([tmp0-1,tmp0,tmp0+1])\n                tmp=[]\n                for d in imgs:\n                    tmp.append(int(d[:-4]))\n                tmp.sort()\n                i1=tmp[i0]\n                filename =  str(i1) + \".dcm\"\n                img = get_img(f'../input/osic-pulmonary-fibrosis-progression/test/{patient}/{filename}') \n                x.append(img)\n                \n                tab.append(self.tab[k])\n                \n\n            except:\n                print(\"wrong\",k)\n       \n        x,a,tab = np.array(x), np.array(a), np.array(tab)\n        x = np.expand_dims(x, axis=-1)\n        return [x, tab]\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res=[]\nfor fold in range(5):\n    model.load_weights(weight_path + \"/fold-{}.h5\".format(fold))\n    print(fold)\n    #for i in range(len(sub)):\n#         print(i)\n#         patient=test_data.loc[i,\"Patient\"]\n#         imgs= os.listdir(f'../input/osic-pulmonary-fibrosis-progression/test/{patient}/')\n#         tmp0=math.ceil(len(imgs)/2)-1\n#         i0=np.random.choice([tmp0-1,tmp0,tmp0+1])\n#         tmp=[]\n#         for d in imgs:\n#             tmp.append(int(d[:-4]))\n#         tmp.sort()\n#         i1=tmp[i0]\n#         filename =  str(i1) + \".dcm\"\n#         img = get_img(f'../input/osic-pulmonary-fibrosis-progression/test/{patient}/{filename}') \n#         tab=test_data.iloc[i,1:].values\n#         tab = tab.astype('float64')\n#         tab=np.array(tab)\n#         img=np.array(img)\n#         img = np.expand_dims(img, axis=-1)\n        #keys=[i]\n    keys=list(range(len(sub)))\n    tmp=model.predict(IGenerator(keys,test_data))\n    res.append(np.squeeze(tmp))\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#np.array(res).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.plot(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res1=np.array(res).reshape(5,-1)\n#res1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res2=np.mean(res1,0)\n#res2.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plt.plot(res2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[\"FVC\"]=res2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[\"Confidence\"]=300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub1=sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]]\nsub1.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub1.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}