{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Training EfficientNet based on \nhttps://www.kaggle.com/khoongweihao/efficientnets-quantile-regression-inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index\n!pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport tensorflow as tf \nimport matplotlib.pyplot as plt\nimport keras\nimport pydicom\nimport tqdm\nimport cv2\nfrom tqdm.notebook import tqdm\nfrom tensorflow.keras import Model\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow_addons.optimizers import RectifiedAdam\nfrom tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)\nfrom tensorflow.keras.models import Model\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv') \nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tab(df):\n    vector = [(df.Age.values[0] - 30) / 30] \n    \n    if df.Sex.values[0] == 'male':\n        vector.append(0)\n    else:\n        vector.append(1)\n    \n    if df.SmokingStatus.values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df.SmokingStatus.values[0] == 'Ex-smoker':\n        vector.extend([1,1])\n    elif df.SmokingStatus.values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    else:\n        vector.extend([1,0])\n    return np.array(vector) \n\n\nA = {} \nTAB = {} \nP = [] \nfor i, p in tqdm(enumerate(train.Patient.unique())):\n    sub = train.loc[train.Patient == p, :] \n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    A[p] = a\n    TAB[p] = get_tab(sub)\n    P.append(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (512, 512))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as Alb\n\naugs = {'Original': None,\n             'Blur': Alb.Blur(p=1.0),\n             #'MedianBlur': A.MedianBlur(blur_limit=5, p=1.0),\n             'GaussianBlur': Alb.GaussianBlur(p=1.0),\n             'MotionBlur': Alb.MotionBlur(p=1.0),\n        'GridDropout': Alb.GridDropout(p=1.0),\n        #'CenterCrop': A.CenterCrop(height=256, width=256, p=1.0),\n        #'RandomRotate90': A.RandomRotate90(p=1.0),\n        # 'ShiftScaleRotate': A.ShiftScaleRotate(p=1.0),\n        #'Rotate': A.Rotate()\n       }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimage = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/9.dcm')\nprint(\"Real SHape = \",image.shape)\nfor ite,(key, aug) in enumerate(augs.items()):\n    if aug is not None:\n        image = aug(image=image)['image']\n        print(\"New Shape = \",image.shape)\n        plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, y = [], []\nfor p in tqdm(train.Patient.unique()):\n    try:\n        ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression-lungs-mask/mask_noise/mask_noise/{p}/')\n        numb = [float(i[:-4]) for i in ldir]\n        for i in ldir:\n            x.append(cv2.imread(f'../input/osic-pulmonary-fibrosis-progression-lungs-mask/mask_noise/mask_noise/{p}/{i}', 0).mean())\n            y.append(float(i[:-4]) / max(numb))\n    except:\n        pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\n\nclass IGenerator(Sequence):\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys, a, tab, batch_size=32):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.a = a\n        self.tab = tab\n        self.batch_size = batch_size\n        \n        self.train_data = {}\n        for p in train.Patient.unique():\n            ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n            numb = [float(i[:-4]) for i in ldir]\n            self.train_data[p] = [i for i in os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/') \n                                  if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15]\n    \n    def __len__(self):\n        return 1000\n    \n    def __getitem__(self, idx):\n        x = []\n        a, tab = [], [] \n        keys = np.random.choice(self.keys, size = self.batch_size)\n        for k in keys:\n            try:\n                i = np.random.choice(self.train_data[k], size=1)[0]\n                image = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                for ite,(key, aug) in enumerate(augs.items()):\n                    if aug is not None:\n                        image = aug(image=image)['image']\n                        x.append(image)\n                        a.append(self.a[k])\n                        tab.append(self.tab[k])\n            except:\n                print(k, i)\n       \n        x,a,tab = np.array(x), np.array(a), np.array(tab)\n        #print(len(x),len(a),len(tab))\n        x = np.expand_dims(x, axis=-1)\n        return [x, tab] , a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(shape=(512,512,1), model_class=None):\n    inp = Input(shape=shape)\n    base = efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False)\n    base.trainable = False\n    x = base(inp)\n    x = GlobalAveragePooling2D()(x)\n    inp2 = Input(shape=(4,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2]) \n    x = Dropout(0.5)(x) \n    x = Dense(1)(x)\n    model = Model([inp, inp2] , x)\n    return model\n\nmodel = build_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), loss='mae')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split \ntr_p, vl_p = train_test_split(P,shuffle=True,train_size= 0.8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"er = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=1e-3,\n    patience=10,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=True,\n)\n\ncheckpoint_path = \"../input/output/training_1/weights{epoch:08d}.h5\"\ncheckpoint_dir = os.path.dirname(checkpoint_path)\n\n# Create a callback that saves the model's weights\ncp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n                                                 save_weights_only=True,\n                                                 verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(IGenerator(keys=tr_p, \n                               a = A, \n                               tab = TAB), \n                    steps_per_epoch = 500,\n                    validation_data=IGenerator(keys=vl_p, \n                               a = A, \n                               tab = TAB),\n                    validation_steps = 40, \n                    callbacks = [er,cp_callback], \n                    epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import DenseNet121\ndensenet = DenseNet121(\n    weights= None,\n    include_top=False,\n    input_shape=(512,512,1)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_densenet_model(densenet,shape=(512,512,1)):\n    inp = Input(shape=shape)\n    #base = efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False)\n    densenet.trainable = False\n    x = densenet(inp)\n    x = GlobalAveragePooling2D()(x)\n    inp2 = Input(shape=(4,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2]) \n    x = Dropout(0.5)(x) \n    x = Dense(1)(x)\n    model = Model([inp, inp2] , x)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_densenet_model(densenet)\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0005), loss='mae')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"densenet_path = \"../input/output/densenet/weights{epoch:08d}.h5\"\nnew_cp = tf.keras.callbacks.ModelCheckpoint(filepath=densenet_path,\n                                                 save_weights_only=True,\n                                                 verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(IGenerator(keys=tr_p, \n                               a = A, \n                               tab = TAB), \n                    steps_per_epoch = 500,\n                    validation_data=IGenerator(keys=vl_p, \n                               a = A, \n                               tab = TAB),\n                    validation_steps = 40, \n                    callbacks = [new_cp], \n                    epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}